upd: import replies to own posts on *key

This commit is contained in:
Mar0xy 2023-11-13 13:07:49 +01:00
parent 9b3993a652
commit 6f76a3a1f8
No known key found for this signature in database
GPG key ID: 56569BBE47D2C828
3 changed files with 47 additions and 9 deletions

View file

@ -16,6 +16,7 @@ import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, Obj
import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js';
import type httpSignature from '@peertube/http-signature'; import type httpSignature from '@peertube/http-signature';
import type * as Bull from 'bullmq'; import type * as Bull from 'bullmq';
import { MiNote } from '@/models/Note.js';
@Injectable() @Injectable()
export class QueueService { export class QueueService {
@ -289,8 +290,8 @@ export class QueueService {
} }
@bindThis @bindThis
public createImportKeyNotesToDbJob(user: ThinUser, targets: string[]) { public createImportKeyNotesToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) {
const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel })); const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel, note }));
return this.dbQueue.addBulk(jobs); return this.dbQueue.addBulk(jobs);
} }

View file

@ -4,7 +4,7 @@ import { Inject, Injectable } from '@nestjs/common';
import { IsNull } from 'typeorm'; import { IsNull } from 'typeorm';
import { ZipReader } from 'slacc'; import { ZipReader } from 'slacc';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote } from '@/models/_.js'; import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository } from '@/models/_.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { DownloadService } from '@/core/DownloadService.js'; import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
@ -18,7 +18,7 @@ import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js'; import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq'; import type * as Bull from 'bullmq';
import type { DbNoteImportToDbJobData, DbNoteImportJobData } from '../types.js'; import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbKeyNoteImportToDbJobData } from '../types.js';
@Injectable() @Injectable()
export class ImportNotesProcessorService { export class ImportNotesProcessorService {
@ -31,6 +31,9 @@ export class ImportNotesProcessorService {
@Inject(DI.driveFilesRepository) @Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository, private driveFilesRepository: DriveFilesRepository,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private queueService: QueueService, private queueService: QueueService,
private utilityService: UtilityService, private utilityService: UtilityService,
private noteCreateService: NoteCreateService, private noteCreateService: NoteCreateService,
@ -66,6 +69,30 @@ export class ImportNotesProcessorService {
} }
} }
// Function was taken from Firefish and edited to remove renoteId and make it run in only one for loop instead of two
@bindThis
private async recreateChain(arr: any[]) {
type NotesMap = {
[id: string]: any;
};
const notesTree: any[] = [];
const lookup: NotesMap = {};
for await (const note of arr) {
lookup[`${note.id}`] = note;
note.childNotes = [];
let parent = null;
if (note.replyId == null) {
notesTree.push(note);
} else {
parent = lookup[`${note.replyId}`];
}
if (parent) parent.childNotes.push(note);
}
return notesTree;
}
@bindThis @bindThis
private isIterable(obj: any) { private isIterable(obj: any) {
if (obj == null) { if (obj == null) {
@ -195,7 +222,8 @@ export class ImportNotesProcessorService {
const notesJson = fs.readFileSync(path, 'utf-8'); const notesJson = fs.readFileSync(path, 'utf-8');
const notes = JSON.parse(notesJson); const notes = JSON.parse(notesJson);
this.queueService.createImportKeyNotesToDbJob(job.data.user, notes); const processedNotes = await this.recreateChain(notes);
this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null);
cleanup(); cleanup();
} }
@ -203,7 +231,7 @@ export class ImportNotesProcessorService {
} }
@bindThis @bindThis
public async processKeyNotesToDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> { public async processKeyNotesToDb(job: Bull.Job<DbKeyNoteImportToDbJobData>): Promise<void> {
const note = job.data.target; const note = job.data.target;
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
@ -212,6 +240,8 @@ export class ImportNotesProcessorService {
if (note.renoteId) return; if (note.renoteId) return;
const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null;
const files: MiDriveFile[] = []; const files: MiDriveFile[] = [];
const date = new Date(note.createdAt); const date = new Date(note.createdAt);
@ -243,8 +273,8 @@ export class ImportNotesProcessorService {
} }
} }
await this.noteCreateService.import(user, { createdAt: date, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw }); const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw });
if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes); if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes, createdNote.id);
} }
@bindThis @bindThis

View file

@ -54,7 +54,7 @@ export type DbJobMap = {
importIGToDb: DbNoteImportToDbJobData; importIGToDb: DbNoteImportToDbJobData;
importMastoToDb: DbNoteImportToDbJobData; importMastoToDb: DbNoteImportToDbJobData;
importPleroToDb: DbNoteImportToDbJobData; importPleroToDb: DbNoteImportToDbJobData;
importKeyNotesToDb: DbNoteImportToDbJobData; importKeyNotesToDb: DbKeyNoteImportToDbJobData;
importFollowing: DbUserImportJobData; importFollowing: DbUserImportJobData;
importFollowingToDb: DbUserImportToDbJobData; importFollowingToDb: DbUserImportToDbJobData;
importMuting: DbUserImportJobData; importMuting: DbUserImportJobData;
@ -110,6 +110,13 @@ export type DbUserImportToDbJobData = {
export type DbNoteImportToDbJobData = { export type DbNoteImportToDbJobData = {
user: ThinUser; user: ThinUser;
target: any; target: any;
note?: MiNote['id'] | null;
};
export type DbKeyNoteImportToDbJobData = {
user: ThinUser;
target: any;
note: MiNote['id'] | null;
}; };
export type ObjectStorageJobData = ObjectStorageFileJobData | Record<string, unknown>; export type ObjectStorageJobData = ObjectStorageFileJobData | Record<string, unknown>;