From c958d935e4e5b33b28c242a0de2c4319859f8541 Mon Sep 17 00:00:00 2001 From: dakkar Date: Thu, 30 Nov 2023 13:24:57 +0000 Subject: [PATCH] thread Pleroma imports as well I have _not_ tested this, but it should work fine, those exports are the same shape as Mastodon's --- packages/backend/src/core/QueueService.ts | 4 +-- .../processors/ImportNotesProcessorService.ts | 25 ++++++++++++------- packages/backend/src/queue/types.ts | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 6acdc0ad8..2ee61eb54 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -284,8 +284,8 @@ export class QueueService { } @bindThis - public createImportPleroToDbJob(user: ThinUser, targets: string[]) { - const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel })); + public createImportPleroToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) { + const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel, note })); return this.dbQueue.addBulk(jobs); } diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index b9a3645f6..552b69d92 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -268,7 +268,8 @@ export class ImportNotesProcessorService { if (isPleroma) { const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8'); const outbox = JSON.parse(outboxJson); - this.queueService.createImportPleroToDbJob(job.data.user, outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note')); + const processedToots = await this.recreateChain(['object', 'id'], ['object', 'inReplyTo'], outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'), true); + this.queueService.createImportPleroToDbJob(job.data.user, processedToots, null); } else { const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8'); const outbox = JSON.parse(outboxJson); @@ -421,13 +422,15 @@ export class ImportNotesProcessorService { } @bindThis - public async processPleroToDb(job: Bull.Job): Promise { + public async processPleroToDb(job: Bull.Job): Promise { const post = job.data.target; const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { return; } + if (post.directMessage) return; + const date = new Date(post.object.published); let text = undefined; const files: MiDriveFile[] = []; @@ -437,15 +440,18 @@ export class ImportNotesProcessorService { if (folder == null) return; if (post.object.inReplyTo != null) { - try { - reply = await this.apNoteService.resolveNote(post.object.inReplyTo); - } catch (error) { - reply = null; + const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null; + if (parentNote) { + reply = parentNote; + } else { + try { + reply = await this.apNoteService.resolveNote(post.object.inReplyTo); + } catch (error) { + reply = null; + } } } - if (post.directMessage) return; - const hashtags = extractApHashtagObjects(post.object.tag).map((x) => x.name).filter((x): x is string => x != null); try { @@ -489,7 +495,8 @@ export class ImportNotesProcessorService { } } - await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: post.object.sensitive ? post.object.summary : null, reply: reply }); + const createdNote = await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: post.object.sensitive ? post.object.summary : null, reply: reply }); + if (post.childNotes) this.queueService.createImportPleroToDbJob(user, post.childNotes, createdNote.id); } @bindThis diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 440435754..432b3d364 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -54,7 +54,7 @@ export type DbJobMap = { importIGToDb: DbNoteImportToDbJobData; importFBToDb: DbNoteImportToDbJobData; importMastoToDb: DbNoteWithParentImportToDbJobData; - importPleroToDb: DbNoteImportToDbJobData; + importPleroToDb: DbNoteWithParentImportToDbJobData; importKeyNotesToDb: DbNoteWithParentImportToDbJobData; importFollowing: DbUserImportJobData; importFollowingToDb: DbUserImportToDbJobData;