From 6f76a3a1f82131210b120139a5322f054a866c7f Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Mon, 13 Nov 2023 13:07:49 +0100 Subject: [PATCH] upd: import replies to own posts on *key --- packages/backend/src/core/QueueService.ts | 5 ++- .../processors/ImportNotesProcessorService.ts | 42 ++++++++++++++++--- packages/backend/src/queue/types.ts | 9 +++- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index ed24cfa56..887048349 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -16,6 +16,7 @@ import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, Obj import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; +import { MiNote } from '@/models/Note.js'; @Injectable() export class QueueService { @@ -289,8 +290,8 @@ export class QueueService { } @bindThis - public createImportKeyNotesToDbJob(user: ThinUser, targets: string[]) { - const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel })); + public createImportKeyNotesToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) { + const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel, note })); return this.dbQueue.addBulk(jobs); } diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 3f9d47271..58f10998b 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -4,7 +4,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull } from 'typeorm'; import { ZipReader } from 'slacc'; import { DI } from '@/di-symbols.js'; -import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote } from '@/models/_.js'; +import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; @@ -18,7 +18,7 @@ import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; -import type { DbNoteImportToDbJobData, DbNoteImportJobData } from '../types.js'; +import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbKeyNoteImportToDbJobData } from '../types.js'; @Injectable() export class ImportNotesProcessorService { @@ -31,6 +31,9 @@ export class ImportNotesProcessorService { @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + private queueService: QueueService, private utilityService: UtilityService, private noteCreateService: NoteCreateService, @@ -66,6 +69,30 @@ export class ImportNotesProcessorService { } } + // Function was taken from Firefish and edited to remove renoteId and make it run in only one for loop instead of two + @bindThis + private async recreateChain(arr: any[]) { + type NotesMap = { + [id: string]: any; + }; + const notesTree: any[] = []; + const lookup: NotesMap = {}; + for await (const note of arr) { + lookup[`${note.id}`] = note; + note.childNotes = []; + let parent = null; + + if (note.replyId == null) { + notesTree.push(note); + } else { + parent = lookup[`${note.replyId}`]; + } + + if (parent) parent.childNotes.push(note); + } + return notesTree; + } + @bindThis private isIterable(obj: any) { if (obj == null) { @@ -195,7 +222,8 @@ export class ImportNotesProcessorService { const notesJson = fs.readFileSync(path, 'utf-8'); const notes = JSON.parse(notesJson); - this.queueService.createImportKeyNotesToDbJob(job.data.user, notes); + const processedNotes = await this.recreateChain(notes); + this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null); cleanup(); } @@ -203,7 +231,7 @@ export class ImportNotesProcessorService { } @bindThis - public async processKeyNotesToDb(job: Bull.Job): Promise { + public async processKeyNotesToDb(job: Bull.Job): Promise { const note = job.data.target; const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { @@ -212,6 +240,8 @@ export class ImportNotesProcessorService { if (note.renoteId) return; + const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null; + const files: MiDriveFile[] = []; const date = new Date(note.createdAt); @@ -243,8 +273,8 @@ export class ImportNotesProcessorService { } } - await this.noteCreateService.import(user, { createdAt: date, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw }); - if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes); + const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw }); + if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes, createdNote.id); } @bindThis diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 9de3a41f0..493d3911b 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -54,7 +54,7 @@ export type DbJobMap = { importIGToDb: DbNoteImportToDbJobData; importMastoToDb: DbNoteImportToDbJobData; importPleroToDb: DbNoteImportToDbJobData; - importKeyNotesToDb: DbNoteImportToDbJobData; + importKeyNotesToDb: DbKeyNoteImportToDbJobData; importFollowing: DbUserImportJobData; importFollowingToDb: DbUserImportToDbJobData; importMuting: DbUserImportJobData; @@ -110,6 +110,13 @@ export type DbUserImportToDbJobData = { export type DbNoteImportToDbJobData = { user: ThinUser; target: any; + note?: MiNote['id'] | null; +}; + +export type DbKeyNoteImportToDbJobData = { + user: ThinUser; + target: any; + note: MiNote['id'] | null; }; export type ObjectStorageJobData = ObjectStorageFileJobData | Record;