diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c27349f6..0d2fb4ccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ ### Server - Enhance: 連合先のレートリミットに引っかかった際にリトライするようになりました - Enhance: ActivityPub Deliver queueでBodyを事前処理するように (#12916) +- Enhance: クリップをエクスポートできるように ## 2023.12.2 diff --git a/locales/index.d.ts b/locales/index.d.ts index 99bc0fc04..75517fa2a 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -2256,6 +2256,7 @@ export interface Locale { "_exportOrImport": { "allNotes": string; "favoritedNotes": string; + "clips": string; "followingList": string; "muteList": string; "blockingList": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 7cf5663a7..8b6b119d7 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -2159,6 +2159,7 @@ _profile: _exportOrImport: allNotes: "全てのノート" favoritedNotes: "お気に入りにしたノート" + clips: "クリップ" followingList: "フォロー" muteList: "ミュート" blockingList: "ブロック" diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 4f99dee64..dc3f248da 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -182,6 +182,16 @@ export class QueueService { }); } + @bindThis + public createExportClipsJob(user: ThinUser) { + return this.dbQueue.add('exportClips', { + user: { id: user.id }, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + @bindThis public createExportFavoritesJob(user: ThinUser) { return this.dbQueue.add('exportFavorites', { diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index e6327002c..9c52c7d76 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -24,6 +24,7 @@ import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmo import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js'; import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; +import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js'; import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js'; import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js'; import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js'; @@ -53,6 +54,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor DeleteDriveFilesProcessorService, ExportCustomEmojisProcessorService, ExportNotesProcessorService, + ExportClipsProcessorService, ExportFavoritesProcessorService, ExportFollowingProcessorService, ExportMutingProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index b872dd65f..bcc1a69f8 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -16,6 +16,7 @@ import { InboxProcessorService } from './processors/InboxProcessorService.js'; import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; +import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js'; import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js'; import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js'; import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js'; @@ -91,6 +92,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService, private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService, private exportNotesProcessorService: ExportNotesProcessorService, + private exportClipsProcessorService: ExportClipsProcessorService, private exportFavoritesProcessorService: ExportFavoritesProcessorService, private exportFollowingProcessorService: ExportFollowingProcessorService, private exportMutingProcessorService: ExportMutingProcessorService, @@ -164,6 +166,7 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); case 'exportNotes': return this.exportNotesProcessorService.process(job); + case 'exportClips': return this.exportClipsProcessorService.process(job); case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); case 'exportFollowing': return this.exportFollowingProcessorService.process(job); case 'exportMuting': return this.exportMutingProcessorService.process(job); diff --git a/packages/backend/src/queue/processors/ExportClipsProcessorService.ts b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts new file mode 100644 index 000000000..5221497bd --- /dev/null +++ b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts @@ -0,0 +1,206 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as fs from 'node:fs'; +import { Writable } from 'node:stream'; +import { Inject, Injectable, StreamableFile } from '@nestjs/common'; +import { MoreThan } from 'typeorm'; +import { format as dateFormat } from 'date-fns'; +import { DI } from '@/di-symbols.js'; +import type { ClipNotesRepository, ClipsRepository, MiClip, MiClipNote, MiUser, NotesRepository, PollsRepository, UsersRepository } from '@/models/_.js'; +import type Logger from '@/logger.js'; +import { DriveService } from '@/core/DriveService.js'; +import { createTemp } from '@/misc/create-temp.js'; +import type { MiPoll } from '@/models/Poll.js'; +import type { MiNote } from '@/models/Note.js'; +import { bindThis } from '@/decorators.js'; +import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import { Packed } from '@/misc/json-schema.js'; +import { IdService } from '@/core/IdService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; + +@Injectable() +export class ExportClipsProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.pollsRepository) + private pollsRepository: PollsRepository, + + @Inject(DI.clipsRepository) + private clipsRepository: ClipsRepository, + + @Inject(DI.clipNotesRepository) + private clipNotesRepository: ClipNotesRepository, + + private driveService: DriveService, + private queueLoggerService: QueueLoggerService, + private idService: IdService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('export-clips'); + } + + @bindThis + public async process(job: Bull.Job): Promise { + this.logger.info(`Exporting clips of ${job.data.user.id} ...`); + + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + // Create temp file + const [path, cleanup] = await createTemp(); + + this.logger.info(`Temp file is ${path}`); + + try { + const stream = Writable.toWeb(fs.createWriteStream(path, { flags: 'a' })); + const writer = stream.getWriter(); + writer.closed.catch(this.logger.error); + + await writer.write('['); + + await this.processClips(writer, user, job); + + await writer.write(']'); + await writer.close(); + + this.logger.succ(`Exported to: ${path}`); + + const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); + + this.logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } + } + + async processClips(writer: WritableStreamDefaultWriter, user: MiUser, job: Bull.Job) { + let exportedClipsCount = 0; + let cursor: MiClip['id'] | null = null; + + while (true) { + const clips = await this.clipsRepository.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); + + if (clips.length === 0) { + job.updateProgress(100); + break; + } + + cursor = clips.at(-1)?.id ?? null; + + for (const clip of clips) { + // Stringify but remove the last `]}` + const content = JSON.stringify(this.serializeClip(clip)).slice(0, -2); + const isFirst = exportedClipsCount === 0; + await writer.write(isFirst ? content : ',\n' + content); + + await this.processClipNotes(writer, clip.id); + + await writer.write(']}'); + exportedClipsCount++; + } + + const total = await this.clipsRepository.countBy({ + userId: user.id, + }); + + job.updateProgress(exportedClipsCount / total); + } + } + + async processClipNotes(writer: WritableStreamDefaultWriter, clipId: string): Promise { + let exportedClipNotesCount = 0; + let cursor: MiClipNote['id'] | null = null; + + while (true) { + const clipNotes = await this.clipNotesRepository.find({ + where: { + clipId, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + relations: ['note', 'note.user'], + }) as (MiClipNote & { note: MiNote & { user: MiUser } })[]; + + if (clipNotes.length === 0) { + break; + } + + cursor = clipNotes.at(-1)?.id ?? null; + + for (const clipNote of clipNotes) { + let poll: MiPoll | undefined; + if (clipNote.note.hasPoll) { + poll = await this.pollsRepository.findOneByOrFail({ noteId: clipNote.note.id }); + } + const content = JSON.stringify(this.serializeClipNote(clipNote, poll)); + const isFirst = exportedClipNotesCount === 0; + await writer.write(isFirst ? content : ',\n' + content); + + exportedClipNotesCount++; + } + } + } + + private serializeClip(clip: MiClip): Record { + return { + id: clip.id, + name: clip.name, + description: clip.description, + lastClippedAt: clip.lastClippedAt?.toISOString(), + clipNotes: [], + }; + } + + private serializeClipNote(clip: MiClipNote & { note: MiNote & { user: MiUser } }, poll: MiPoll | undefined): Record { + return { + id: clip.id, + createdAt: this.idService.parse(clip.id).date.toISOString(), + note: { + id: clip.note.id, + text: clip.note.text, + createdAt: this.idService.parse(clip.note.id).date.toISOString(), + fileIds: clip.note.fileIds, + replyId: clip.note.replyId, + renoteId: clip.note.renoteId, + poll: poll, + cw: clip.note.cw, + visibility: clip.note.visibility, + visibleUserIds: clip.note.visibleUserIds, + localOnly: clip.note.localOnly, + reactionAcceptance: clip.note.reactionAcceptance, + uri: clip.note.uri, + url: clip.note.url, + user: { + id: clip.note.user.id, + name: clip.note.user.name, + username: clip.note.user.username, + host: clip.note.user.host, + uri: clip.note.user.uri, + }, + }, + }; + } +} diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 86a64d712..a3a980544 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -208,6 +208,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportClips from './endpoints/i/export-clips.js'; import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js'; @@ -569,6 +570,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default }; const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default }; const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default }; +const $i_exportClips: Provider = { provide: 'ep:i/export-clips', useClass: ep___i_exportClips.default }; const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default }; const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default }; const $i_exportAntennas: Provider = { provide: 'ep:i/export-antennas', useClass: ep___i_exportAntennas.default }; @@ -934,6 +936,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportClips, $i_exportFavorites, $i_exportUserLists, $i_exportAntennas, @@ -1293,6 +1296,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportClips, $i_exportFavorites, $i_exportUserLists, $i_exportAntennas, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 41232091c..bd8aa4af7 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -3,8 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type { Schema } from '@/misc/json-schema.js'; import { permissions } from 'misskey-js'; +import type { Schema } from '@/misc/json-schema.js'; import { RolePolicies } from '@/core/RoleService.js'; import * as ep___admin_meta from './endpoints/admin/meta.js'; @@ -209,6 +209,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportClips from './endpoints/i/export-clips.js'; import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js'; @@ -568,6 +569,7 @@ const eps = [ ['i/export-following', ep___i_exportFollowing], ['i/export-mute', ep___i_exportMute], ['i/export-notes', ep___i_exportNotes], + ['i/export-clips', ep___i_exportClips], ['i/export-favorites', ep___i_exportFavorites], ['i/export-user-lists', ep___i_exportUserLists], ['i/export-antennas', ep___i_exportAntennas], diff --git a/packages/backend/src/server/api/endpoints/i/export-clips.ts b/packages/backend/src/server/api/endpoints/i/export-clips.ts new file mode 100644 index 000000000..9435a2b23 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/export-clips.ts @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import ms from 'ms'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueueService } from '@/core/QueueService.js'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1day'), + max: 1, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + private queueService: QueueService, + ) { + super(meta, paramDef, async (ps, me) => { + this.queueService.createExportClipsJob(me); + }); + } +} diff --git a/packages/backend/test/e2e/exports.ts b/packages/backend/test/e2e/exports.ts new file mode 100644 index 000000000..9686f2b7f --- /dev/null +++ b/packages/backend/test/e2e/exports.ts @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +process.env.NODE_ENV = 'test'; + +import * as assert from 'assert'; +import { signup, api, startServer, startJobQueue, port, post } from '../utils.js'; +import type { INestApplicationContext } from '@nestjs/common'; +import type * as misskey from 'misskey-js'; + +describe('export-clips', () => { + let app: INestApplicationContext; + let alice: misskey.entities.SignupResponse; + let bob: misskey.entities.SignupResponse; + + // XXX: Any better way to get the result? + async function pollFirstDriveFile() { + while (true) { + const files = (await api('/drive/files', {}, alice)).body; + if (!files.length) { + await new Promise(r => setTimeout(r, 100)); + continue; + } + if (files.length > 1) { + throw new Error('Too many files?'); + } + const file = (await api('/drive/files/show', { fileId: files[0].id }, alice)).body; + const res = await fetch(new URL(new URL(file.url).pathname, `http://127.0.0.1:${port}`)); + return await res.json(); + } + } + + beforeAll(async () => { + app = await startServer(); + await startJobQueue(); + alice = await signup({ username: 'alice' }); + bob = await signup({ username: 'bob' }); + }, 1000 * 60 * 2); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + // Clean all clips and files of alice + const clips = (await api('/clips/list', {}, alice)).body; + for (const clip of clips) { + const res = await api('/clips/delete', { clipId: clip.id }, alice); + if (res.status !== 204) { + throw new Error('Failed to delete clip'); + } + } + const files = (await api('/drive/files', {}, alice)).body; + for (const file of files) { + const res = await api('/drive/files/delete', { fileId: file.id }, alice); + if (res.status !== 204) { + throw new Error('Failed to delete file'); + } + } + }); + + test('basic export', async () => { + let res = await api('/clips/create', { + name: 'foo', + description: 'bar', + }, alice); + assert.strictEqual(res.status, 200); + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'foo'); + assert.strictEqual(exported[0].description, 'bar'); + assert.strictEqual(exported[0].clipNotes.length, 0); + }); + + test('export with notes', async () => { + let res = await api('/clips/create', { + name: 'foo', + description: 'bar', + }, alice); + assert.strictEqual(res.status, 200); + const clip = res.body; + + const note1 = await post(alice, { + text: 'baz1', + }); + + const note2 = await post(alice, { + text: 'baz2', + poll: { + choices: ['sakura', 'izumi', 'ako'], + }, + }); + + for (const note of [note1, note2]) { + res = await api('/clips/add-note', { + clipId: clip.id, + noteId: note.id, + }, alice); + assert.strictEqual(res.status, 204); + } + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'foo'); + assert.strictEqual(exported[0].description, 'bar'); + assert.strictEqual(exported[0].clipNotes.length, 2); + assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1'); + assert.strictEqual(exported[0].clipNotes[1].note.text, 'baz2'); + assert.deepStrictEqual(exported[0].clipNotes[1].note.poll.choices[0], 'sakura'); + }); + + test('multiple clips', async () => { + let res = await api('/clips/create', { + name: 'kawaii', + description: 'kawaii', + }, alice); + assert.strictEqual(res.status, 200); + const clip1 = res.body; + + res = await api('/clips/create', { + name: 'yuri', + description: 'yuri', + }, alice); + assert.strictEqual(res.status, 200); + const clip2 = res.body; + + const note1 = await post(alice, { + text: 'baz1', + }); + + const note2 = await post(alice, { + text: 'baz2', + }); + + res = await api('/clips/add-note', { + clipId: clip1.id, + noteId: note1.id, + }, alice); + assert.strictEqual(res.status, 204); + + res = await api('/clips/add-note', { + clipId: clip2.id, + noteId: note2.id, + }, alice); + assert.strictEqual(res.status, 204); + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'kawaii'); + assert.strictEqual(exported[0].clipNotes.length, 1); + assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1'); + assert.strictEqual(exported[1].name, 'yuri'); + assert.strictEqual(exported[1].clipNotes.length, 1); + assert.strictEqual(exported[1].clipNotes[0].note.text, 'baz2'); + }); + + test('Clipping other user\'s note', async () => { + let res = await api('/clips/create', { + name: 'kawaii', + description: 'kawaii', + }, alice); + assert.strictEqual(res.status, 200); + const clip = res.body; + + const note = await post(bob, { + text: 'baz', + visibility: 'followers', + }); + + res = await api('/clips/add-note', { + clipId: clip.id, + noteId: note.id, + }, alice); + assert.strictEqual(res.status, 204); + + res = await api('/i/export-clips', {}, alice); + assert.strictEqual(res.status, 204); + + const exported = await pollFirstDriveFile(); + assert.strictEqual(exported[0].name, 'kawaii'); + assert.strictEqual(exported[0].clipNotes.length, 1); + assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz'); + assert.strictEqual(exported[0].clipNotes[0].note.user.username, 'bob'); + }); +}); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 46b8ea9cd..7c9428d47 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -17,7 +17,7 @@ import { entities } from '../src/postgres.js'; import { loadConfig } from '../src/config.js'; import type * as misskey from 'misskey-js'; -export { server as startServer } from '@/boot/common.js'; +export { server as startServer, jobQueue as startJobQueue } from '@/boot/common.js'; interface UserToken { token: string; diff --git a/packages/frontend/src/pages/settings/import-export.vue b/packages/frontend/src/pages/settings/import-export.vue index 990eff99c..70d718f1a 100644 --- a/packages/frontend/src/pages/settings/import-export.vue +++ b/packages/frontend/src/pages/settings/import-export.vue @@ -21,6 +21,14 @@ SPDX-License-Identifier: AGPL-3.0-only {{ i18n.ts.export }} + + + + + + {{ i18n.ts.export }} + +
@@ -157,6 +165,10 @@ const exportFavorites = () => { misskeyApi('i/export-favorites', {}).then(onExportSuccess).catch(onError); }; +const exportClips = () => { + misskeyApi('i/export-clips', {}).then(onExportSuccess).catch(onError); +}; + const exportFollowing = () => { misskeyApi('i/export-following', { excludeMuting: excludeMutingUsers.value,