feat: export clips (#12931)

* feat: export clips

* Update CHANGELOG.md
This commit is contained in:
Kagami Sascha Rosylight 2024-01-07 02:35:58 +01:00 committed by GitHub
parent f2dee7b25e
commit 2a9db983fc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 473 additions and 2 deletions

View file

@ -27,6 +27,7 @@
### Server
- Enhance: 連合先のレートリミットに引っかかった際にリトライするようになりました
- Enhance: ActivityPub Deliver queueでBodyを事前処理するように (#12916)
- Enhance: クリップをエクスポートできるように
## 2023.12.2

1
locales/index.d.ts vendored
View file

@ -2256,6 +2256,7 @@ export interface Locale {
"_exportOrImport": {
"allNotes": string;
"favoritedNotes": string;
"clips": string;
"followingList": string;
"muteList": string;
"blockingList": string;

View file

@ -2159,6 +2159,7 @@ _profile:
_exportOrImport:
allNotes: "全てのノート"
favoritedNotes: "お気に入りにしたノート"
clips: "クリップ"
followingList: "フォロー"
muteList: "ミュート"
blockingList: "ブロック"

View file

@ -182,6 +182,16 @@ export class QueueService {
});
}
@bindThis
public createExportClipsJob(user: ThinUser) {
return this.dbQueue.add('exportClips', {
user: { id: user.id },
}, {
removeOnComplete: true,
removeOnFail: true,
});
}
@bindThis
public createExportFavoritesJob(user: ThinUser) {
return this.dbQueue.add('exportFavorites', {

View file

@ -24,6 +24,7 @@ import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmo
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js';
import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js';
import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js';
import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js';
@ -53,6 +54,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
DeleteDriveFilesProcessorService,
ExportCustomEmojisProcessorService,
ExportNotesProcessorService,
ExportClipsProcessorService,
ExportFavoritesProcessorService,
ExportFollowingProcessorService,
ExportMutingProcessorService,

View file

@ -16,6 +16,7 @@ import { InboxProcessorService } from './processors/InboxProcessorService.js';
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js';
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
@ -91,6 +92,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
private exportNotesProcessorService: ExportNotesProcessorService,
private exportClipsProcessorService: ExportClipsProcessorService,
private exportFavoritesProcessorService: ExportFavoritesProcessorService,
private exportFollowingProcessorService: ExportFollowingProcessorService,
private exportMutingProcessorService: ExportMutingProcessorService,
@ -164,6 +166,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
case 'exportNotes': return this.exportNotesProcessorService.process(job);
case 'exportClips': return this.exportClipsProcessorService.process(job);
case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);
case 'exportFollowing': return this.exportFollowingProcessorService.process(job);
case 'exportMuting': return this.exportMutingProcessorService.process(job);

View file

@ -0,0 +1,206 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as fs from 'node:fs';
import { Writable } from 'node:stream';
import { Inject, Injectable, StreamableFile } from '@nestjs/common';
import { MoreThan } from 'typeorm';
import { format as dateFormat } from 'date-fns';
import { DI } from '@/di-symbols.js';
import type { ClipNotesRepository, ClipsRepository, MiClip, MiClipNote, MiUser, NotesRepository, PollsRepository, UsersRepository } from '@/models/_.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js';
import type { MiPoll } from '@/models/Poll.js';
import type { MiNote } from '@/models/Note.js';
import { bindThis } from '@/decorators.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import { Packed } from '@/misc/json-schema.js';
import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
export class ExportClipsProcessorService {
private logger: Logger;
constructor(
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.pollsRepository)
private pollsRepository: PollsRepository,
@Inject(DI.clipsRepository)
private clipsRepository: ClipsRepository,
@Inject(DI.clipNotesRepository)
private clipNotesRepository: ClipNotesRepository,
private driveService: DriveService,
private queueLoggerService: QueueLoggerService,
private idService: IdService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('export-clips');
}
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting clips of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
return;
}
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
try {
const stream = Writable.toWeb(fs.createWriteStream(path, { flags: 'a' }));
const writer = stream.getWriter();
writer.closed.catch(this.logger.error);
await writer.write('[');
await this.processClips(writer, user, job);
await writer.write(']');
await writer.close();
this.logger.succ(`Exported to: ${path}`);
const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' });
this.logger.succ(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}
}
async processClips(writer: WritableStreamDefaultWriter, user: MiUser, job: Bull.Job<DbJobDataWithUser>) {
let exportedClipsCount = 0;
let cursor: MiClip['id'] | null = null;
while (true) {
const clips = await this.clipsRepository.find({
where: {
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
});
if (clips.length === 0) {
job.updateProgress(100);
break;
}
cursor = clips.at(-1)?.id ?? null;
for (const clip of clips) {
// Stringify but remove the last `]}`
const content = JSON.stringify(this.serializeClip(clip)).slice(0, -2);
const isFirst = exportedClipsCount === 0;
await writer.write(isFirst ? content : ',\n' + content);
await this.processClipNotes(writer, clip.id);
await writer.write(']}');
exportedClipsCount++;
}
const total = await this.clipsRepository.countBy({
userId: user.id,
});
job.updateProgress(exportedClipsCount / total);
}
}
async processClipNotes(writer: WritableStreamDefaultWriter, clipId: string): Promise<void> {
let exportedClipNotesCount = 0;
let cursor: MiClipNote['id'] | null = null;
while (true) {
const clipNotes = await this.clipNotesRepository.find({
where: {
clipId,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
relations: ['note', 'note.user'],
}) as (MiClipNote & { note: MiNote & { user: MiUser } })[];
if (clipNotes.length === 0) {
break;
}
cursor = clipNotes.at(-1)?.id ?? null;
for (const clipNote of clipNotes) {
let poll: MiPoll | undefined;
if (clipNote.note.hasPoll) {
poll = await this.pollsRepository.findOneByOrFail({ noteId: clipNote.note.id });
}
const content = JSON.stringify(this.serializeClipNote(clipNote, poll));
const isFirst = exportedClipNotesCount === 0;
await writer.write(isFirst ? content : ',\n' + content);
exportedClipNotesCount++;
}
}
}
private serializeClip(clip: MiClip): Record<string, unknown> {
return {
id: clip.id,
name: clip.name,
description: clip.description,
lastClippedAt: clip.lastClippedAt?.toISOString(),
clipNotes: [],
};
}
private serializeClipNote(clip: MiClipNote & { note: MiNote & { user: MiUser } }, poll: MiPoll | undefined): Record<string, unknown> {
return {
id: clip.id,
createdAt: this.idService.parse(clip.id).date.toISOString(),
note: {
id: clip.note.id,
text: clip.note.text,
createdAt: this.idService.parse(clip.note.id).date.toISOString(),
fileIds: clip.note.fileIds,
replyId: clip.note.replyId,
renoteId: clip.note.renoteId,
poll: poll,
cw: clip.note.cw,
visibility: clip.note.visibility,
visibleUserIds: clip.note.visibleUserIds,
localOnly: clip.note.localOnly,
reactionAcceptance: clip.note.reactionAcceptance,
uri: clip.note.uri,
url: clip.note.url,
user: {
id: clip.note.user.id,
name: clip.note.user.name,
username: clip.note.user.username,
host: clip.note.user.host,
uri: clip.note.user.uri,
},
},
};
}
}

View file

@ -208,6 +208,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
import * as ep___i_exportMute from './endpoints/i/export-mute.js';
import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
import * as ep___i_exportClips from './endpoints/i/export-clips.js';
import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
@ -569,6 +570,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass:
const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default };
const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default };
const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default };
const $i_exportClips: Provider = { provide: 'ep:i/export-clips', useClass: ep___i_exportClips.default };
const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default };
const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default };
const $i_exportAntennas: Provider = { provide: 'ep:i/export-antennas', useClass: ep___i_exportAntennas.default };
@ -934,6 +936,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$i_exportFollowing,
$i_exportMute,
$i_exportNotes,
$i_exportClips,
$i_exportFavorites,
$i_exportUserLists,
$i_exportAntennas,
@ -1293,6 +1296,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$i_exportFollowing,
$i_exportMute,
$i_exportNotes,
$i_exportClips,
$i_exportFavorites,
$i_exportUserLists,
$i_exportAntennas,

View file

@ -3,8 +3,8 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import type { Schema } from '@/misc/json-schema.js';
import { permissions } from 'misskey-js';
import type { Schema } from '@/misc/json-schema.js';
import { RolePolicies } from '@/core/RoleService.js';
import * as ep___admin_meta from './endpoints/admin/meta.js';
@ -209,6 +209,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
import * as ep___i_exportMute from './endpoints/i/export-mute.js';
import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
import * as ep___i_exportClips from './endpoints/i/export-clips.js';
import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
@ -568,6 +569,7 @@ const eps = [
['i/export-following', ep___i_exportFollowing],
['i/export-mute', ep___i_exportMute],
['i/export-notes', ep___i_exportNotes],
['i/export-clips', ep___i_exportClips],
['i/export-favorites', ep___i_exportFavorites],
['i/export-user-lists', ep___i_exportUserLists],
['i/export-antennas', ep___i_exportAntennas],

View file

@ -0,0 +1,35 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Injectable } from '@nestjs/common';
import ms from 'ms';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { QueueService } from '@/core/QueueService.js';
export const meta = {
secure: true,
requireCredential: true,
limit: {
duration: ms('1day'),
max: 1,
},
} as const;
export const paramDef = {
type: 'object',
properties: {},
required: [],
} as const;
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor(
private queueService: QueueService,
) {
super(meta, paramDef, async (ps, me) => {
this.queueService.createExportClipsJob(me);
});
}
}

View file

@ -0,0 +1,194 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
process.env.NODE_ENV = 'test';
import * as assert from 'assert';
import { signup, api, startServer, startJobQueue, port, post } from '../utils.js';
import type { INestApplicationContext } from '@nestjs/common';
import type * as misskey from 'misskey-js';
describe('export-clips', () => {
let app: INestApplicationContext;
let alice: misskey.entities.SignupResponse;
let bob: misskey.entities.SignupResponse;
// XXX: Any better way to get the result?
async function pollFirstDriveFile() {
while (true) {
const files = (await api('/drive/files', {}, alice)).body;
if (!files.length) {
await new Promise(r => setTimeout(r, 100));
continue;
}
if (files.length > 1) {
throw new Error('Too many files?');
}
const file = (await api('/drive/files/show', { fileId: files[0].id }, alice)).body;
const res = await fetch(new URL(new URL(file.url).pathname, `http://127.0.0.1:${port}`));
return await res.json();
}
}
beforeAll(async () => {
app = await startServer();
await startJobQueue();
alice = await signup({ username: 'alice' });
bob = await signup({ username: 'bob' });
}, 1000 * 60 * 2);
afterAll(async () => {
await app.close();
});
beforeEach(async () => {
// Clean all clips and files of alice
const clips = (await api('/clips/list', {}, alice)).body;
for (const clip of clips) {
const res = await api('/clips/delete', { clipId: clip.id }, alice);
if (res.status !== 204) {
throw new Error('Failed to delete clip');
}
}
const files = (await api('/drive/files', {}, alice)).body;
for (const file of files) {
const res = await api('/drive/files/delete', { fileId: file.id }, alice);
if (res.status !== 204) {
throw new Error('Failed to delete file');
}
}
});
test('basic export', async () => {
let res = await api('/clips/create', {
name: 'foo',
description: 'bar',
}, alice);
assert.strictEqual(res.status, 200);
res = await api('/i/export-clips', {}, alice);
assert.strictEqual(res.status, 204);
const exported = await pollFirstDriveFile();
assert.strictEqual(exported[0].name, 'foo');
assert.strictEqual(exported[0].description, 'bar');
assert.strictEqual(exported[0].clipNotes.length, 0);
});
test('export with notes', async () => {
let res = await api('/clips/create', {
name: 'foo',
description: 'bar',
}, alice);
assert.strictEqual(res.status, 200);
const clip = res.body;
const note1 = await post(alice, {
text: 'baz1',
});
const note2 = await post(alice, {
text: 'baz2',
poll: {
choices: ['sakura', 'izumi', 'ako'],
},
});
for (const note of [note1, note2]) {
res = await api('/clips/add-note', {
clipId: clip.id,
noteId: note.id,
}, alice);
assert.strictEqual(res.status, 204);
}
res = await api('/i/export-clips', {}, alice);
assert.strictEqual(res.status, 204);
const exported = await pollFirstDriveFile();
assert.strictEqual(exported[0].name, 'foo');
assert.strictEqual(exported[0].description, 'bar');
assert.strictEqual(exported[0].clipNotes.length, 2);
assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1');
assert.strictEqual(exported[0].clipNotes[1].note.text, 'baz2');
assert.deepStrictEqual(exported[0].clipNotes[1].note.poll.choices[0], 'sakura');
});
test('multiple clips', async () => {
let res = await api('/clips/create', {
name: 'kawaii',
description: 'kawaii',
}, alice);
assert.strictEqual(res.status, 200);
const clip1 = res.body;
res = await api('/clips/create', {
name: 'yuri',
description: 'yuri',
}, alice);
assert.strictEqual(res.status, 200);
const clip2 = res.body;
const note1 = await post(alice, {
text: 'baz1',
});
const note2 = await post(alice, {
text: 'baz2',
});
res = await api('/clips/add-note', {
clipId: clip1.id,
noteId: note1.id,
}, alice);
assert.strictEqual(res.status, 204);
res = await api('/clips/add-note', {
clipId: clip2.id,
noteId: note2.id,
}, alice);
assert.strictEqual(res.status, 204);
res = await api('/i/export-clips', {}, alice);
assert.strictEqual(res.status, 204);
const exported = await pollFirstDriveFile();
assert.strictEqual(exported[0].name, 'kawaii');
assert.strictEqual(exported[0].clipNotes.length, 1);
assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1');
assert.strictEqual(exported[1].name, 'yuri');
assert.strictEqual(exported[1].clipNotes.length, 1);
assert.strictEqual(exported[1].clipNotes[0].note.text, 'baz2');
});
test('Clipping other user\'s note', async () => {
let res = await api('/clips/create', {
name: 'kawaii',
description: 'kawaii',
}, alice);
assert.strictEqual(res.status, 200);
const clip = res.body;
const note = await post(bob, {
text: 'baz',
visibility: 'followers',
});
res = await api('/clips/add-note', {
clipId: clip.id,
noteId: note.id,
}, alice);
assert.strictEqual(res.status, 204);
res = await api('/i/export-clips', {}, alice);
assert.strictEqual(res.status, 204);
const exported = await pollFirstDriveFile();
assert.strictEqual(exported[0].name, 'kawaii');
assert.strictEqual(exported[0].clipNotes.length, 1);
assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz');
assert.strictEqual(exported[0].clipNotes[0].note.user.username, 'bob');
});
});

View file

@ -17,7 +17,7 @@ import { entities } from '../src/postgres.js';
import { loadConfig } from '../src/config.js';
import type * as misskey from 'misskey-js';
export { server as startServer } from '@/boot/common.js';
export { server as startServer, jobQueue as startJobQueue } from '@/boot/common.js';
interface UserToken {
token: string;

View file

@ -21,6 +21,14 @@ SPDX-License-Identifier: AGPL-3.0-only
<MkButton primary :class="$style.button" inline @click="exportFavorites()"><i class="ti ti-download"></i> {{ i18n.ts.export }}</MkButton>
</MkFolder>
</FormSection>
<FormSection>
<template #label><i class="ti ti-star"></i> {{ i18n.ts._exportOrImport.clips }}</template>
<MkFolder>
<template #label>{{ i18n.ts.export }}</template>
<template #icon><i class="ti ti-download"></i></template>
<MkButton primary :class="$style.button" inline @click="exportClips()"><i class="ti ti-download"></i> {{ i18n.ts.export }}</MkButton>
</MkFolder>
</FormSection>
<FormSection>
<template #label><i class="ti ti-users"></i> {{ i18n.ts._exportOrImport.followingList }}</template>
<div class="_gaps_s">
@ -157,6 +165,10 @@ const exportFavorites = () => {
misskeyApi('i/export-favorites', {}).then(onExportSuccess).catch(onError);
};
const exportClips = () => {
misskeyApi('i/export-clips', {}).then(onExportSuccess).catch(onError);
};
const exportFollowing = () => {
misskeyApi('i/export-following', {
excludeMuting: excludeMutingUsers.value,