自分の投稿情報をエクスポートできるように (#4144)

* wip

* 正しいJSONを生成するように

* データを整形

locales/ja-JP.yml

@@ -557,6 +557,9 @@ common/views/components/profile-editor.vue:
   email-address: "メールアドレス"
   email-verified: "メールアドレスが確認されました"
   email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。"
+  export: "エクスポート"
+  export-notes: "すべての投稿のエクスポート"
+  export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。"
 
 common/views/components/user-list-editor.vue:
   users: "ユーザー"

src/client/app/common/views/components/profile-editor.vue

@@ -87,6 +87,14 @@
 			<ui-button @click="updateEmail()">{{ $t('save') }}</ui-button>
 		</div>
 	</section>
+
+	<section>
+		<header>{{ $t('export') }}</header>
+
+		<div>
+			<ui-button @click="exportNotes()">{{ $t('export-notes') }}</ui-button>
+		</div>
+	</section>
 </ui-card>
 </template>
 
@@ -252,6 +260,15 @@ export default Vue.extend({
 					email: this.email == '' ? null : this.email
 				});
 			});
+		},
+
+		exportNotes() {
+			this.$root.api('i/export-notes', {});
+
+			this.$root.dialog({
+				type: 'info',
+				text: this.$t('export-requested')
+			});
 		}
 	}
 });

src/queue/index.ts

@@ -1,9 +1,9 @@
 import * as Queue from 'bee-queue';
 import config from '../config';
-import http from './processors/http';
+
 import { ILocalUser } from '../models/user';
-import Logger from '../misc/logger';
 import { program } from '../argv';
+import handler from './processors';
 
 const enableQueue = config.redis != null && !program.disableQueue;
 
@@ -36,7 +36,7 @@ export function createHttpJob(data: any) {
 			.backoff('exponential', 16384) // 16s
 			.save();
 	} else {
-		return http({ data }, () => {});
+		return handler({ data }, () => {});
 	}
 }
 
@@ -51,10 +51,18 @@ export function deliver(user: ILocalUser, content: any, to: any) {
 	});
 }
 
-export const queueLogger = new Logger('queue');
+export function createExportNotesJob(user: ILocalUser) {
+	if (!enableQueue) throw 'queue disabled';
+
+	return queue.createJob({
+		type: 'exportNotes',
+		user: user
+	})
+		.save();
+}
 
 export default function() {
 	if (enableQueue) {
-		queue.process(128, http);
+		queue.process(128, handler);
 	}
 }

src/queue/logger.ts

@@ -0,0 +1,3 @@
+import Logger from '../misc/logger';
+
+export const queueLogger = new Logger('queue', 'orange');

src/queue/processors/export-notes.ts

@@ -0,0 +1,128 @@
+import * as bq from 'bee-queue';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+import * as mongo from 'mongodb';
+
+import { queueLogger } from '../logger';
+import Note, { INote } from '../../models/note';
+import addFile from '../../services/drive/add-file';
+import User from '../../models/user';
+import dateFormat = require('dateformat');
+
+const logger = queueLogger.createSubLogger('export-notes');
+
+export async function exportNotes(job: bq.Job, done: any): Promise<void> {
+	logger.info(`Exporting notes of ${job.data.user._id} ...`);
+
+	const user = await User.findOne({
+		_id: new mongo.ObjectID(job.data.user._id.toString())
+	});
+
+	// Create temp file
+	const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
+		tmp.file((e, path, fd, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+
+	logger.info(`Temp file is ${path}`);
+
+	const stream = fs.createWriteStream(path, { flags: 'a' });
+
+	await new Promise((res, rej) => {
+		stream.write('[', err => {
+			if (err) {
+				logger.error(err);
+				rej(err);
+			} else {
+				res();
+			}
+		});
+	});
+
+	let exportedNotesCount = 0;
+	let ended = false;
+	let cursor: any = null;
+
+	while (!ended) {
+		const notes = await Note.find({
+			userId: user._id,
+			...(cursor ? { _id: { $gt: cursor } } : {})
+		}, {
+			limit: 100,
+			sort: {
+				_id: 1
+			}
+		});
+
+		if (notes.length === 0) {
+			ended = true;
+			job.reportProgress(100);
+			break;
+		}
+
+		cursor = notes[notes.length - 1]._id;
+
+		for (const note of notes) {
+			const content = JSON.stringify(serialize(note));
+			await new Promise((res, rej) => {
+				stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
+					if (err) {
+						logger.error(err);
+						rej(err);
+					} else {
+						res();
+					}
+				});
+			});
+			exportedNotesCount++;
+		}
+
+		const total = await Note.count({
+			userId: user._id,
+		});
+
+		job.reportProgress(exportedNotesCount / total);
+	}
+
+	await new Promise((res, rej) => {
+		stream.write(']', err => {
+			if (err) {
+				logger.error(err);
+				rej(err);
+			} else {
+				res();
+			}
+		});
+	});
+
+	stream.end();
+	logger.succ(`Exported to: ${path}`);
+
+	const fileName = dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json';
+	const driveFile = await addFile(user, path, fileName);
+
+	logger.succ(`Exported to: ${driveFile._id}`);
+	cleanup();
+	done();
+}
+
+function serialize(note: INote): any {
+	return {
+		id: note._id,
+		text: note.text,
+		createdAt: note.createdAt,
+		fileIds: note.fileIds,
+		replyId: note.replyId,
+		renoteId: note.renoteId,
+		poll: note.poll,
+		cw: note.cw,
+		viaMobile: note.viaMobile,
+		visibility: note.visibility,
+		visibleUserIds: note.visibleUserIds,
+		appId: note.appId,
+		geo: note.geo,
+		localOnly: note.localOnly
+	};
+}

src/queue/processors/http/deliver.ts

@@ -1,7 +1,7 @@
 import * as bq from 'bee-queue';
 
 import request from '../../../remote/activitypub/request';
-import { queueLogger } from '../..';
+import { queueLogger } from '../../logger';
 
 export default async (job: bq.Job, done: any): Promise<void> => {
 	try {

src/queue/processors/index.ts

@@ -1,10 +1,12 @@
-import deliver from './deliver';
-import processInbox from './process-inbox';
-import { queueLogger } from '../..';
+import deliver from './http/deliver';
+import processInbox from './http/process-inbox';
+import { exportNotes } from './export-notes';
+import { queueLogger } from '../logger';
 
 const handlers: any = {
 	deliver,
 	processInbox,
+	exportNotes,
 };
 
 export default (job: any, done: any) => {

src/server/api/endpoints/i/export-notes.ts

@@ -0,0 +1,18 @@
+import define from '../../define';
+import { createExportNotesJob } from '../../../../queue';
+import ms = require('ms');
+
+export const meta = {
+	secure: true,
+	requireCredential: true,
+	limit: {
+		duration: ms('1day'),
+		max: 1,
+	},
+};
+
+export default define(meta, (ps, user) => new Promise(async (res, rej) => {
+	createExportNotesJob(user);
+
+	res();
+}));