upd: add Facebook to note importing

packages/backend/src/core/QueueService.ts

@@ -301,6 +301,12 @@ export class QueueService {
 		return this.dbQueue.addBulk(jobs);
 	}
 
+	@bindThis
+	public createImportFBToDbJob(user: ThinUser, targets: string[]) {
+		const jobs = targets.map(rel => this.generateToDbJobData('importFBToDb', { user, target: rel }));
+		return this.dbQueue.addBulk(jobs);
+	}
+
 	@bindThis
 	public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) {
 		const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies }));
@@ -336,7 +342,7 @@ export class QueueService {
 	}
 
 	@bindThis
-	private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb' | 'importTweetsToDb' | 'importIGToDb' | 'importMastoToDb' | 'importPleroToDb' | 'importKeyNotesToDb', D extends DbJobData<T>>(name: T, data: D): {
+	private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb' | 'importTweetsToDb' | 'importIGToDb' | 'importFBToDb' | 'importMastoToDb' | 'importPleroToDb' | 'importKeyNotesToDb', D extends DbJobData<T>>(name: T, data: D): {
 		name: string,
 		data: D,
 		opts: Bull.JobsOptions,

packages/backend/src/queue/QueueProcessorService.ts

@@ -179,6 +179,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
 				case 'importNotes': return this.importNotesProcessorService.process(job);
 				case 'importTweetsToDb': return this.importNotesProcessorService.processTwitterDb(job);
 				case 'importIGToDb': return this.importNotesProcessorService.processIGDb(job);
+				case 'importFBToDb': return this.importNotesProcessorService.processFBDb(job);
 				case 'importMastoToDb': return this.importNotesProcessorService.processMastoToDb(job);
 				case 'importPleroToDb': return this.importNotesProcessorService.processPleroToDb(job);
 				case 'importKeyNotesToDb': return this.importNotesProcessorService.processKeyNotesToDb(job);

packages/backend/src/queue/processors/ImportNotesProcessorService.ts

@@ -1,13 +1,11 @@
 import * as fs from 'node:fs';
 import * as vm from 'node:vm';
 import { Inject, Injectable } from '@nestjs/common';
-import { IsNull } from 'typeorm';
 import { ZipReader } from 'slacc';
 import { DI } from '@/di-symbols.js';
-import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository } from '@/models/_.js';
+import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository, MiUser } from '@/models/_.js';
 import type Logger from '@/logger.js';
 import { DownloadService } from '@/core/DownloadService.js';
-import { UtilityService } from '@/core/UtilityService.js';
 import { bindThis } from '@/decorators.js';
 import { QueueService } from '@/core/QueueService.js';
 import { createTemp, createTempDir } from '@/misc/create-temp.js';
@@ -35,7 +33,6 @@ export class ImportNotesProcessorService {
 		private notesRepository: NotesRepository,
 
 		private queueService: QueueService,
-		private utilityService: UtilityService,
 		private noteCreateService: NoteCreateService,
 		private mfmService: MfmService,
 		private apNoteService: ApNoteService,
@@ -47,9 +44,9 @@ export class ImportNotesProcessorService {
 	}
 
 	@bindThis
-	private async uploadFiles(dir: any, user: any) {
+	private async uploadFiles(dir: string, user: MiUser) {
 		const fileList = fs.readdirSync(dir);
-		for (const file of fileList) {
+		for await (const file of fileList) {
 			const name = `${dir}/${file}`;
 			if (fs.statSync(name).isDirectory()) {
 				await this.uploadFiles(name, user);
@@ -172,6 +169,34 @@ export class ImportNotesProcessorService {
 			} finally {
 				cleanup();
 			}
+		} else if (type === 'Facebook' || file.name.startsWith('facebook-') && file.name.endsWith('.zip')) {
+			const [path, cleanup] = await createTempDir();
+
+			this.logger.info(`Temp dir is ${path}`);
+
+			const destPath = path + '/facebook.zip';
+
+			try {
+				fs.writeFileSync(destPath, '', 'binary');
+				await this.downloadService.downloadUrl(file.url, destPath);
+			} catch (e) { // TODO: 何度か再試行
+				if (e instanceof Error || typeof e === 'string') {
+					this.logger.error(e);
+				}
+				throw e;
+			}
+
+			const outputPath = path + '/facebook';
+			try {
+				this.logger.succ(`Unzipping to ${outputPath}`);
+				ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
+				const postsJson = fs.readFileSync(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8');
+				const posts = JSON.parse(postsJson);
+				await this.uploadFiles(outputPath + '/your_activity_across_facebook/posts/media', user);
+				this.queueService.createImportFBToDbJob(job.data.user, posts);
+			} finally {
+				cleanup();
+			}
 		} else if (file.name.endsWith('.zip')) {
 			const [path, cleanup] = await createTempDir();
 
@@ -535,4 +560,41 @@ export class ImportNotesProcessorService {
 			this.logger.warn(`Error: ${e}`);
 		}
 	}
+
+	@bindThis
+	public async processFBDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
+		const post = job.data.target;
+		const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
+		if (user == null) {
+			return;
+		}
+
+		if (!this.isIterable(post.data) || !post.data[0].post) return;
+
+		const date = new Date(post.timestamp * 1000);
+		const title = decodeFBString(post.data[0].post);
+		const files: MiDriveFile[] = [];
+
+		function decodeFBString(str: string) {
+			const arr = [];
+			for (let i = 0; i < str.length; i++) {
+				arr.push(str.charCodeAt(i));
+			}
+			return Buffer.from(arr).toString('utf8');
+		}
+
+		if (post.attachments && this.isIterable(post.attachments) && this.isIterable(post.attachments.data)) {
+			for await (const file of post.attachments.data) {
+				if (!file.media) return;
+				const slashdex = file.media.uri.lastIndexOf('/');
+				const name = file.media.uri.substring(slashdex + 1);
+				const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id });
+				if (exists) {
+					files.push(exists);
+				}
+			}
+		}
+
+		await this.noteCreateService.import(user, { createdAt: date, text: title, files: files });
+	}
 }

packages/backend/src/queue/types.ts

@@ -52,6 +52,7 @@ export type DbJobMap = {
 	importNotes: DbNoteImportJobData;
 	importTweetsToDb: DbKeyNoteImportToDbJobData;
 	importIGToDb: DbNoteImportToDbJobData;
+	importFBToDb: DbNoteImportToDbJobData;
 	importMastoToDb: DbNoteImportToDbJobData;
 	importPleroToDb: DbNoteImportToDbJobData;
 	importKeyNotesToDb: DbKeyNoteImportToDbJobData;

packages/frontend/src/pages/settings/import-export.vue

@@ -22,6 +22,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 					<option value="Mastodon">Mastodon/Pleroma/Akkoma</option>
 					<option value="Twitter">Twitter</option>
 					<option value="Instagram">Instagram</option>
+					<option value="Facebook">Facebook</option>
 				</MkRadios>
 				<MkButton primary :class="$style.button" inline @click="importNotes($event)"><i class="ph-upload ph-bold ph-lg"></i> {{ i18n.ts.import }}</MkButton>
 			</MkFolder>