graphite/Sharkey
1
0
Fork 0
mirror of https://git.joinsharkey.org/Sharkey/Sharkey.git synced 2024-11-27 02:03:09 +02:00
Code Issues Projects Releases Packages Wiki Activity

make almost all fs ops async

Browse source 
there's no `fs.promises.exists`
This commit is contained in:
dakkar 2024-02-03 11:33:42 +00:00
parent 7684f45a5e
commit 3a3a051bb5
1 changed files with 12 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
packages/backend/src/queue/processors/ImportNotesProcessorService.ts
View file

@ -51,7 +51,7 @@ export class ImportNotesProcessorService {
@bindThis @bindThis
private async uploadFiles(dir: string, user: MiUser, folder?: MiDriveFolder['id']) { private async uploadFiles(dir: string, user: MiUser, folder?: MiDriveFolder['id']) {
const fileList = fs.readdirSync(dir); const fileList = await fs.promises.readdir(dir);
for await (const file of fileList) { for await (const file of fileList) {
const name = `${dir}/${file}`; const name = `${dir}/${file}`;
if (fs.statSync(name).isDirectory()) { if (fs.statSync(name).isDirectory()) {
@ -173,7 +173,7 @@ export class ImportNotesProcessorService {
const destPath = path + '/twitter.zip'; const destPath = path + '/twitter.zip';
try { try {
fs.writeFileSync(destPath, '', 'binary'); await fs.promises.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath); await this.downloadService.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
@ -187,7 +187,7 @@ export class ImportNotesProcessorService {
this.logger.succ(`Unzipping to ${outputPath}`); this.logger.succ(`Unzipping to ${outputPath}`);
ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
const unprocessedTweets = this.parseTwitterFile(fs.readFileSync(outputPath + '/data/tweets.js', 'utf-8')); const unprocessedTweets = this.parseTwitterFile(await fs.promises.readFile(outputPath + '/data/tweets.js', 'utf-8'));
//Make sure that it isnt null (because if something went wrong in parseTwitterFile it returns null) //Make sure that it isnt null (because if something went wrong in parseTwitterFile it returns null)
if (unprocessedTweetJson) { if (unprocessedTweetJson) {
@ -208,7 +208,7 @@ export class ImportNotesProcessorService {
const destPath = path + '/facebook.zip'; const destPath = path + '/facebook.zip';
try { try {
fs.writeFileSync(destPath, '', 'binary'); await fs.promises.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath); await this.downloadService.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
@ -221,7 +221,7 @@ export class ImportNotesProcessorService {
try { try {
this.logger.succ(`Unzipping to ${outputPath}`); this.logger.succ(`Unzipping to ${outputPath}`);
ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
const postsJson = fs.readFileSync(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8'); const postsJson = await fs.promises.readFile(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8');
const posts = JSON.parse(postsJson); const posts = JSON.parse(postsJson);
const facebookFolder = await this.driveFoldersRepository.findOneBy({ name: 'Facebook', userId: job.data.user.id, parentId: folder?.id }); const facebookFolder = await this.driveFoldersRepository.findOneBy({ name: 'Facebook', userId: job.data.user.id, parentId: folder?.id });
if (facebookFolder == null && folder) { if (facebookFolder == null && folder) {
@ -241,7 +241,7 @@ export class ImportNotesProcessorService {
const destPath = path + '/unknown.zip'; const destPath = path + '/unknown.zip';
try { try {
fs.writeFileSync(destPath, '', 'binary'); await fs.promises.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath); await this.downloadService.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
@ -257,7 +257,7 @@ export class ImportNotesProcessorService {
const isInstagram = type === 'Instagram' || fs.existsSync(outputPath + '/instagram_live') || fs.existsSync(outputPath + '/instagram_ads_and_businesses'); const isInstagram = type === 'Instagram' || fs.existsSync(outputPath + '/instagram_live') || fs.existsSync(outputPath + '/instagram_ads_and_businesses');
const isOutbox = type === 'Mastodon' || fs.existsSync(outputPath + '/outbox.json'); const isOutbox = type === 'Mastodon' || fs.existsSync(outputPath + '/outbox.json');
if (isInstagram) { if (isInstagram) {
const postsJson = fs.readFileSync(outputPath + '/content/posts_1.json', 'utf-8'); const postsJson = await fs.promises.readFile(outputPath + '/content/posts_1.json', 'utf-8');
const posts = JSON.parse(postsJson); const posts = JSON.parse(postsJson);
const igFolder = await this.driveFoldersRepository.findOneBy({ name: 'Instagram', userId: job.data.user.id, parentId: folder?.id }); const igFolder = await this.driveFoldersRepository.findOneBy({ name: 'Instagram', userId: job.data.user.id, parentId: folder?.id });
if (igFolder == null && folder) { if (igFolder == null && folder) {
@ -267,16 +267,16 @@ export class ImportNotesProcessorService {
} }
this.queueService.createImportIGToDbJob(job.data.user, posts); this.queueService.createImportIGToDbJob(job.data.user, posts);
} else if (isOutbox) { } else if (isOutbox) {
const actorJson = fs.readFileSync(outputPath + '/actor.json', 'utf-8'); const actorJson = await fs.promises.readFile(outputPath + '/actor.json', 'utf-8');
const actor = JSON.parse(actorJson); const actor = JSON.parse(actorJson);
const isPleroma = actor['@context'].some((v: any) => typeof v === 'string' && v.match(/litepub(.*)/)); const isPleroma = actor['@context'].some((v: any) => typeof v === 'string' && v.match(/litepub(.*)/));
if (isPleroma) { if (isPleroma) {
const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8'); const outboxJson = await fs.promises.readFile(outputPath + '/outbox.json', 'utf-8');
const outbox = JSON.parse(outboxJson); const outbox = JSON.parse(outboxJson);
const processedToots = await this.recreateChain(['object', 'id'], ['object', 'inReplyTo'], outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'), true); const processedToots = await this.recreateChain(['object', 'id'], ['object', 'inReplyTo'], outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'), true);
this.queueService.createImportPleroToDbJob(job.data.user, processedToots, null); this.queueService.createImportPleroToDbJob(job.data.user, processedToots, null);
} else { } else {
const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8'); const outboxJson = await fs.promises.readFile(outputPath + '/outbox.json', 'utf-8');
const outbox = JSON.parse(outboxJson); const outbox = JSON.parse(outboxJson);
let mastoFolder = await this.driveFoldersRepository.findOneBy({ name: 'Mastodon', userId: job.data.user.id, parentId: folder?.id }); let mastoFolder = await this.driveFoldersRepository.findOneBy({ name: 'Mastodon', userId: job.data.user.id, parentId: folder?.id });
if (mastoFolder == null && folder) { if (mastoFolder == null && folder) {
@ -299,7 +299,7 @@ export class ImportNotesProcessorService {
this.logger.info(`Temp dir is ${path}`); this.logger.info(`Temp dir is ${path}`);
try { try {
fs.writeFileSync(path, '', 'utf-8'); await fs.promises.writeFile(path, '', 'utf-8');
await this.downloadService.downloadUrl(file.url, path); await this.downloadService.downloadUrl(file.url, path);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
@ -308,7 +308,7 @@ export class ImportNotesProcessorService {
throw e; throw e;
} }
const notesJson = fs.readFileSync(path, 'utf-8'); const notesJson = await fs.promises.readFile(path, 'utf-8');
const notes = JSON.parse(notesJson); const notes = JSON.parse(notesJson);
const processedNotes = await this.recreateChain(['id'], ['replyId'], notes, false); const processedNotes = await this.recreateChain(['id'], ['replyId'], notes, false);
this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null); this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null);