From a80003cde57da44b39189e32d43c5297f963f0cf Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 28 May 2023 20:58:39 +0900 Subject: [PATCH 01/20] =?UTF-8?q?fix(frontend):=20Zen=20UI=E3=81=A7?= =?UTF-8?q?=E3=80=81=E3=83=87=E3=83=83=E3=82=AD=E8=A8=AD=E5=AE=9A=E3=81=A7?= =?UTF-8?q?=E7=9B=B4=E6=8E=A5/=E4=BB=A5=E5=A4=96=E3=82=92=E8=A1=A8?= =?UTF-8?q?=E7=A4=BA=E3=81=97=E3=81=9F=E3=81=A8=E3=81=8D=E3=83=87=E3=83=83?= =?UTF-8?q?=E3=82=AD=E3=81=AB=E6=88=BB=E3=82=8B=E3=83=9C=E3=82=BF=E3=83=B3?= =?UTF-8?q?=E3=82=92=E8=A1=A8=E7=A4=BA=20(#10909)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(frontend): デッキ設定で直接/以外を表示したときのZen UIでデッキに戻るボタン * fix style * ?zenが指定されていた場合はボタンを表示しない --- packages/frontend/src/ui/zen.vue | 44 ++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/frontend/src/ui/zen.vue b/packages/frontend/src/ui/zen.vue index e656f00bb..d516a5df7 100644 --- a/packages/frontend/src/ui/zen.vue +++ b/packages/frontend/src/ui/zen.vue @@ -1,9 +1,17 @@ @@ -31,4 +46,29 @@ document.documentElement.style.overflowY = 'scroll'; min-height: 100dvh; box-sizing: border-box; } + +.rootWithBottom { + min-height: calc(100dvh - (60px + (var(--margin) * 2) + env(safe-area-inset-bottom, 0px))); + box-sizing: border-box; +} + +.bottom { + height: calc(60px + (var(--margin) * 2) + env(safe-area-inset-bottom, 0px)); + width: 100%; + margin-top: auto; +} + +.button { + position: fixed !important; + padding: 0; + aspect-ratio: 1; + width: 100%; + max-width: 60px; + margin: auto; + border-radius: 100%; + background: var(--panel); + color: var(--fg); + right: var(--margin); + bottom: calc(var(--margin) + env(safe-area-inset-bottom, 0px)); +} From 7cbd852fe5a2d1ef84049c48c30f51f1170cc5de Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 29 May 2023 06:37:13 +0900 Subject: [PATCH 02/20] =?UTF-8?q?pnpm=20dev=E3=81=A7Ctrl+C=E3=81=A7?= =?UTF-8?q?=E7=B5=82=E4=BA=86=E3=81=95=E3=81=9B=E3=81=A6=E3=82=82=E3=83=97?= =?UTF-8?q?=E3=83=AD=E3=82=BB=E3=82=B9=E3=81=8C=E5=AE=8C=E5=85=A8=E3=81=AB?= =?UTF-8?q?=E6=AE=BA=E3=81=9B=E3=81=AA=E3=81=84=E3=81=AE=E3=82=92=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=20(#10914)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/dev.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/dev.js b/scripts/dev.js index db7bc11fe..2f20d8f07 100644 --- a/scripts/dev.js +++ b/scripts/dev.js @@ -44,11 +44,17 @@ const fs = require('fs'); if (!stat) throw new Error('not exist yet'); if (stat.size === 0) throw new Error('not built yet'); - await execa('pnpm', ['start'], { + const subprocess = await execa('pnpm', ['start'], { cwd: __dirname + '/../', stdout: process.stdout, stderr: process.stderr, }); + + // なぜかworkerだけが終了してmasterが残るのでその対策 + process.on('SIGINT', () => { + subprocess.kill('SIGINT'); + process.exit(0); + }); } catch (e) { await new Promise(resolve => setTimeout(resolve, 3000)); start(); From fd7b77c542b51313d8b8ea60124725fe65a295d5 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 11:54:49 +0900 Subject: [PATCH 03/20] enhance(backend): migrate bull to bullmq (#10910) * wip * wip * Update QueueService.ts * wip * refactor * :v: * fix * Update QueueStatsService.ts * refactor * Update ApNoteService.ts * Update mock-resolver.ts * refactor * Update mock-resolver.ts --- CHANGELOG.md | 1 + packages/backend/package.json | 3 +- packages/backend/src/config.ts | 2 +- packages/backend/src/core/CaptchaService.ts | 20 +- .../src/core/FetchInstanceMetadataService.ts | 6 +- .../backend/src/core/NoteCreateService.ts | 2 +- packages/backend/src/core/QueueModule.ts | 53 +-- packages/backend/src/core/QueueService.ts | 65 +++- .../core/activitypub/LdSignatureService.ts | 4 +- .../core/activitypub/models/ApNoteService.ts | 6 +- .../backend/src/daemons/QueueStatsService.ts | 16 +- packages/backend/src/misc/id/aid.ts | 2 +- packages/backend/src/misc/prelude/time.ts | 17 +- .../src/queue/QueueProcessorService.ts | 320 +++++++++++------- packages/backend/src/queue/const.ts | 26 ++ packages/backend/src/queue/get-job-info.ts | 15 - .../AggregateRetentionProcessorService.ts | 6 +- .../CheckExpiredMutingsProcessorService.ts | 5 +- .../processors/CleanChartsProcessorService.ts | 5 +- .../queue/processors/CleanProcessorService.ts | 5 +- .../CleanRemoteFilesProcessorService.ts | 11 +- .../DeleteAccountProcessorService.ts | 6 +- .../DeleteDriveFilesProcessorService.ts | 14 +- .../processors/DeleteFileProcessorService.ts | 6 +- .../processors/DeliverProcessorService.ts | 10 +- .../EndedPollNotificationProcessorService.ts | 7 +- .../ExportAntennasProcessorService.ts | 6 +- .../ExportBlockingProcessorService.ts | 15 +- .../ExportCustomEmojisProcessorService.ts | 39 +-- .../ExportFavoritesProcessorService.ts | 11 +- .../ExportFollowingProcessorService.ts | 11 +- .../ExportMutingProcessorService.ts | 15 +- .../processors/ExportNotesProcessorService.ts | 11 +- .../ExportUserListsProcessorService.ts | 11 +- .../ImportAntennasProcessorService.ts | 6 +- .../ImportBlockingProcessorService.ts | 13 +- .../ImportCustomEmojisProcessorService.ts | 6 +- .../ImportFollowingProcessorService.ts | 13 +- .../ImportMutingProcessorService.ts | 13 +- .../ImportUserListsProcessorService.ts | 7 +- .../queue/processors/InboxProcessorService.ts | 38 +-- .../RelationshipProcessorService.ts | 2 +- .../ResyncChartsProcessorService.ts | 5 +- .../processors/TickChartsProcessorService.ts | 5 +- .../WebhookDeliverProcessorService.ts | 6 +- .../server/api/endpoints/admin/relays/add.ts | 2 +- .../api/endpoints/notes/search-by-tag.ts | 4 +- .../src/server/api/endpoints/reset-db.ts | 2 +- packages/backend/test/misc/mock-resolver.ts | 6 +- packages/backend/test/utils.ts | 2 +- packages/frontend/src/scripts/time.ts | 17 +- packages/shared/.eslintrc.js | 2 +- pnpm-lock.yaml | 121 ++++--- 53 files changed, 532 insertions(+), 490 deletions(-) create mode 100644 packages/backend/src/queue/const.ts delete mode 100644 packages/backend/src/queue/get-job-info.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ecbad51e9..6b107c72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ - fix:ロールタイムラインにて全ての投稿が流れてしまう問題の修正 ### Server +- bullをbull-mqにアップグレードし、ジョブキューのパフォーマンスを改善 - Fix: お知らせの画像URLを空にできない問題を修正 ## 13.12.2 diff --git a/packages/backend/package.json b/packages/backend/package.json index 71da3bb55..99c04d6bf 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -76,7 +76,7 @@ "autwh": "0.1.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", - "bull": "4.10.4", + "bullmq": "3.14.1", "cacheable-lookup": "6.1.0", "cbor": "8.1.0", "chalk": "5.2.0", @@ -167,7 +167,6 @@ "@types/accepts": "1.3.5", "@types/archiver": "5.3.2", "@types/bcryptjs": "2.4.2", - "@types/bull": "4.10.0", "@types/cbor": "6.0.0", "@types/color-convert": "2.0.0", "@types/content-disposition": "0.5.5", diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 744f99959..9d1945e4d 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -190,6 +190,6 @@ function tryCreateUrl(url: string) { try { return new URL(url); } catch (e) { - throw `url="${url}" is not a valid URL.`; + throw new Error(`url="${url}" is not a valid URL.`); } } diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 7aaa1b833..1a52a229c 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -30,7 +30,7 @@ export class CaptchaService { }, { throwErrorWhenResponseNotOk: false }); if (!res.ok) { - throw `${res.status}`; + throw new Error(`${res.status}`); } return await res.json() as CaptchaResponse; @@ -39,48 +39,48 @@ export class CaptchaService { @bindThis public async verifyRecaptcha(secret: string, response: string | null | undefined): Promise { if (response == null) { - throw 'recaptcha-failed: no response provided'; + throw new Error('recaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => { - throw `recaptcha-request-failed: ${err}`; + throw new Error(`recaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw `recaptcha-failed: ${errorCodes}`; + throw new Error(`recaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyHcaptcha(secret: string, response: string | null | undefined): Promise { if (response == null) { - throw 'hcaptcha-failed: no response provided'; + throw new Error('hcaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => { - throw `hcaptcha-request-failed: ${err}`; + throw new Error(`hcaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw `hcaptcha-failed: ${errorCodes}`; + throw new Error(`hcaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyTurnstile(secret: string, response: string | null | undefined): Promise { if (response == null) { - throw 'turnstile-failed: no response provided'; + throw new Error('turnstile-failed: no response provided'); } const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => { - throw `turnstile-request-failed: ${err}`; + throw new Error(`turnstile-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw `turnstile-failed: ${errorCodes}`; + throw new Error(`turnstile-failed: ${errorCodes}`); } } } diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 8103d5afe..9de633350 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -116,14 +116,14 @@ export class FetchInstanceMetadataService { const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo') .catch(err => { if (err.statusCode === 404) { - throw 'No nodeinfo provided'; + throw new Error('No nodeinfo provided'); } else { throw err.statusCode ?? err.message; } }) as Record; if (wellknown.links == null || !Array.isArray(wellknown.links)) { - throw 'No wellknown links'; + throw new Error('No wellknown links'); } const links = wellknown.links as any[]; @@ -134,7 +134,7 @@ export class FetchInstanceMetadataService { const link = lnik2_1 ?? lnik2_0 ?? lnik1_0; if (link == null) { - throw 'No nodeinfo link provided'; + throw new Error('No nodeinfo link provided'); } const info = await this.httpRequestService.getJson(link.href) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 977c9052c..2fd7a8ac8 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -510,7 +510,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - Date.now(); - this.queueService.endedPollNotificationQueue.add({ + this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { delay, diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index 1d7394777..6db9bb14c 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -1,42 +1,11 @@ import { setTimeout } from 'node:timers/promises'; import { Inject, Module, OnApplicationShutdown } from '@nestjs/common'; -import Bull from 'bull'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { Provider } from '@nestjs/common'; -import type { DeliverJobData, InboxJobData, DbJobData, ObjectStorageJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData, DbJobMap } from '../queue/types.js'; - -function q(config: Config, name: string, limitPerSec = -1) { - return new Bull(name, { - redis: { - port: config.redisForJobQueue.port, - host: config.redisForJobQueue.host, - family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, - password: config.redisForJobQueue.pass, - db: config.redisForJobQueue.db ?? 0, - }, - prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue` : 'queue', - limiter: limitPerSec > 0 ? { - max: limitPerSec, - duration: 1000, - } : undefined, - settings: { - backoffStrategies: { - apBackoff, - }, - }, - }); -} - -// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 -function apBackoff(attemptsMade: number, err: Error) { - const baseDelay = 60 * 1000; // 1min - const maxBackoff = 8 * 60 * 60 * 1000; // 8hours - let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; - backoff = Math.min(backoff, maxBackoff); - backoff += Math.round(backoff * Math.random() * 0.2); - return backoff; -} +import type { DeliverJobData, InboxJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData } from '../queue/types.js'; export type SystemQueue = Bull.Queue>; export type EndedPollNotificationQueue = Bull.Queue; @@ -49,49 +18,49 @@ export type WebhookDeliverQueue = Bull.Queue; const $system: Provider = { provide: 'queue:system', - useFactory: (config: Config) => q(config, 'system'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)), inject: [DI.config], }; const $endedPollNotification: Provider = { provide: 'queue:endedPollNotification', - useFactory: (config: Config) => q(config, 'endedPollNotification'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)), inject: [DI.config], }; const $deliver: Provider = { provide: 'queue:deliver', - useFactory: (config: Config) => q(config, 'deliver', config.deliverJobPerSec ?? 128), + useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)), inject: [DI.config], }; const $inbox: Provider = { provide: 'queue:inbox', - useFactory: (config: Config) => q(config, 'inbox', config.inboxJobPerSec ?? 16), + useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)), inject: [DI.config], }; const $db: Provider = { provide: 'queue:db', - useFactory: (config: Config) => q(config, 'db'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)), inject: [DI.config], }; const $relationship: Provider = { provide: 'queue:relationship', - useFactory: (config: Config) => q(config, 'relationship', config.relashionshipJobPerSec ?? 64), + useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)), inject: [DI.config], }; const $objectStorage: Provider = { provide: 'queue:objectStorage', - useFactory: (config: Config) => q(config, 'objectStorage'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)), inject: [DI.config], }; const $webhookDeliver: Provider = { provide: 'queue:webhookDeliver', - useFactory: (config: Config) => q(config, 'webhookDeliver', 64), + useFactory: (config: Config) => new Bull.Queue(QUEUE.WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.WEBHOOK_DELIVER)), inject: [DI.config], }; diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index b4ffffecc..2ae8a2b75 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -1,6 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; import { v4 as uuid } from 'uuid'; -import Bull from 'bull'; import type { IActivity } from '@/core/activitypub/type.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; @@ -11,6 +10,7 @@ import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; +import type * as Bull from 'bullmq'; @Injectable() export class QueueService { @@ -26,7 +26,43 @@ export class QueueService { @Inject('queue:relationship') public relationshipQueue: RelationshipQueue, @Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue, @Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue, - ) {} + ) { + this.systemQueue.add('tickCharts', { + }, { + repeat: { pattern: '55 * * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('resyncCharts', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('cleanCharts', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('aggregateRetention', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('clean', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('checkExpiredMutings', { + }, { + repeat: { pattern: '*/5 * * * *' }, + removeOnComplete: true, + }); + } @bindThis public deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { @@ -42,11 +78,10 @@ export class QueueService { isSharedInbox, }; - return this.deliverQueue.add(data, { + return this.deliverQueue.add(to, data, { attempts: this.config.deliverJobMaxAttempts ?? 12, - timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -60,11 +95,10 @@ export class QueueService { signature, }; - return this.inboxQueue.add(data, { + return this.inboxQueue.add('', data, { attempts: this.config.inboxJobMaxAttempts ?? 8, - timeout: 5 * 60 * 1000, // 5min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -212,7 +246,7 @@ export class QueueService { private generateToDbJobData>(name: T, data: D): { name: string, data: D, - opts: Bull.JobOptions, + opts: Bull.JobsOptions, } { return { name, @@ -299,10 +333,10 @@ export class QueueService { } @bindThis - private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobOptions = {}): { + private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobsOptions = {}): { name: string, data: RelationshipJobData, - opts: Bull.JobOptions, + opts: Bull.JobsOptions, } { return { name, @@ -351,11 +385,10 @@ export class QueueService { eventId: uuid(), }; - return this.webhookDeliverQueue.add(data, { + return this.webhookDeliverQueue.add(webhook.id, data, { attempts: 4, - timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -367,11 +400,11 @@ export class QueueService { this.deliverQueue.once('cleaned', (jobs, status) => { //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.deliverQueue.clean(0, 'delayed'); + this.deliverQueue.clean(0, Infinity, 'delayed'); this.inboxQueue.once('cleaned', (jobs, status) => { //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.inboxQueue.clean(0, 'delayed'); + this.inboxQueue.clean(0, Infinity, 'delayed'); } } diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index 2dc1a410a..20fe2a0a7 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -94,7 +94,7 @@ class LdSignature { @bindThis private getLoader() { return async (url: string): Promise => { - if (!url.match('^https?\:\/\/')) throw `Invalid URL ${url}`; + if (!url.match('^https?\:\/\/')) throw new Error(`Invalid URL ${url}`); if (this.preLoad) { if (url in CONTEXTS) { @@ -126,7 +126,7 @@ class LdSignature { timeout: this.loderTimeout, }, { throwErrorWhenResponseNotOk: false }).then(res => { if (!res.ok) { - throw `${res.status} ${res.statusText}`; + throw new Error(`${res.status} ${res.statusText}`); } else { return res.json(); } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 87a9db405..76757f530 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -18,6 +18,7 @@ import { PollService } from '@/core/PollService.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { checkHttps } from '@/misc/check-https.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; // eslint-disable-next-line @typescript-eslint/consistent-type-imports import { ApLoggerService } from '../ApLoggerService.js'; @@ -32,7 +33,6 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; -import { checkHttps } from '@/misc/check-https.js'; @Injectable() export class ApNoteService { @@ -230,7 +230,7 @@ export class ApNoteService { quote = results.filter((x): x is { status: 'ok', res: Note | null } => x.status === 'ok').map(x => x.res).find(x => x); if (!quote) { if (results.some(x => x.status === 'temperror')) { - throw 'quote resolve failed'; + throw new Error('quote resolve failed'); } } } @@ -311,7 +311,7 @@ export class ApNoteService { // ブロックしてたら中断 const meta = await this.metaService.fetch(); - if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) throw { statusCode: 451 }; + if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) throw new StatusError('blocked host', 451); const unlock = await this.appLockService.getApLock(uri); diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index b717434e0..0a5b3184d 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -1,7 +1,11 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import Xev from 'xev'; +import * as Bull from 'bullmq'; import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { OnApplicationShutdown } from '@nestjs/common'; const ev = new Xev(); @@ -13,6 +17,9 @@ export class QueueStatsService implements OnApplicationShutdown { private intervalId: NodeJS.Timer; constructor( + @Inject(DI.config) + private config: Config, + private queueService: QueueService, ) { } @@ -31,11 +38,14 @@ export class QueueStatsService implements OnApplicationShutdown { let activeDeliverJobs = 0; let activeInboxJobs = 0; - this.queueService.deliverQueue.on('global:active', () => { + const deliverQueueEvents = new Bull.QueueEvents(QUEUE.DELIVER, baseQueueOptions(this.config, QUEUE.DELIVER)); + const inboxQueueEvents = new Bull.QueueEvents(QUEUE.INBOX, baseQueueOptions(this.config, QUEUE.INBOX)); + + deliverQueueEvents.on('active', () => { activeDeliverJobs++; }); - this.queueService.inboxQueue.on('global:active', () => { + inboxQueueEvents.on('active', () => { activeInboxJobs++; }); diff --git a/packages/backend/src/misc/id/aid.ts b/packages/backend/src/misc/id/aid.ts index 9e206ee98..f0cbc9900 100644 --- a/packages/backend/src/misc/id/aid.ts +++ b/packages/backend/src/misc/id/aid.ts @@ -21,7 +21,7 @@ function getNoise(): string { export function genAid(date: Date): string { const t = date.getTime(); - if (isNaN(t)) throw 'Failed to create AID: Invalid Date'; + if (isNaN(t)) throw new Error('Failed to create AID: Invalid Date'); counter++; return getTime(t) + getNoise(); } diff --git a/packages/backend/src/misc/prelude/time.ts b/packages/backend/src/misc/prelude/time.ts index 34e8b6b17..b21978b18 100644 --- a/packages/backend/src/misc/prelude/time.ts +++ b/packages/backend/src/misc/prelude/time.ts @@ -5,15 +5,16 @@ const dateTimeIntervals = { }; export function dateUTC(time: number[]): Date { - const d = time.length === 2 ? Date.UTC(time[0], time[1]) - : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) - : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) - : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) - : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) - : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) - : null; + const d = + time.length === 2 ? Date.UTC(time[0], time[1]) + : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) + : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) + : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) + : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) + : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) + : null; - if (!d) throw 'wrong number of arguments'; + if (!d) throw new Error('wrong number of arguments'); return new Date(d); } diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index dc025f988..011082cd3 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -1,10 +1,9 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; -import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; -import { getJobInfo } from './get-job-info.js'; import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; @@ -35,6 +34,33 @@ import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMu import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; +import { QUEUE, baseQueueOptions } from './const.js'; + +// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 +function httpRelatedBackoff(attemptsMade: number) { + const baseDelay = 60 * 1000; // 1min + const maxBackoff = 8 * 60 * 60 * 1000; // 8hours + let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; + backoff = Math.min(backoff, maxBackoff); + backoff += Math.round(backoff * Math.random() * 0.2); + return backoff; +} + +function getJobInfo(job: Bull.Job | undefined, increment = false): string { + if (job == null) return '-'; + + const age = Date.now() - job.timestamp; + + const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` + : age > 10000 ? `${Math.floor(age / 1000)}s` + : `${age}ms`; + + // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする + const currentAttempts = job.attemptsMade + (increment ? 1 : 0); + const maxAttempts = job.opts ? job.opts.attempts : 0; + + return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; +} @Injectable() export class QueueProcessorService { @@ -45,7 +71,6 @@ export class QueueProcessorService { private config: Config, private queueLoggerService: QueueLoggerService, - private queueService: QueueService, private webhookDeliverProcessorService: WebhookDeliverProcessorService, private endedPollNotificationProcessorService: EndedPollNotificationProcessorService, private deliverProcessorService: DeliverProcessorService, @@ -97,146 +122,191 @@ export class QueueProcessorService { } } - const systemLogger = this.logger.createSubLogger('system'); - const deliverLogger = this.logger.createSubLogger('deliver'); - const webhookLogger = this.logger.createSubLogger('webhook'); - const inboxLogger = this.logger.createSubLogger('inbox'); - const dbLogger = this.logger.createSubLogger('db'); - const relationshipLogger = this.logger.createSubLogger('relationship'); - const objectStorageLogger = this.logger.createSubLogger('objectStorage'); + //#region system + const systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { + switch (job.name) { + case 'tickCharts': return this.tickChartsProcessorService.process(); + case 'resyncCharts': return this.resyncChartsProcessorService.process(); + case 'cleanCharts': return this.cleanChartsProcessorService.process(); + case 'aggregateRetention': return this.aggregateRetentionProcessorService.process(); + case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process(); + case 'clean': return this.cleanProcessorService.process(); + default: throw new Error(`unrecognized job type ${job.name} for system`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.SYSTEM), + }); - this.queueService.systemQueue - .on('waiting', (jobId) => systemLogger.debug(`waiting id=${jobId}`)) + const systemLogger = this.logger.createSubLogger('system'); + + systemQueueWorker .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => systemLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => systemLogger.warn(`stalled id=${job.id}`)); + .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => systemLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.deliverQueue - .on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) - .on('error', (job: any, err: Error) => deliverLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); + //#region db + const dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { + switch (job.name) { + case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); + case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); + case 'exportNotes': return this.exportNotesProcessorService.process(job); + case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); + case 'exportFollowing': return this.exportFollowingProcessorService.process(job); + case 'exportMuting': return this.exportMutingProcessorService.process(job); + case 'exportBlocking': return this.exportBlockingProcessorService.process(job); + case 'exportUserLists': return this.exportUserListsProcessorService.process(job); + case 'exportAntennas': return this.exportAntennasProcessorService.process(job); + case 'importFollowing': return this.importFollowingProcessorService.process(job); + case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); + case 'importMuting': return this.importMutingProcessorService.process(job); + case 'importBlocking': return this.importBlockingProcessorService.process(job); + case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job); + case 'importUserLists': return this.importUserListsProcessorService.process(job); + case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job); + case 'importAntennas': return this.importAntennasProcessorService.process(job); + case 'deleteAccount': return this.deleteAccountProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for db`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.DB), + }); - this.queueService.inboxQueue - .on('waiting', (jobId) => inboxLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) - .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) - .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => inboxLogger.warn(`stalled ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`)); + const dbLogger = this.logger.createSubLogger('db'); - this.queueService.dbQueue - .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`)) + dbQueueWorker .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); + .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => dbLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.relationshipQueue - .on('waiting', (jobId) => relationshipLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => relationshipLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => relationshipLogger.warn(`stalled id=${job.id}`)); + //#region deliver + const deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.DELIVER), + concurrency: this.config.deliverJobConcurrency ?? 128, + limiter: { + max: this.config.deliverJobPerSec ?? 128, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); - this.queueService.objectStorageQueue - .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`)); + const deliverLogger = this.logger.createSubLogger('deliver'); - this.queueService.webhookDeliverQueue - .on('waiting', (jobId) => webhookLogger.debug(`waiting id=${jobId}`)) + deliverQueueWorker + .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => deliverLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region inbox + const inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.INBOX), + concurrency: this.config.inboxJobConcurrency ?? 16, + limiter: { + max: this.config.inboxJobPerSec ?? 16, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); + + const inboxLogger = this.logger.createSubLogger('inbox'); + + inboxQueueWorker + .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) + .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) + .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => inboxLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region webhook deliver + const webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER), + concurrency: 64, + limiter: { + max: 64, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); + + const webhookLogger = this.logger.createSubLogger('webhook'); + + webhookDeliverQueueWorker .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) - .on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); + .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => webhookLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.systemQueue.add('tickCharts', { + //#region relationship + const relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { + switch (job.name) { + case 'follow': return this.relationshipProcessorService.processFollow(job); + case 'unfollow': return this.relationshipProcessorService.processUnfollow(job); + case 'block': return this.relationshipProcessorService.processBlock(job); + case 'unblock': return this.relationshipProcessorService.processUnblock(job); + default: throw new Error(`unrecognized job type ${job.name} for relationship`); + } }, { - repeat: { cron: '55 * * * *' }, - removeOnComplete: true, + ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP), + concurrency: this.config.relashionshipJobConcurrency ?? 16, + limiter: { + max: this.config.relashionshipJobPerSec ?? 64, + duration: 1000, + }, }); - this.queueService.systemQueue.add('resyncCharts', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('cleanCharts', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('aggregateRetention', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('clean', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('checkExpiredMutings', { - }, { - repeat: { cron: '*/5 * * * *' }, - removeOnComplete: true, - }); - - this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job)); - this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job)); - this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done)); - this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job)); - - this.queueService.dbQueue.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportUserLists', (job, done) => this.exportUserListsProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportAntennas', (job, done) => this.exportAntennasProcessorService.process(job, done)); - this.queueService.dbQueue.process('importFollowing', (job, done) => this.importFollowingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importFollowingToDb', (job) => this.importFollowingProcessorService.processDb(job)); - this.queueService.dbQueue.process('importMuting', (job, done) => this.importMutingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importBlocking', (job, done) => this.importBlockingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importBlockingToDb', (job) => this.importBlockingProcessorService.processDb(job)); - this.queueService.dbQueue.process('importUserLists', (job, done) => this.importUserListsProcessorService.process(job, done)); - this.queueService.dbQueue.process('importCustomEmojis', (job, done) => this.importCustomEmojisProcessorService.process(job, done)); - this.queueService.dbQueue.process('importAntennas', (job, done) => this.importAntennasProcessorService.process(job, done)); - this.queueService.dbQueue.process('deleteAccount', (job) => this.deleteAccountProcessorService.process(job)); - - this.queueService.objectStorageQueue.process('deleteFile', 16, (job) => this.deleteFileProcessorService.process(job)); - this.queueService.objectStorageQueue.process('cleanRemoteFiles', 16, (job, done) => this.cleanRemoteFilesProcessorService.process(job, done)); + const relationshipLogger = this.logger.createSubLogger('relationship'); - { - const maxJobs = this.config.relashionshipJobConcurrency ?? 16; - this.queueService.relationshipQueue.process('follow', maxJobs, (job) => this.relationshipProcessorService.processFollow(job)); - this.queueService.relationshipQueue.process('unfollow', maxJobs, (job) => this.relationshipProcessorService.processUnfollow(job)); - this.queueService.relationshipQueue.process('block', maxJobs, (job) => this.relationshipProcessorService.processBlock(job)); - this.queueService.relationshipQueue.process('unblock', maxJobs, (job) => this.relationshipProcessorService.processUnblock(job)); - } + relationshipQueueWorker + .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => relationshipLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.systemQueue.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done)); - this.queueService.systemQueue.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done)); - this.queueService.systemQueue.process('clean', (job, done) => this.cleanProcessorService.process(job, done)); + //#region object storage + const objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { + switch (job.name) { + case 'deleteFile': return this.deleteFileProcessorService.process(job); + case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for objectStorage`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE), + concurrency: 16, + }); + + const objectStorageLogger = this.logger.createSubLogger('objectStorage'); + + objectStorageQueueWorker + .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => objectStorageLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region ended poll notification + const endedPollNotificationWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), + }); + //#endregion } } diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts new file mode 100644 index 000000000..d240fe70e --- /dev/null +++ b/packages/backend/src/queue/const.ts @@ -0,0 +1,26 @@ +import { Config } from '@/config.js'; +import type * as Bull from 'bullmq'; + +export const QUEUE = { + DELIVER: 'deliver', + INBOX: 'inbox', + SYSTEM: 'system', + ENDED_POLL_NOTIFICATION: 'endedPollNotification', + DB: 'db', + RELATIONSHIP: 'relationship', + OBJECT_STORAGE: 'objectStorage', + WEBHOOK_DELIVER: 'webhookDeliver', +}; + +export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { + return { + connection: { + port: config.redisForJobQueue.port, + host: config.redisForJobQueue.host, + family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, + password: config.redisForJobQueue.pass, + db: config.redisForJobQueue.db ?? 0, + }, + prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, + }; +} diff --git a/packages/backend/src/queue/get-job-info.ts b/packages/backend/src/queue/get-job-info.ts deleted file mode 100644 index d33e349c3..000000000 --- a/packages/backend/src/queue/get-job-info.ts +++ /dev/null @@ -1,15 +0,0 @@ -import Bull from 'bull'; - -export function getJobInfo(job: Bull.Job, increment = false) { - const age = Date.now() - job.timestamp; - - const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` - : age > 10000 ? `${Math.floor(age / 1000)}s` - : `${age}ms`; - - // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする - const currentAttempts = job.attemptsMade + (increment ? 1 : 0); - const maxAttempts = job.opts ? job.opts.attempts : 0; - - return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; -} diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts index e2720b4fe..600ce0828 100644 --- a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -9,7 +9,7 @@ import { deepClone } from '@/misc/clone.js'; import { IdService } from '@/core/IdService.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class AggregateRetentionProcessorService { @@ -32,7 +32,7 @@ export class AggregateRetentionProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Aggregating retention...'); const now = new Date(); @@ -62,7 +62,6 @@ export class AggregateRetentionProcessorService { } catch (err) { if (isDuplicateKeyValueError(err)) { this.logger.succ('Skip because it has already been processed by another worker.'); - done(); return; } throw err; @@ -88,6 +87,5 @@ export class AggregateRetentionProcessorService { } this.logger.succ('Retention aggregated.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts index 2476d71a5..c4ee212ba 100644 --- a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CheckExpiredMutingsProcessorService { @@ -27,7 +27,7 @@ export class CheckExpiredMutingsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Checking expired mutings...'); const expired = await this.mutingsRepository.createQueryBuilder('muting') @@ -41,6 +41,5 @@ export class CheckExpiredMutingsProcessorService { } this.logger.succ('All expired mutings checked.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts index b45816704..22d7c1b4f 100644 --- a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanChartsProcessorService { @@ -45,7 +45,7 @@ export class CleanChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Clean charts...'); await Promise.all([ @@ -64,6 +64,5 @@ export class CleanChartsProcessorService { ]); this.logger.succ('All charts successfully cleaned.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index 1936e8df2..cefa6da5e 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanProcessorService { @@ -36,7 +36,7 @@ export class CleanProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Cleaning...'); this.userIpsRepository.delete({ @@ -72,6 +72,5 @@ export class CleanProcessorService { } this.logger.succ('Cleaned.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 5a33c2718..c54bf59ae 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -5,9 +5,9 @@ import type { DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanRemoteFilesProcessorService { @@ -27,7 +27,7 @@ export class CleanRemoteFilesProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(job: Bull.Job>): Promise { this.logger.info('Deleting cached remote files...'); let deletedCount = 0; @@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService { }); if (files.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -62,10 +62,9 @@ export class CleanRemoteFilesProcessorService { isLink: false, }); - job.progress(deletedCount / total); + job.updateProgress(deletedCount / total); } this.logger.succ('All cached remote files has been deleted.'); - done(); } } diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index e36a78de6..39dd801af 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -8,10 +8,10 @@ import { DriveService } from '@/core/DriveService.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Note } from '@/models/entities/Note.js'; import { EmailService } from '@/core/EmailService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserDeleteJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserDeleteJobData } from '../types.js'; @Injectable() export class DeleteAccountProcessorService { diff --git a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts index 604497cf5..6772c5dc7 100644 --- a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts @@ -5,10 +5,10 @@ import type { UsersRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class DeleteDriveFilesProcessorService { @@ -31,12 +31,11 @@ export class DeleteDriveFilesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Deleting drive files of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -56,7 +55,7 @@ export class DeleteDriveFilesProcessorService { }); if (files.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -71,10 +70,9 @@ export class DeleteDriveFilesProcessorService { userId: user.id, }); - job.progress(deletedCount / total); + job.updateProgress(deletedCount / total); } this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); - done(); } } diff --git a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts index 2fb2f56f8..edf87bd92 100644 --- a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts @@ -3,10 +3,10 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { ObjectStorageFileJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { ObjectStorageFileJobData } from '../types.js'; @Injectable() export class DeleteFileProcessorService { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index f293bd4d7..406e9df85 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -1,4 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, InstancesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -16,7 +17,6 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { DeliverJobData } from '../types.js'; @Injectable() @@ -121,15 +121,13 @@ export class DeliverProcessorService { isSuspended: true, }); }); - return `${host} is gone`; + throw new Bull.UnrecoverableError(`${host} is gone`); } - // HTTPステータスコード4xxはクライアントエラーであり、それはつまり - // 何回再送しても成功することはないということなのでエラーにはしないでおく - return `${res.statusCode} ${res.statusMessage}`; + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } // 5xx etc. - throw `${res.statusCode} ${res.statusMessage}`; + throw new Error(`${res.statusCode} ${res.statusMessage}`); } else { // DNS error, socket error, timeout ... throw res; diff --git a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts index 501ed4090..21501592f 100644 --- a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts +++ b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts @@ -6,7 +6,7 @@ import type Logger from '@/logger.js'; import { NotificationService } from '@/core/NotificationService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { EndedPollNotificationJobData } from '../types.js'; @Injectable() @@ -30,10 +30,9 @@ export class EndedPollNotificationProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const note = await this.notesRepository.findOneBy({ id: job.data.noteId }); if (note == null || !note.hasPoll) { - done(); return; } @@ -51,7 +50,5 @@ export class EndedPollNotificationProcessorService { noteId: note.id, }); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index 894903e79..ac52325c8 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -12,7 +12,7 @@ import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DBExportAntennasData } from '../types.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ExportAntennasProcessorService { @@ -39,10 +39,9 @@ export class ExportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } const [path, cleanup] = await createTemp(); @@ -96,7 +95,6 @@ export class ExportAntennasProcessorService { this.logger.succ('Exported to: ' + driveFile.id); } finally { cleanup(); - done(); } } } diff --git a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts index c7b54070d..eb758e162 100644 --- a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class ExportBlockingProcessorService { @@ -36,12 +36,11 @@ export class ExportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -69,7 +68,7 @@ export class ExportBlockingProcessorService { }); if (blockings.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -99,7 +98,7 @@ export class ExportBlockingProcessorService { blockerId: user.id, }); - job.progress(exportedCount / total); + job.updateProgress(exportedCount / total); } stream.end(); @@ -112,7 +111,5 @@ export class ExportBlockingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index b50f373ef..3203d9f3e 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -13,7 +13,7 @@ import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ExportCustomEmojisProcessorService { @@ -37,12 +37,11 @@ export class ExportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Exporting custom emojis ...'); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -117,24 +116,26 @@ export class ExportCustomEmojisProcessorService { metaStream.end(); // Create archive - const [archivePath, archiveCleanup] = await createTemp(); - const archiveStream = fs.createWriteStream(archivePath); - const archive = archiver('zip', { - zlib: { level: 0 }, - }); - archiveStream.on('close', async () => { - this.logger.succ(`Exported to: ${archivePath}`); + await new Promise(async (resolve) => { + const [archivePath, archiveCleanup] = await createTemp(); + const archiveStream = fs.createWriteStream(archivePath); + const archive = archiver('zip', { + zlib: { level: 0 }, + }); + archiveStream.on('close', async () => { + this.logger.succ(`Exported to: ${archivePath}`); - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; - const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; + const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); - this.logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); - archiveCleanup(); - done(); + this.logger.succ(`Exported to: ${driveFile.id}`); + cleanup(); + archiveCleanup(); + resolve(); + }); + archive.pipe(archiveStream); + archive.directory(path, false); + archive.finalize(); }); - archive.pipe(archiveStream); - archive.directory(path, false); - archive.finalize(); } } diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts index f2f2383a8..76c38a6b8 100644 --- a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -42,12 +42,11 @@ export class ExportFavoritesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -91,7 +90,7 @@ export class ExportFavoritesProcessorService { }) as (NoteFavorite & { note: Note & { user: User } })[]; if (favorites.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -112,7 +111,7 @@ export class ExportFavoritesProcessorService { userId: user.id, }); - job.progress(exportedFavoritesCount / total); + job.updateProgress(exportedFavoritesCount / total); } await write(']'); @@ -127,8 +126,6 @@ export class ExportFavoritesProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts index fa9c1ac1e..8726cb140 100644 --- a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts @@ -10,10 +10,10 @@ import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import type { Following } from '@/models/entities/Following.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbExportFollowingData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbExportFollowingData } from '../types.js'; @Injectable() export class ExportFollowingProcessorService { @@ -40,12 +40,11 @@ export class ExportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -116,7 +115,5 @@ export class ExportFollowingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts index b14bf5f5b..0f11a9e84 100644 --- a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class ExportMutingProcessorService { @@ -39,12 +39,11 @@ export class ExportMutingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -73,7 +72,7 @@ export class ExportMutingProcessorService { }); if (mutes.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -103,7 +102,7 @@ export class ExportMutingProcessorService { muterId: user.id, }); - job.progress(exportedCount / total); + job.updateProgress(exportedCount / total); } stream.end(); @@ -116,7 +115,5 @@ export class ExportMutingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index e4f12ad10..24fb33188 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -39,12 +39,11 @@ export class ExportNotesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting notes of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -87,7 +86,7 @@ export class ExportNotesProcessorService { }) as Note[]; if (notes.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -108,7 +107,7 @@ export class ExportNotesProcessorService { userId: user.id, }); - job.progress(exportedNotesCount / total); + job.updateProgress(exportedNotesCount / total); } await write(']'); @@ -123,8 +122,6 @@ export class ExportNotesProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts index 54bde4404..ec6335805 100644 --- a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class ExportUserListsProcessorService { @@ -39,12 +39,11 @@ export class ExportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -92,7 +91,5 @@ export class ExportUserListsProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index d06131b8c..575cad69d 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import { DBAntennaImportJobData } from '../types.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; const validate = new Ajv().compile({ type: 'object', @@ -59,7 +59,7 @@ export class ImportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const now = new Date(); try { for (const antenna of job.data.antenna) { @@ -89,8 +89,6 @@ export class ImportAntennasProcessorService { } } catch (err: any) { this.logger.error(err); - } finally { - done(); } } } diff --git a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts index 3f075b02d..2f1a9e5b0 100644 --- a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; @Injectable() export class ImportBlockingProcessorService { @@ -34,12 +34,11 @@ export class ImportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -47,7 +46,6 @@ export class ImportBlockingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -56,7 +54,6 @@ export class ImportBlockingProcessorService { this.queueService.createImportBlockingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); - done(); } @bindThis @@ -85,7 +82,7 @@ export class ImportBlockingProcessorService { } if (target == null) { - throw `Unable to resolve user: @${username}@${host}`; + throw new Error(`Unable to resolve user: @${username}@${host}`); } // skip myself diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 600468a28..d86256787 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -12,7 +12,7 @@ import { DriveService } from '@/core/DriveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; // TODO: 名前衝突時の動作を選べるようにする @@ -45,14 +45,13 @@ export class ImportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Importing custom emojis ...'); const file = await this.driveFilesRepository.findOneBy({ id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -116,7 +115,6 @@ export class ImportCustomEmojisProcessorService { cleanup(); this.logger.succ('Imported'); - done(); }); unzipStream.pipe(extractor); this.logger.succ(`Unzipping to ${outputPath}`); diff --git a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts index aa5cf12c5..15bee9672 100644 --- a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; @Injectable() export class ImportFollowingProcessorService { @@ -34,12 +34,11 @@ export class ImportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -47,7 +46,6 @@ export class ImportFollowingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -56,7 +54,6 @@ export class ImportFollowingProcessorService { this.queueService.createImportFollowingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); - done(); } @bindThis @@ -85,7 +82,7 @@ export class ImportFollowingProcessorService { } if (target == null) { - throw `Unable to resolve user: @${username}@${host}`; + throw new Error(`Unable to resolve user: @${username}@${host}`); } // skip myself diff --git a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts index 379994ee7..723935cd3 100644 --- a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts @@ -9,10 +9,10 @@ import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData } from '../types.js'; @Injectable() export class ImportMutingProcessorService { @@ -38,12 +38,11 @@ export class ImportMutingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -51,7 +50,6 @@ export class ImportMutingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -83,7 +81,7 @@ export class ImportMutingProcessorService { } if (target == null) { - throw `cannot resolve user: @${username}@${host}`; + throw new Error(`cannot resolve user: @${username}@${host}`); } // skip myself @@ -98,6 +96,5 @@ export class ImportMutingProcessorService { } this.logger.succ('Imported'); - done(); } } diff --git a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts index c42386341..824ee8157 100644 --- a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts @@ -12,7 +12,7 @@ import { IdService } from '@/core/IdService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; @Injectable() @@ -46,12 +46,11 @@ export class ImportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -59,7 +58,6 @@ export class ImportUserListsProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -109,6 +107,5 @@ export class ImportUserListsProcessorService { } this.logger.succ('Imported'); - done(); } } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index ab8b1e9e2..ce1d7aaa1 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -1,8 +1,8 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import httpSignature from '@peertube/http-signature'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; -import type { InstancesRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { MetaService } from '@/core/MetaService.js'; @@ -23,10 +23,8 @@ import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { InboxJobData } from '../types.js'; -// ユーザーのinboxにアクティビティが届いた時の処理 @Injectable() export class InboxProcessorService { private logger: Logger; @@ -35,12 +33,6 @@ export class InboxProcessorService { @Inject(DI.config) private config: Config, - @Inject(DI.instancesRepository) - private instancesRepository: InstancesRepository, - - @Inject(DI.driveFilesRepository) - private driveFilesRepository: DriveFilesRepository, - private utilityService: UtilityService, private metaService: MetaService, private apInboxService: ApInboxService, @@ -93,24 +85,24 @@ export class InboxProcessorService { try { authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor)); } catch (err) { - // 対象が4xxならスキップ + // 対象が4xxならスキップ if (err instanceof StatusError) { if (err.isClientError) { - return `skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`; + throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); } - throw `Error in actor ${activity.actor} - ${err.statusCode ?? err}`; + throw new Error(`Error in actor ${activity.actor} - ${err.statusCode ?? err}`); } } } // それでもわからなければ終了 if (authUser == null) { - return 'skip: failed to resolve user'; + throw new Bull.UnrecoverableError('skip: failed to resolve user'); } // publicKey がなくても終了 if (authUser.key == null) { - return 'skip: failed to resolve user publicKey'; + throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey'); } // HTTP-Signatureの検証 @@ -118,10 +110,10 @@ export class InboxProcessorService { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { - // 一致しなくても、でもLD-Signatureがありそうならそっちも見る + // 一致しなくても、でもLD-Signatureがありそうならそっちも見る if (activity.signature) { if (activity.signature.type !== 'RsaSignature2017') { - return `skip: unsupported LD-signature type ${activity.signature.type}`; + throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`); } // activity.signature.creator: https://example.oom/users/user#main-key @@ -134,32 +126,32 @@ export class InboxProcessorService { // keyIdからLD-Signatureのユーザーを取得 authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator); if (authUser == null) { - return 'skip: LD-Signatureのユーザーが取得できませんでした'; + throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); } if (authUser.key == null) { - return 'skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'; + throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } // LD-Signature検証 const ldSignature = this.ldSignatureService.use(); const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { - return 'skip: LD-Signatureの検証に失敗しました'; + throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } // もう一度actorチェック if (authUser.user.uri !== activity.actor) { - return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`; + throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); } // ブロックしてたら中断 const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) { - return `Blocked request: ${ldHost}`; + throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); } } else { - return `skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`; + throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); } } @@ -168,7 +160,7 @@ export class InboxProcessorService { const signerHost = this.utilityService.extractDbHost(authUser.user.uri!); const activityIdHost = this.utilityService.extractDbHost(activity.id); if (signerHost !== activityIdHost) { - return `skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`; + throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`); } } diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index ff454df45..722260d94 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; diff --git a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts index e5840f3da..eab8e1e68 100644 --- a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts @@ -15,7 +15,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ResyncChartsProcessorService { @@ -43,7 +43,7 @@ export class ResyncChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Resync charts...'); // TODO: ユーザーごとのチャートも更新する @@ -55,6 +55,5 @@ export class ResyncChartsProcessorService { ]); this.logger.succ('All charts successfully resynced.'); - done(); } } diff --git a/packages/backend/src/queue/processors/TickChartsProcessorService.ts b/packages/backend/src/queue/processors/TickChartsProcessorService.ts index 7ff84c15a..f1696bf56 100644 --- a/packages/backend/src/queue/processors/TickChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/TickChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class TickChartsProcessorService { @@ -45,7 +45,7 @@ export class TickChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Tick charts...'); await Promise.all([ @@ -64,6 +64,5 @@ export class TickChartsProcessorService { ]); this.logger.succ('All charts successfully ticked.'); - done(); } } diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index 84a5c21c4..8b40c1674 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -1,4 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { WebhooksRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -7,7 +8,6 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { WebhookDeliverJobData } from '../types.js'; @Injectable() @@ -66,11 +66,11 @@ export class WebhookDeliverProcessorService { if (res instanceof StatusError) { // 4xx if (res.isClientError) { - return `${res.statusCode} ${res.statusMessage}`; + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } // 5xx etc. - throw `${res.statusCode} ${res.statusMessage}`; + throw new Error(`${res.statusCode} ${res.statusMessage}`); } else { // DNS error, socket error, timeout ... throw res; diff --git a/packages/backend/src/server/api/endpoints/admin/relays/add.ts b/packages/backend/src/server/api/endpoints/admin/relays/add.ts index f12738bd3..f2d4aa899 100644 --- a/packages/backend/src/server/api/endpoints/admin/relays/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/relays/add.ts @@ -62,7 +62,7 @@ export default class extends Endpoint { ) { super(meta, paramDef, async (ps, me) => { try { - if (new URL(ps.inbox).protocol !== 'https:') throw 'https only'; + if (new URL(ps.inbox).protocol !== 'https:') throw new Error('https only'); } catch { throw new ApiError(meta.errors.invalidUrl); } diff --git a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts index 2956bf1cb..742df0ca9 100644 --- a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts +++ b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts @@ -82,14 +82,14 @@ export default class extends Endpoint { try { if (ps.tag) { - if (!safeForSql(normalizeForSearch(ps.tag))) throw 'Injection'; + if (!safeForSql(normalizeForSearch(ps.tag))) throw new Error('Injection'); query.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`); } else { query.andWhere(new Brackets(qb => { for (const tags of ps.query!) { qb.orWhere(new Brackets(qb => { for (const tag of tags) { - if (!safeForSql(normalizeForSearch(tag))) throw 'Injection'; + if (!safeForSql(normalizeForSearch(tag))) throw new Error('Injection'); qb.andWhere(`'{"${normalizeForSearch(tag)}"}' <@ note.tags`); } })); diff --git a/packages/backend/src/server/api/endpoints/reset-db.ts b/packages/backend/src/server/api/endpoints/reset-db.ts index 4ced6d3ff..1d4825f81 100644 --- a/packages/backend/src/server/api/endpoints/reset-db.ts +++ b/packages/backend/src/server/api/endpoints/reset-db.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { private redisClient: Redis.Redis, ) { super(meta, paramDef, async (ps, me) => { - if (process.env.NODE_ENV !== 'test') throw 'NODE_ENV is not a test'; + if (process.env.NODE_ENV !== 'test') throw new Error('NODE_ENV is not a test'); await redisClient.flushdb(); await resetDb(this.db); diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index 6b31e6861..a7bcd859a 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -52,11 +52,7 @@ export class MockResolver extends Resolver { const r = this._rs.get(value); if (!r) { - throw { - name: 'StatusError', - statusCode: 404, - message: 'Not registed for mock', - }; + throw new Error('Not registed for mock'); } const object = JSON.parse(r.content); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 1a4a0b354..22f7d81e4 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -478,7 +478,7 @@ export async function testPaginationConsistency=12'} + /bullmq@3.14.1: + resolution: {integrity: sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==} dependencies: - cron-parser: 4.7.1 - debuglog: 1.0.1 - get-port: 5.1.1 + cron-parser: 4.8.1 + glob: 8.1.0 ioredis: 5.3.2 lodash: 4.17.21 - msgpackr: 1.8.1 + msgpackr: 1.9.2 semver: 7.5.1 - uuid: 8.3.2 + tslib: 2.5.2 + uuid: 9.0.0 transitivePeerDependencies: - supports-color + dev: false /busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} @@ -9306,6 +9302,7 @@ packages: /cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} + dev: false /co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} @@ -9607,11 +9604,12 @@ packages: readable-stream: 3.6.0 dev: false - /cron-parser@4.7.1: - resolution: {integrity: sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==} + /cron-parser@4.8.1: + resolution: {integrity: sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==} engines: {node: '>=12.0.0'} dependencies: - luxon: 3.2.1 + luxon: 3.3.0 + dev: false /cropperjs@2.0.0-beta.2: resolution: {integrity: sha512-jDRSODDGKmi9vp3p/+WXkxMqV/AE+GpSld1U3cHZDRdLy9UykRzurSe8k1dR0TExn45ygCMrv31qkg+K3EeXXw==} @@ -9890,9 +9888,6 @@ packages: ms: 2.1.2 supports-color: 8.1.1 - /debuglog@1.0.1: - resolution: {integrity: sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==} - /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -10084,6 +10079,7 @@ packages: /denque@2.1.0: resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} engines: {node: '>=0.10'} + dev: false /depd@1.1.2: resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} @@ -11753,6 +11749,7 @@ packages: /get-port@5.1.1: resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} engines: {node: '>=8'} + dev: true /get-stream@3.0.0: resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==} @@ -12606,6 +12603,7 @@ packages: standard-as-callback: 2.1.0 transitivePeerDependencies: - supports-color + dev: false /iota-array@1.0.0: resolution: {integrity: sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==} @@ -14288,6 +14286,7 @@ packages: /lodash.defaults@4.2.0: resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + dev: false /lodash.difference@4.5.0: resolution: {integrity: sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==} @@ -14319,6 +14318,7 @@ packages: /lodash.isarguments@3.1.0: resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + dev: false /lodash.isempty@4.4.0: resolution: {integrity: sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==} @@ -14472,9 +14472,10 @@ packages: engines: {node: '>=16.14'} dev: true - /luxon@3.2.1: - resolution: {integrity: sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==} + /luxon@3.3.0: + resolution: {integrity: sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==} engines: {node: '>=12'} + dev: false /lz-string@1.5.0: resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} @@ -14950,24 +14951,27 @@ packages: engines: {node: '>=12.13'} dev: false - /msgpackr-extract@2.2.0: - resolution: {integrity: sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==} + /msgpackr-extract@3.0.2: + resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==} + hasBin: true requiresBuild: true dependencies: - node-gyp-build-optional-packages: 5.0.3 + node-gyp-build-optional-packages: 5.0.7 optionalDependencies: - '@msgpackr-extract/msgpackr-extract-darwin-arm64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-darwin-x64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-arm': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-arm64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-x64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-win32-x64': 2.2.0 + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2 + dev: false optional: true - /msgpackr@1.8.1: - resolution: {integrity: sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==} + /msgpackr@1.9.2: + resolution: {integrity: sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==} optionalDependencies: - msgpackr-extract: 2.2.0 + msgpackr-extract: 3.0.2 + dev: false /msw-storybook-addon@1.8.0(msw@1.2.1): resolution: {integrity: sha512-dw3vZwqjixmiur0vouRSOax7wPSu9Og2Hspy9JZFHf49bZRjwDiLF0Pfn2NXEkGviYJOJiGxS1ejoTiUwoSg4A==} @@ -15219,8 +15223,10 @@ packages: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 - /node-gyp-build-optional-packages@5.0.3: - resolution: {integrity: sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==} + /node-gyp-build-optional-packages@5.0.7: + resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==} + hasBin: true + dev: false optional: true /node-gyp-build@4.6.0: @@ -17252,6 +17258,7 @@ packages: /redis-errors@1.2.0: resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} engines: {node: '>=4'} + dev: false /redis-info@3.1.0: resolution: {integrity: sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==} @@ -17269,6 +17276,7 @@ packages: engines: {node: '>=4'} dependencies: redis-errors: 1.2.0 + dev: false /redis@4.5.1: resolution: {integrity: sha512-oxXSoIqMJCQVBTfxP6BNTCtDMyh9G6Vi5wjdPdV/sRKkufyZslDqCScSGcOr6XGR/reAWZefz7E4leM31RgdBA==} @@ -18395,6 +18403,7 @@ packages: /standard-as-callback@2.1.0: resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false /start-server-and-test@2.0.0: resolution: {integrity: sha512-UqKLw0mJbfrsG1jcRLTUlvuRi9sjNuUiDOLI42r7R5fA9dsFoywAy9DoLXNYys9B886E4RCKb+qM1Gzu96h7DQ==} From b6f21b6edb95882d1616c36ed7eeb1f78809e2f6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 13:21:26 +0900 Subject: [PATCH 04/20] refactor --- packages/backend/src/GlobalModule.ts | 6 +++++- packages/backend/src/core/AntennaService.ts | 15 ++++++++++----- packages/backend/src/core/CacheService.ts | 7 ++++++- packages/backend/src/core/MetaService.ts | 7 ++++++- packages/backend/src/core/NoteCreateService.ts | 8 +++++++- packages/backend/src/core/NoteReadService.ts | 8 +++++++- packages/backend/src/core/NotificationService.ts | 8 +++++++- packages/backend/src/core/QueueModule.ts | 6 +++++- packages/backend/src/core/RoleService.ts | 7 ++++++- packages/backend/src/core/WebhookService.ts | 7 ++++++- .../src/core/chart/ChartManagementService.ts | 8 +++++++- packages/backend/src/daemons/JanitorService.ts | 7 ++++++- packages/backend/src/daemons/QueueStatsService.ts | 9 +++++++-- .../backend/src/daemons/ServerStatsService.ts | 7 ++++++- packages/backend/src/server/ServerService.ts | 8 +++++++- packages/backend/src/server/api/ApiCallService.ts | 7 ++++++- 16 files changed, 104 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 564787392..406e3192b 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -100,7 +100,7 @@ export class GlobalModule implements OnApplicationShutdown { @Inject(DI.redisForSub) private redisForSub: Redis.Redis, ) {} - async onApplicationShutdown(signal: string): Promise { + public async dispose(): Promise { if (process.env.NODE_ENV === 'test') { // XXX: // Shutting down the existing connections causes errors on Jest as @@ -116,4 +116,8 @@ export class GlobalModule implements OnApplicationShutdown { this.redisForSub.disconnect(), ]); } + + async onApplicationShutdown(signal: string): Promise { + await this.dispose(); + } } diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 2d4226a32..d8df37191 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -55,11 +55,6 @@ export class AntennaService implements OnApplicationShutdown { this.redisForSub.on('message', this.onRedisMessage); } - @bindThis - public onApplicationShutdown(signal?: string | undefined) { - this.redisForSub.off('message', this.onRedisMessage); - } - @bindThis private async onRedisMessage(_: string, data: string): Promise { const obj = JSON.parse(data); @@ -196,4 +191,14 @@ export class AntennaService implements OnApplicationShutdown { return this.antennas; } + + @bindThis + public dispose(): void { + this.redisForSub.off('message', this.onRedisMessage); + } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index cf1e81ffc..de33e4c24 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -166,7 +166,12 @@ export class CacheService implements OnApplicationShutdown { } @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { this.redisForSub.off('message', this.onMessage); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 0b861be8d..5acc9ad9a 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -120,8 +120,13 @@ export class MetaService implements OnApplicationShutdown { } @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { clearInterval(this.intervalId); this.redisForSub.off('message', this.onMessage); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 2fd7a8ac8..1c8491bf5 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -790,7 +790,13 @@ export class NoteCreateService implements OnApplicationShutdown { return mentionedUsers; } - onApplicationShutdown(signal?: string | undefined) { + @bindThis + public dispose(): void { this.#shutdownController.abort(); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/NoteReadService.ts b/packages/backend/src/core/NoteReadService.ts index 1129bd159..e57e57d31 100644 --- a/packages/backend/src/core/NoteReadService.ts +++ b/packages/backend/src/core/NoteReadService.ts @@ -122,7 +122,13 @@ export class NoteReadService implements OnApplicationShutdown { } } - onApplicationShutdown(signal?: string | undefined): void { + @bindThis + public dispose(): void { this.#shutdownController.abort(); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/NotificationService.ts b/packages/backend/src/core/NotificationService.ts index a245908c9..ed47165f7 100644 --- a/packages/backend/src/core/NotificationService.ts +++ b/packages/backend/src/core/NotificationService.ts @@ -152,7 +152,13 @@ export class NotificationService implements OnApplicationShutdown { */ } - onApplicationShutdown(signal?: string | undefined): void { + @bindThis + public dispose(): void { this.#shutdownController.abort(); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index 6db9bb14c..3384ca457 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -100,7 +100,7 @@ export class QueueModule implements OnApplicationShutdown { @Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue, ) {} - async onApplicationShutdown(signal: string): Promise { + public async dispose(): Promise { if (process.env.NODE_ENV === 'test') { // XXX: // Shutting down the existing connections causes errors on Jest as @@ -120,4 +120,8 @@ export class QueueModule implements OnApplicationShutdown { this.webhookDeliverQueue.close(), ]); } + + async onApplicationShutdown(signal: string): Promise { + await this.dispose(); + } } diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 130ec5ec8..40ae10666 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -433,7 +433,12 @@ export class RoleService implements OnApplicationShutdown { } @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { this.redisForSub.off('message', this.onMessage); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/WebhookService.ts b/packages/backend/src/core/WebhookService.ts index 57baade77..467755a07 100644 --- a/packages/backend/src/core/WebhookService.ts +++ b/packages/backend/src/core/WebhookService.ts @@ -81,7 +81,12 @@ export class WebhookService implements OnApplicationShutdown { } @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { this.redisForSub.off('message', this.onMessage); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/chart/ChartManagementService.ts b/packages/backend/src/core/chart/ChartManagementService.ts index 03e361265..b0e9e534d 100644 --- a/packages/backend/src/core/chart/ChartManagementService.ts +++ b/packages/backend/src/core/chart/ChartManagementService.ts @@ -60,7 +60,8 @@ export class ChartManagementService implements OnApplicationShutdown { }, 1000 * 60 * 20); } - async onApplicationShutdown(signal: string): Promise { + @bindThis + public async dispose(): Promise { clearInterval(this.saveIntervalId); if (process.env.NODE_ENV !== 'test') { await Promise.all( @@ -68,4 +69,9 @@ export class ChartManagementService implements OnApplicationShutdown { ); } } + + @bindThis + async onApplicationShutdown(signal: string): Promise { + await this.dispose(); + } } diff --git a/packages/backend/src/daemons/JanitorService.ts b/packages/backend/src/daemons/JanitorService.ts index 8cdfb703f..f826d5062 100644 --- a/packages/backend/src/daemons/JanitorService.ts +++ b/packages/backend/src/daemons/JanitorService.ts @@ -34,7 +34,12 @@ export class JanitorService implements OnApplicationShutdown { } @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { clearInterval(this.intervalId); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index 0a5b3184d..53a0d14cd 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -81,9 +81,14 @@ export class QueueStatsService implements OnApplicationShutdown { this.intervalId = setInterval(tick, interval); } - + @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { clearInterval(this.intervalId); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } diff --git a/packages/backend/src/daemons/ServerStatsService.ts b/packages/backend/src/daemons/ServerStatsService.ts index bb190cf60..6cd71c0e2 100644 --- a/packages/backend/src/daemons/ServerStatsService.ts +++ b/packages/backend/src/daemons/ServerStatsService.ts @@ -63,9 +63,14 @@ export class ServerStatsService implements OnApplicationShutdown { } @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { clearInterval(this.intervalId); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } // CPU STAT diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index 9257fee13..ce6a1f704 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -222,7 +222,13 @@ export class ServerService implements OnApplicationShutdown { await fastify.ready(); } - async onApplicationShutdown(signal: string): Promise { + @bindThis + public async dispose(): Promise { await this.#fastify.close(); } + + @bindThis + async onApplicationShutdown(signal: string): Promise { + await this.dispose(); + } } diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index e3483c82c..dad1a4132 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -359,7 +359,12 @@ export class ApiCallService implements OnApplicationShutdown { } @bindThis - public onApplicationShutdown(signal?: string | undefined) { + public dispose(): void { clearInterval(this.userIpHistoriesClearIntervalId); } + + @bindThis + public onApplicationShutdown(signal?: string | undefined): void { + this.dispose(); + } } From 4790ddfad68ab4889db11c3f9c7f0159a69d4829 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 13:30:26 +0900 Subject: [PATCH 05/20] refactor of QueueProcessorService --- .../src/queue/QueueProcessorService.ts | 84 ++++++++++++++----- 1 file changed, 64 insertions(+), 20 deletions(-) diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 011082cd3..da5069c29 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Bull from 'bullmq'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; @@ -63,8 +63,16 @@ function getJobInfo(job: Bull.Job | undefined, increment = false): string { } @Injectable() -export class QueueProcessorService { +export class QueueProcessorService implements OnApplicationShutdown { private logger: Logger; + private systemQueueWorker: Bull.Worker; + private dbQueueWorker: Bull.Worker; + private deliverQueueWorker: Bull.Worker; + private inboxQueueWorker: Bull.Worker; + private webhookDeliverQueueWorker: Bull.Worker; + private relationshipQueueWorker: Bull.Worker; + private objectStorageQueueWorker: Bull.Worker; + private endedPollNotificationQueueWorker: Bull.Worker; constructor( @Inject(DI.config) @@ -102,10 +110,7 @@ export class QueueProcessorService { private cleanProcessorService: CleanProcessorService, ) { this.logger = this.queueLoggerService.logger; - } - @bindThis - public start() { function renderError(e: Error): any { if (e) { // 何故かeがundefinedで来ることがある return { @@ -123,7 +128,7 @@ export class QueueProcessorService { } //#region system - const systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { + this.systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { switch (job.name) { case 'tickCharts': return this.tickChartsProcessorService.process(); case 'resyncCharts': return this.resyncChartsProcessorService.process(); @@ -135,11 +140,12 @@ export class QueueProcessorService { } }, { ...baseQueueOptions(this.config, QUEUE.SYSTEM), + autorun: false, }); const systemLogger = this.logger.createSubLogger('system'); - systemQueueWorker + this.systemQueueWorker .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) @@ -148,7 +154,7 @@ export class QueueProcessorService { //#endregion //#region db - const dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { + this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { switch (job.name) { case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); @@ -172,11 +178,12 @@ export class QueueProcessorService { } }, { ...baseQueueOptions(this.config, QUEUE.DB), + autorun: false, }); const dbLogger = this.logger.createSubLogger('db'); - dbQueueWorker + this.dbQueueWorker .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) @@ -185,8 +192,9 @@ export class QueueProcessorService { //#endregion //#region deliver - const deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), { + this.deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), { ...baseQueueOptions(this.config, QUEUE.DELIVER), + autorun: false, concurrency: this.config.deliverJobConcurrency ?? 128, limiter: { max: this.config.deliverJobPerSec ?? 128, @@ -199,7 +207,7 @@ export class QueueProcessorService { const deliverLogger = this.logger.createSubLogger('deliver'); - deliverQueueWorker + this.deliverQueueWorker .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) @@ -208,8 +216,9 @@ export class QueueProcessorService { //#endregion //#region inbox - const inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), { + this.inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), { ...baseQueueOptions(this.config, QUEUE.INBOX), + autorun: false, concurrency: this.config.inboxJobConcurrency ?? 16, limiter: { max: this.config.inboxJobPerSec ?? 16, @@ -222,7 +231,7 @@ export class QueueProcessorService { const inboxLogger = this.logger.createSubLogger('inbox'); - inboxQueueWorker + this.inboxQueueWorker .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) @@ -231,8 +240,9 @@ export class QueueProcessorService { //#endregion //#region webhook deliver - const webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), { + this.webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), { ...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER), + autorun: false, concurrency: 64, limiter: { max: 64, @@ -245,7 +255,7 @@ export class QueueProcessorService { const webhookLogger = this.logger.createSubLogger('webhook'); - webhookDeliverQueueWorker + this.webhookDeliverQueueWorker .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) @@ -254,7 +264,7 @@ export class QueueProcessorService { //#endregion //#region relationship - const relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { + this.relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { switch (job.name) { case 'follow': return this.relationshipProcessorService.processFollow(job); case 'unfollow': return this.relationshipProcessorService.processUnfollow(job); @@ -264,6 +274,7 @@ export class QueueProcessorService { } }, { ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP), + autorun: false, concurrency: this.config.relashionshipJobConcurrency ?? 16, limiter: { max: this.config.relashionshipJobPerSec ?? 64, @@ -273,7 +284,7 @@ export class QueueProcessorService { const relationshipLogger = this.logger.createSubLogger('relationship'); - relationshipQueueWorker + this.relationshipQueueWorker .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) @@ -282,7 +293,7 @@ export class QueueProcessorService { //#endregion //#region object storage - const objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { + this.objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { switch (job.name) { case 'deleteFile': return this.deleteFileProcessorService.process(job); case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job); @@ -290,12 +301,13 @@ export class QueueProcessorService { } }, { ...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE), + autorun: false, concurrency: 16, }); const objectStorageLogger = this.logger.createSubLogger('objectStorage'); - objectStorageQueueWorker + this.objectStorageQueueWorker .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) @@ -304,9 +316,41 @@ export class QueueProcessorService { //#endregion //#region ended poll notification - const endedPollNotificationWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), { + this.endedPollNotificationQueueWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), { ...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), + autorun: false, }); //#endregion } + + @bindThis + public start() { + this.systemQueueWorker.run(); + this.dbQueueWorker.run(); + this.deliverQueueWorker.run(); + this.inboxQueueWorker.run(); + this.webhookDeliverQueueWorker.run(); + this.relationshipQueueWorker.run(); + this.objectStorageQueueWorker.run(); + this.endedPollNotificationQueueWorker.run(); + } + + @bindThis + public async stop(): Promise { + await Promise.all([ + this.systemQueueWorker.close(), + this.dbQueueWorker.close(), + this.deliverQueueWorker.close(), + this.inboxQueueWorker.close(), + this.webhookDeliverQueueWorker.close(), + this.relationshipQueueWorker.close(), + this.objectStorageQueueWorker.close(), + this.endedPollNotificationQueueWorker.close(), + ]); + } + + @bindThis + public async onApplicationShutdown(signal?: string | undefined): Promise { + await this.stop(); + } } From b35b9bc27ff762938495d2ce73521fbe68643793 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 13:30:57 +0900 Subject: [PATCH 06/20] Update QueueProcessorService.ts --- .../src/queue/QueueProcessorService.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index da5069c29..42f9c1af7 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -324,15 +324,17 @@ export class QueueProcessorService implements OnApplicationShutdown { } @bindThis - public start() { - this.systemQueueWorker.run(); - this.dbQueueWorker.run(); - this.deliverQueueWorker.run(); - this.inboxQueueWorker.run(); - this.webhookDeliverQueueWorker.run(); - this.relationshipQueueWorker.run(); - this.objectStorageQueueWorker.run(); - this.endedPollNotificationQueueWorker.run(); + public async start(): Promise { + await Promise.all([ + this.systemQueueWorker.run(), + this.dbQueueWorker.run(), + this.deliverQueueWorker.run(), + this.inboxQueueWorker.run(), + this.webhookDeliverQueueWorker.run(), + this.relationshipQueueWorker.run(), + this.objectStorageQueueWorker.run(), + this.endedPollNotificationQueueWorker.run(), + ]); } @bindThis From f930eaee020520c1f9496803dcf445b2f2955526 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 13:32:19 +0900 Subject: [PATCH 07/20] perf(backend): use websockets/ws instead of theturtle32/WebSocket-Node (#10884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(backend): use websockets/ws instead of theturtle32/WebSocket-Node Resolve #10883 * refactor * Update StreamingApiServerService.ts * Update StreamingApiServerService.ts * :v: * Update StreamingApiServerService.ts * fix main stream init * fix timing 2 * setIntervalの重複を避ける（気休め） * add comment * :v: --------- Co-authored-by: tamaina --- packages/backend/package.json | 5 +- packages/backend/src/server/ServerService.ts | 3 +- .../src/server/api/AuthenticateService.ts | 2 +- .../server/api/StreamingApiServerService.ts | 121 ++++++++++-------- .../backend/src/server/api/stream/index.ts | 21 ++- pnpm-lock.yaml | 63 +++------ 6 files changed, 107 insertions(+), 108 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 99c04d6bf..66bc7ee47 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -35,6 +35,7 @@ "@swc/core-win32-x64-msvc": "1.3.56", "@tensorflow/tfjs": "4.4.0", "@tensorflow/tfjs-node": "4.4.0", + "bufferutil": "^4.0.7", "slacc-android-arm-eabi": "0.0.9", "slacc-android-arm64": "0.0.9", "slacc-darwin-arm64": "0.0.9", @@ -46,7 +47,8 @@ "slacc-linux-arm64-musl": "0.0.9", "slacc-linux-x64-gnu": "0.0.9", "slacc-win32-arm64-msvc": "0.0.9", - "slacc-win32-x64-msvc": "0.0.9" + "slacc-win32-x64-msvc": "0.0.9", + "utf-8-validate": "^6.0.3" }, "dependencies": { "@aws-sdk/client-s3": "3.321.1", @@ -157,7 +159,6 @@ "uuid": "9.0.0", "vary": "1.1.2", "web-push": "3.6.1", - "websocket": "1.0.34", "ws": "8.13.0", "xev": "3.0.2" }, diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index ce6a1f704..c3d45e4ad 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -194,7 +194,7 @@ export class ServerService implements OnApplicationShutdown { fastify.register(this.clientServerService.createServer); - this.streamingApiServerService.attachStreamingApi(fastify.server); + this.streamingApiServerService.attach(fastify.server); fastify.server.on('error', err => { switch ((err as any).code) { @@ -224,6 +224,7 @@ export class ServerService implements OnApplicationShutdown { @bindThis public async dispose(): Promise { + await this.streamingApiServerService.detach(); await this.#fastify.close(); } diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index 6548c475b..e23591d87 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -36,7 +36,7 @@ export class AuthenticateService { } @bindThis - public async authenticate(token: string | null | undefined): Promise<[LocalUser | null | undefined, AccessToken | null | undefined]> { + public async authenticate(token: string | null | undefined): Promise<[LocalUser | null, AccessToken | null]> { if (token == null) { return [null, null]; } diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index 258e8de03..fdda581ad 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -1,23 +1,25 @@ import { EventEmitter } from 'events'; import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import * as websocket from 'websocket'; +import * as WebSocket from 'ws'; import { DI } from '@/di-symbols.js'; -import type { UsersRepository, BlockingsRepository, ChannelFollowingsRepository, FollowingsRepository, MutingsRepository, UserProfilesRepository, RenoteMutingsRepository } from '@/models/index.js'; +import type { UsersRepository, AccessToken } from '@/models/index.js'; import type { Config } from '@/config.js'; import { NoteReadService } from '@/core/NoteReadService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { NotificationService } from '@/core/NotificationService.js'; import { bindThis } from '@/decorators.js'; import { CacheService } from '@/core/CacheService.js'; -import { AuthenticateService } from './AuthenticateService.js'; +import { LocalUser } from '@/models/entities/User'; +import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; import MainStreamConnection from './stream/index.js'; import { ChannelsService } from './stream/ChannelsService.js'; -import type { ParsedUrlQuery } from 'querystring'; import type * as http from 'node:http'; @Injectable() export class StreamingApiServerService { + #wss: WebSocket.WebSocketServer; + constructor( @Inject(DI.config) private config: Config, @@ -28,24 +30,6 @@ export class StreamingApiServerService { @Inject(DI.usersRepository) private usersRepository: UsersRepository, - @Inject(DI.followingsRepository) - private followingsRepository: FollowingsRepository, - - @Inject(DI.mutingsRepository) - private mutingsRepository: MutingsRepository, - - @Inject(DI.renoteMutingsRepository) - private renoteMutingsRepository: RenoteMutingsRepository, - - @Inject(DI.blockingsRepository) - private blockingsRepository: BlockingsRepository, - - @Inject(DI.channelFollowingsRepository) - private channelFollowingsRepository: ChannelFollowingsRepository, - - @Inject(DI.userProfilesRepository) - private userProfilesRepository: UserProfilesRepository, - private cacheService: CacheService, private noteReadService: NoteReadService, private authenticateService: AuthenticateService, @@ -55,25 +39,65 @@ export class StreamingApiServerService { } @bindThis - public attachStreamingApi(server: http.Server) { - // Init websocket server - const ws = new websocket.server({ - httpServer: server, + public attach(server: http.Server): void { + this.#wss = new WebSocket.WebSocketServer({ + noServer: true, }); - ws.on('request', async (request) => { - const q = request.resourceURL.query as ParsedUrlQuery; - - // TODO: トークンが間違ってるなどしてauthenticateに失敗したら - // コネクション切断するなりエラーメッセージ返すなりする - // (現状はエラーがキャッチされておらずサーバーのログに流れて邪魔なので) - const [user, miapp] = await this.authenticateService.authenticate(q.i as string); - - if (user?.isSuspended) { - request.reject(400); + server.on('upgrade', async (request, socket, head) => { + if (request.url == null) { + socket.write('HTTP/1.1 400 Bad Request\r\n\r\n'); + socket.destroy(); return; } + const q = new URL(request.url, `http://${request.headers.host}`).searchParams; + + let user: LocalUser | null = null; + let app: AccessToken | null = null; + + try { + [user, app] = await this.authenticateService.authenticate(q.get('i')); + } catch (e) { + if (e instanceof AuthenticationError) { + socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n'); + } else { + socket.write('HTTP/1.1 500 Internal Server Error\r\n\r\n'); + } + socket.destroy(); + return; + } + + if (user?.isSuspended) { + socket.write('HTTP/1.1 403 Forbidden\r\n\r\n'); + socket.destroy(); + return; + } + + const stream = new MainStreamConnection( + this.channelsService, + this.noteReadService, + this.notificationService, + this.cacheService, + user, app, + ); + + await stream.init(); + + this.#wss.handleUpgrade(request, socket, head, (ws) => { + this.#wss.emit('connection', ws, request, { + stream, user, app, + }); + }); + }); + + this.#wss.on('connection', async (connection: WebSocket.WebSocket, request: http.IncomingMessage, ctx: { + stream: MainStreamConnection, + user: LocalUser | null; + app: AccessToken | null + }) => { + const { stream, user, app } = ctx; + const ev = new EventEmitter(); async function onRedisMessage(_: string, data: string): Promise { @@ -83,19 +107,7 @@ export class StreamingApiServerService { this.redisForSub.on('message', onRedisMessage); - const main = new MainStreamConnection( - this.channelsService, - this.noteReadService, - this.notificationService, - this.cacheService, - ev, user, miapp, - ); - - await main.init(); - - const connection = request.accept(); - - main.init2(connection); + await stream.listen(ev, connection); const intervalId = user ? setInterval(() => { this.usersRepository.update(user.id, { @@ -110,16 +122,23 @@ export class StreamingApiServerService { connection.once('close', () => { ev.removeAllListeners(); - main.dispose(); + stream.dispose(); this.redisForSub.off('message', onRedisMessage); if (intervalId) clearInterval(intervalId); }); connection.on('message', async (data) => { - if (data.type === 'utf8' && data.utf8Data === 'ping') { + if (data.toString() === 'ping') { connection.send('pong'); } }); }); } + + @bindThis + public detach(): Promise { + return new Promise((resolve) => { + this.#wss.close(() => resolve()); + }); + } } diff --git a/packages/backend/src/server/api/stream/index.ts b/packages/backend/src/server/api/stream/index.ts index fee56e366..8b1c2c09c 100644 --- a/packages/backend/src/server/api/stream/index.ts +++ b/packages/backend/src/server/api/stream/index.ts @@ -1,3 +1,4 @@ +import * as WebSocket from 'ws'; import type { User } from '@/models/entities/User.js'; import type { AccessToken } from '@/models/entities/AccessToken.js'; import type { Packed } from '@/misc/json-schema.js'; @@ -7,7 +8,6 @@ import { bindThis } from '@/decorators.js'; import { CacheService } from '@/core/CacheService.js'; import { UserProfile } from '@/models/index.js'; import type { ChannelsService } from './ChannelsService.js'; -import type * as websocket from 'websocket'; import type { EventEmitter } from 'events'; import type Channel from './channel.js'; import type { StreamEventEmitter, StreamMessages } from './types.js'; @@ -18,7 +18,7 @@ import type { StreamEventEmitter, StreamMessages } from './types.js'; export default class Connection { public user?: User; public token?: AccessToken; - private wsConnection: websocket.connection; + private wsConnection: WebSocket.WebSocket; public subscriber: StreamEventEmitter; private channels: Channel[] = []; private subscribingNotes: any = {}; @@ -37,11 +37,9 @@ export default class Connection { private notificationService: NotificationService, private cacheService: CacheService, - subscriber: EventEmitter, user: User | null | undefined, token: AccessToken | null | undefined, ) { - this.subscriber = subscriber; if (user) this.user = user; if (token) this.token = token; } @@ -70,12 +68,16 @@ export default class Connection { if (this.user != null) { await this.fetch(); - this.fetchIntervalId = setInterval(this.fetch, 1000 * 10); + if (!this.fetchIntervalId) { + this.fetchIntervalId = setInterval(this.fetch, 1000 * 10); + } } } @bindThis - public async init2(wsConnection: websocket.connection) { + public async listen(subscriber: EventEmitter, wsConnection: WebSocket.WebSocket) { + this.subscriber = subscriber; + this.wsConnection = wsConnection; this.wsConnection.on('message', this.onWsConnectionMessage); @@ -88,14 +90,11 @@ export default class Connection { * クライアントからメッセージ受信時 */ @bindThis - private async onWsConnectionMessage(data: websocket.Message) { - if (data.type !== 'utf8') return; - if (data.utf8Data == null) return; - + private async onWsConnectionMessage(data: WebSocket.RawData) { let obj: Record; try { - obj = JSON.parse(data.utf8Data); + obj = JSON.parse(data.toString()); } catch (e) { return; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7930458cb..57e8cf850 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,7 +96,7 @@ importers: version: 8.2.1 '@fastify/http-proxy': specifier: 9.1.0 - version: 9.1.0 + version: 9.1.0(bufferutil@4.0.7)(utf-8-validate@6.0.3) '@fastify/multipart': specifier: 7.6.0 version: 7.6.0 @@ -219,7 +219,7 @@ importers: version: 4.1.0 jsdom: specifier: 21.1.1 - version: 21.1.1 + version: 21.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3) json5: specifier: 2.2.3 version: 2.2.3 @@ -388,12 +388,9 @@ importers: web-push: specifier: 3.6.1 version: 3.6.1 - websocket: - specifier: 1.0.34 - version: 1.0.34 ws: specifier: 8.13.0 - version: 8.13.0 + version: 8.13.0(bufferutil@4.0.7)(utf-8-validate@6.0.3) xev: specifier: 3.0.2 version: 3.0.2 @@ -437,6 +434,9 @@ importers: '@tensorflow/tfjs-node': specifier: 4.4.0 version: 4.4.0(seedrandom@3.0.5) + bufferutil: + specifier: ^4.0.7 + version: 4.0.7 slacc-android-arm-eabi: specifier: 0.0.9 version: 0.0.9 @@ -473,6 +473,9 @@ importers: slacc-win32-x64-msvc: specifier: 0.0.9 version: 0.0.9 + utf-8-validate: + specifier: ^6.0.3 + version: 6.0.3 devDependencies: '@jest/globals': specifier: 29.5.0 @@ -3852,12 +3855,12 @@ packages: fast-json-stringify: 5.7.0 dev: false - /@fastify/http-proxy@9.1.0: + /@fastify/http-proxy@9.1.0(bufferutil@4.0.7)(utf-8-validate@6.0.3): resolution: {integrity: sha512-vgHCTDKOqLB437zQJiLWFFnsrYfFZ6Lfwu/xXQoKqRUKIPDt+xG6LBRtf8s5MNqfFVoTE7kw1U/0qdRGDsMp4Q==} dependencies: '@fastify/reply-from': 9.0.1 fastify-plugin: 4.5.0 - ws: 8.13.0 + ws: 8.13.0(bufferutil@4.0.7)(utf-8-validate@6.0.3) transitivePeerDependencies: - bufferutil - utf-8-validate @@ -5532,7 +5535,7 @@ packages: ts-dedent: 2.2.0 util-deprecate: 1.0.2 watchpack: 2.4.0 - ws: 8.13.0 + ws: 8.13.0(bufferutil@4.0.7)(utf-8-validate@6.0.3) transitivePeerDependencies: - bufferutil - encoding @@ -8702,7 +8705,6 @@ packages: requiresBuild: true dependencies: node-gyp-build: 4.6.0 - dev: false /bullmq@3.14.1: resolution: {integrity: sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==} @@ -13872,7 +13874,7 @@ packages: - supports-color dev: true - /jsdom@21.1.1: + /jsdom@21.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3): resolution: {integrity: sha512-Jjgdmw48RKcdAIQyUD1UdBh2ecH7VqwaXPN3ehoZN6MqgVbMn+lRm1aAT1AsdJRAJpwfa4IpwgzySn61h2qu3w==} engines: {node: '>=14'} peerDependencies: @@ -13905,7 +13907,7 @@ packages: whatwg-encoding: 2.0.0 whatwg-mimetype: 3.0.0 whatwg-url: 12.0.1 - ws: 8.13.0 + ws: 8.13.0(bufferutil@4.0.7)(utf-8-validate@6.0.3) xml-name-validator: 4.0.0 transitivePeerDependencies: - bufferutil @@ -15231,7 +15233,7 @@ packages: /node-gyp-build@4.6.0: resolution: {integrity: sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==} - dev: false + hasBin: true /node-gyp@9.3.1: resolution: {integrity: sha512-4Q16ZCqq3g8awk6UplT7AuxQ35XN4R/yf/+wSAwcBUAjg7l58RTactWaP8fIDTi0FzI7YcVLujwExakZlfWkXg==} @@ -19276,12 +19278,6 @@ packages: resolution: {integrity: sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==} dev: false - /typedarray-to-buffer@3.1.5: - resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} - dependencies: - is-typedarray: 1.0.0 - dev: false - /typedarray@0.0.6: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} @@ -19656,13 +19652,12 @@ packages: resolution: {integrity: sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==} engines: {node: '>=0.10.0'} - /utf-8-validate@5.0.10: - resolution: {integrity: sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==} + /utf-8-validate@6.0.3: + resolution: {integrity: sha512-uIuGf9TWQ/y+0Lp+KGZCMuJWc3N9BHA+l/UmHd/oUHwJJDeysyTRxNQVkbzsIWfGFbRe3OcgML/i0mvVRPOyDA==} engines: {node: '>=6.14.2'} requiresBuild: true dependencies: node-gyp-build: 4.6.0 - dev: false /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -20132,20 +20127,6 @@ packages: resolution: {integrity: sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw==} dev: false - /websocket@1.0.34: - resolution: {integrity: sha512-PRDso2sGwF6kM75QykIesBijKSVceR6jL2G8NGYyq2XrItNC2P5/qL5XeR056GhA+Ly7JMFvJb9I312mJfmqnQ==} - engines: {node: '>=4.0.0'} - dependencies: - bufferutil: 4.0.7 - debug: 2.6.9 - es5-ext: 0.10.62 - typedarray-to-buffer: 3.1.5 - utf-8-validate: 5.0.10 - yaeti: 0.0.6 - transitivePeerDependencies: - - supports-color - dev: false - /well-known-symbols@2.0.0: resolution: {integrity: sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==} engines: {node: '>=6'} @@ -20336,7 +20317,7 @@ packages: async-limiter: 1.0.1 dev: true - /ws@8.13.0: + /ws@8.13.0(bufferutil@4.0.7)(utf-8-validate@6.0.3): resolution: {integrity: sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==} engines: {node: '>=10.0.0'} peerDependencies: @@ -20347,6 +20328,9 @@ packages: optional: true utf-8-validate: optional: true + dependencies: + bufferutil: 4.0.7 + utf-8-validate: 6.0.3 /xev@3.0.2: resolution: {integrity: sha512-8kxuH95iMXzHZj+fwqfA4UrPcYOy6bGIgfWzo9Ji23JoEc30ge/Z++Ubkiuy8c0+M64nXmmxrmJ7C8wnuBhluw==} @@ -20394,11 +20378,6 @@ packages: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} - /yaeti@0.0.6: - resolution: {integrity: sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug==} - engines: {node: '>=0.10.32'} - dev: false - /yallist@2.1.2: resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==} From afa4cd9112b429afa53396c2435baf5ef3b3dea9 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 13:34:55 +0900 Subject: [PATCH 08/20] 13.13.0-beta.4 --- CHANGELOG.md | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b107c72f..36bc63b0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ ### Server - bullをbull-mqにアップグレードし、ジョブキューのパフォーマンスを改善 +- ストリーミングのパフォーマンスを改善 - Fix: お知らせの画像URLを空にできない問題を修正 ## 13.12.2 diff --git a/package.json b/package.json index 772362319..58257a8b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "misskey", - "version": "13.13.0-beta.3", + "version": "13.13.0-beta.4", "codename": "nasubi", "repository": { "type": "git", From 8c66fad96b28ae498b78610d88089d9c9a798eb1 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 17:13:12 +0900 Subject: [PATCH 09/20] lint --- packages/frontend/.eslintrc.js | 1 + packages/frontend/src/pages/admin/RolesEditorFormula.vue | 4 ++-- packages/frontend/src/pages/admin/roles.role.vue | 6 +++--- packages/frontend/src/pages/admin/roles.vue | 8 ++++---- packages/frontend/src/pages/admin/security.vue | 8 ++++---- packages/frontend/src/pages/gallery/edit.vue | 2 +- packages/frontend/src/pages/gallery/index.vue | 6 +++--- packages/frontend/src/pages/gallery/post.vue | 2 +- packages/frontend/src/widgets/server-metric/index.vue | 2 +- 9 files changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/frontend/.eslintrc.js b/packages/frontend/.eslintrc.js index 303b74ca1..24c3ad4b8 100644 --- a/packages/frontend/.eslintrc.js +++ b/packages/frontend/.eslintrc.js @@ -56,6 +56,7 @@ module.exports = { 'vue/require-v-for-key': 'warn', 'vue/no-unused-components': 'warn', 'vue/no-unused-vars': 'warn', + 'vue/no-dupe-keys': 'warn', 'vue/valid-v-for': 'warn', 'vue/return-in-computed-property': 'warn', 'vue/no-setup-props-destructure': 'warn', diff --git a/packages/frontend/src/pages/admin/RolesEditorFormula.vue b/packages/frontend/src/pages/admin/RolesEditorFormula.vue index 343d2c4c5..36e3c3239 100644 --- a/packages/frontend/src/pages/admin/RolesEditorFormula.vue +++ b/packages/frontend/src/pages/admin/RolesEditorFormula.vue @@ -25,11 +25,11 @@
- + diff --git a/packages/frontend/src/pages/admin/roles.role.vue b/packages/frontend/src/pages/admin/roles.role.vue index 6eac90257..4ed6abf20 100644 --- a/packages/frontend/src/pages/admin/roles.role.vue +++ b/packages/frontend/src/pages/admin/roles.role.vue @@ -2,7 +2,7 @@
- +
{{ i18n.ts.edit }} @@ -11,9 +11,9 @@ - + - + diff --git a/packages/frontend/src/pages/admin/roles.vue b/packages/frontend/src/pages/admin/roles.vue index e8dbe1c5f..6634d9cba 100644 --- a/packages/frontend/src/pages/admin/roles.vue +++ b/packages/frontend/src/pages/admin/roles.vue @@ -2,7 +2,7 @@
- +
@@ -14,7 +14,7 @@ - + @@ -156,13 +156,13 @@
- +
- +
diff --git a/packages/frontend/src/pages/admin/security.vue b/packages/frontend/src/pages/admin/security.vue index cd8ef9e68..efb9f81f2 100644 --- a/packages/frontend/src/pages/admin/security.vue +++ b/packages/frontend/src/pages/admin/security.vue @@ -1,7 +1,7 @@