From fd7b77c542b51313d8b8ea60124725fe65a295d5 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 29 May 2023 11:54:49 +0900 Subject: [PATCH] enhance(backend): migrate bull to bullmq (#10910) * wip * wip * Update QueueService.ts * wip * refactor * :v: * fix * Update QueueStatsService.ts * refactor * Update ApNoteService.ts * Update mock-resolver.ts * refactor * Update mock-resolver.ts --- CHANGELOG.md | 1 + packages/backend/package.json | 3 +- packages/backend/src/config.ts | 2 +- packages/backend/src/core/CaptchaService.ts | 20 +- .../src/core/FetchInstanceMetadataService.ts | 6 +- .../backend/src/core/NoteCreateService.ts | 2 +- packages/backend/src/core/QueueModule.ts | 53 +-- packages/backend/src/core/QueueService.ts | 65 +++- .../core/activitypub/LdSignatureService.ts | 4 +- .../core/activitypub/models/ApNoteService.ts | 6 +- .../backend/src/daemons/QueueStatsService.ts | 16 +- packages/backend/src/misc/id/aid.ts | 2 +- packages/backend/src/misc/prelude/time.ts | 17 +- .../src/queue/QueueProcessorService.ts | 320 +++++++++++------- packages/backend/src/queue/const.ts | 26 ++ packages/backend/src/queue/get-job-info.ts | 15 - .../AggregateRetentionProcessorService.ts | 6 +- .../CheckExpiredMutingsProcessorService.ts | 5 +- .../processors/CleanChartsProcessorService.ts | 5 +- .../queue/processors/CleanProcessorService.ts | 5 +- .../CleanRemoteFilesProcessorService.ts | 11 +- .../DeleteAccountProcessorService.ts | 6 +- .../DeleteDriveFilesProcessorService.ts | 14 +- .../processors/DeleteFileProcessorService.ts | 6 +- .../processors/DeliverProcessorService.ts | 10 +- .../EndedPollNotificationProcessorService.ts | 7 +- .../ExportAntennasProcessorService.ts | 6 +- .../ExportBlockingProcessorService.ts | 15 +- .../ExportCustomEmojisProcessorService.ts | 39 +-- .../ExportFavoritesProcessorService.ts | 11 +- .../ExportFollowingProcessorService.ts | 11 +- .../ExportMutingProcessorService.ts | 15 +- .../processors/ExportNotesProcessorService.ts | 11 +- .../ExportUserListsProcessorService.ts | 11 +- .../ImportAntennasProcessorService.ts | 6 +- .../ImportBlockingProcessorService.ts | 13 +- .../ImportCustomEmojisProcessorService.ts | 6 +- .../ImportFollowingProcessorService.ts | 13 +- .../ImportMutingProcessorService.ts | 13 +- .../ImportUserListsProcessorService.ts | 7 +- .../queue/processors/InboxProcessorService.ts | 38 +-- .../RelationshipProcessorService.ts | 2 +- .../ResyncChartsProcessorService.ts | 5 +- .../processors/TickChartsProcessorService.ts | 5 +- .../WebhookDeliverProcessorService.ts | 6 +- .../server/api/endpoints/admin/relays/add.ts | 2 +- .../api/endpoints/notes/search-by-tag.ts | 4 +- .../src/server/api/endpoints/reset-db.ts | 2 +- packages/backend/test/misc/mock-resolver.ts | 6 +- packages/backend/test/utils.ts | 2 +- packages/frontend/src/scripts/time.ts | 17 +- packages/shared/.eslintrc.js | 2 +- pnpm-lock.yaml | 121 ++++--- 53 files changed, 532 insertions(+), 490 deletions(-) create mode 100644 packages/backend/src/queue/const.ts delete mode 100644 packages/backend/src/queue/get-job-info.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ecbad51e9..6b107c72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ - fix:ロールタイムラインにて全ての投稿が流れてしまう問題の修正 ### Server +- bullをbull-mqにアップグレードし、ジョブキューのパフォーマンスを改善 - Fix: お知らせの画像URLを空にできない問題を修正 ## 13.12.2 diff --git a/packages/backend/package.json b/packages/backend/package.json index 71da3bb55..99c04d6bf 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -76,7 +76,7 @@ "autwh": "0.1.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", - "bull": "4.10.4", + "bullmq": "3.14.1", "cacheable-lookup": "6.1.0", "cbor": "8.1.0", "chalk": "5.2.0", @@ -167,7 +167,6 @@ "@types/accepts": "1.3.5", "@types/archiver": "5.3.2", "@types/bcryptjs": "2.4.2", - "@types/bull": "4.10.0", "@types/cbor": "6.0.0", "@types/color-convert": "2.0.0", "@types/content-disposition": "0.5.5", diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 744f99959..9d1945e4d 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -190,6 +190,6 @@ function tryCreateUrl(url: string) { try { return new URL(url); } catch (e) { - throw `url="${url}" is not a valid URL.`; + throw new Error(`url="${url}" is not a valid URL.`); } } diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 7aaa1b833..1a52a229c 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -30,7 +30,7 @@ export class CaptchaService { }, { throwErrorWhenResponseNotOk: false }); if (!res.ok) { - throw `${res.status}`; + throw new Error(`${res.status}`); } return await res.json() as CaptchaResponse; @@ -39,48 +39,48 @@ export class CaptchaService { @bindThis public async verifyRecaptcha(secret: string, response: string | null | undefined): Promise { if (response == null) { - throw 'recaptcha-failed: no response provided'; + throw new Error('recaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => { - throw `recaptcha-request-failed: ${err}`; + throw new Error(`recaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw `recaptcha-failed: ${errorCodes}`; + throw new Error(`recaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyHcaptcha(secret: string, response: string | null | undefined): Promise { if (response == null) { - throw 'hcaptcha-failed: no response provided'; + throw new Error('hcaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => { - throw `hcaptcha-request-failed: ${err}`; + throw new Error(`hcaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw `hcaptcha-failed: ${errorCodes}`; + throw new Error(`hcaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyTurnstile(secret: string, response: string | null | undefined): Promise { if (response == null) { - throw 'turnstile-failed: no response provided'; + throw new Error('turnstile-failed: no response provided'); } const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => { - throw `turnstile-request-failed: ${err}`; + throw new Error(`turnstile-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw `turnstile-failed: ${errorCodes}`; + throw new Error(`turnstile-failed: ${errorCodes}`); } } } diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 8103d5afe..9de633350 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -116,14 +116,14 @@ export class FetchInstanceMetadataService { const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo') .catch(err => { if (err.statusCode === 404) { - throw 'No nodeinfo provided'; + throw new Error('No nodeinfo provided'); } else { throw err.statusCode ?? err.message; } }) as Record; if (wellknown.links == null || !Array.isArray(wellknown.links)) { - throw 'No wellknown links'; + throw new Error('No wellknown links'); } const links = wellknown.links as any[]; @@ -134,7 +134,7 @@ export class FetchInstanceMetadataService { const link = lnik2_1 ?? lnik2_0 ?? lnik1_0; if (link == null) { - throw 'No nodeinfo link provided'; + throw new Error('No nodeinfo link provided'); } const info = await this.httpRequestService.getJson(link.href) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 977c9052c..2fd7a8ac8 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -510,7 +510,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (data.poll && data.poll.expiresAt) { const delay = data.poll.expiresAt.getTime() - Date.now(); - this.queueService.endedPollNotificationQueue.add({ + this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { delay, diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index 1d7394777..6db9bb14c 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -1,42 +1,11 @@ import { setTimeout } from 'node:timers/promises'; import { Inject, Module, OnApplicationShutdown } from '@nestjs/common'; -import Bull from 'bull'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { Provider } from '@nestjs/common'; -import type { DeliverJobData, InboxJobData, DbJobData, ObjectStorageJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData, DbJobMap } from '../queue/types.js'; - -function q(config: Config, name: string, limitPerSec = -1) { - return new Bull(name, { - redis: { - port: config.redisForJobQueue.port, - host: config.redisForJobQueue.host, - family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, - password: config.redisForJobQueue.pass, - db: config.redisForJobQueue.db ?? 0, - }, - prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue` : 'queue', - limiter: limitPerSec > 0 ? { - max: limitPerSec, - duration: 1000, - } : undefined, - settings: { - backoffStrategies: { - apBackoff, - }, - }, - }); -} - -// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 -function apBackoff(attemptsMade: number, err: Error) { - const baseDelay = 60 * 1000; // 1min - const maxBackoff = 8 * 60 * 60 * 1000; // 8hours - let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; - backoff = Math.min(backoff, maxBackoff); - backoff += Math.round(backoff * Math.random() * 0.2); - return backoff; -} +import type { DeliverJobData, InboxJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData } from '../queue/types.js'; export type SystemQueue = Bull.Queue>; export type EndedPollNotificationQueue = Bull.Queue; @@ -49,49 +18,49 @@ export type WebhookDeliverQueue = Bull.Queue; const $system: Provider = { provide: 'queue:system', - useFactory: (config: Config) => q(config, 'system'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)), inject: [DI.config], }; const $endedPollNotification: Provider = { provide: 'queue:endedPollNotification', - useFactory: (config: Config) => q(config, 'endedPollNotification'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)), inject: [DI.config], }; const $deliver: Provider = { provide: 'queue:deliver', - useFactory: (config: Config) => q(config, 'deliver', config.deliverJobPerSec ?? 128), + useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)), inject: [DI.config], }; const $inbox: Provider = { provide: 'queue:inbox', - useFactory: (config: Config) => q(config, 'inbox', config.inboxJobPerSec ?? 16), + useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)), inject: [DI.config], }; const $db: Provider = { provide: 'queue:db', - useFactory: (config: Config) => q(config, 'db'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)), inject: [DI.config], }; const $relationship: Provider = { provide: 'queue:relationship', - useFactory: (config: Config) => q(config, 'relationship', config.relashionshipJobPerSec ?? 64), + useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)), inject: [DI.config], }; const $objectStorage: Provider = { provide: 'queue:objectStorage', - useFactory: (config: Config) => q(config, 'objectStorage'), + useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)), inject: [DI.config], }; const $webhookDeliver: Provider = { provide: 'queue:webhookDeliver', - useFactory: (config: Config) => q(config, 'webhookDeliver', 64), + useFactory: (config: Config) => new Bull.Queue(QUEUE.WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.WEBHOOK_DELIVER)), inject: [DI.config], }; diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index b4ffffecc..2ae8a2b75 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -1,6 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; import { v4 as uuid } from 'uuid'; -import Bull from 'bull'; import type { IActivity } from '@/core/activitypub/type.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; @@ -11,6 +10,7 @@ import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; +import type * as Bull from 'bullmq'; @Injectable() export class QueueService { @@ -26,7 +26,43 @@ export class QueueService { @Inject('queue:relationship') public relationshipQueue: RelationshipQueue, @Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue, @Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue, - ) {} + ) { + this.systemQueue.add('tickCharts', { + }, { + repeat: { pattern: '55 * * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('resyncCharts', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('cleanCharts', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('aggregateRetention', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('clean', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); + + this.systemQueue.add('checkExpiredMutings', { + }, { + repeat: { pattern: '*/5 * * * *' }, + removeOnComplete: true, + }); + } @bindThis public deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { @@ -42,11 +78,10 @@ export class QueueService { isSharedInbox, }; - return this.deliverQueue.add(data, { + return this.deliverQueue.add(to, data, { attempts: this.config.deliverJobMaxAttempts ?? 12, - timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -60,11 +95,10 @@ export class QueueService { signature, }; - return this.inboxQueue.add(data, { + return this.inboxQueue.add('', data, { attempts: this.config.inboxJobMaxAttempts ?? 8, - timeout: 5 * 60 * 1000, // 5min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -212,7 +246,7 @@ export class QueueService { private generateToDbJobData>(name: T, data: D): { name: string, data: D, - opts: Bull.JobOptions, + opts: Bull.JobsOptions, } { return { name, @@ -299,10 +333,10 @@ export class QueueService { } @bindThis - private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobOptions = {}): { + private generateRelationshipJobData(name: 'follow' | 'unfollow' | 'block' | 'unblock', data: RelationshipJobData, opts: Bull.JobsOptions = {}): { name: string, data: RelationshipJobData, - opts: Bull.JobOptions, + opts: Bull.JobsOptions, } { return { name, @@ -351,11 +385,10 @@ export class QueueService { eventId: uuid(), }; - return this.webhookDeliverQueue.add(data, { + return this.webhookDeliverQueue.add(webhook.id, data, { attempts: 4, - timeout: 1 * 60 * 1000, // 1min backoff: { - type: 'apBackoff', + type: 'custom', }, removeOnComplete: true, removeOnFail: true, @@ -367,11 +400,11 @@ export class QueueService { this.deliverQueue.once('cleaned', (jobs, status) => { //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.deliverQueue.clean(0, 'delayed'); + this.deliverQueue.clean(0, Infinity, 'delayed'); this.inboxQueue.once('cleaned', (jobs, status) => { //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.inboxQueue.clean(0, 'delayed'); + this.inboxQueue.clean(0, Infinity, 'delayed'); } } diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index 2dc1a410a..20fe2a0a7 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -94,7 +94,7 @@ class LdSignature { @bindThis private getLoader() { return async (url: string): Promise => { - if (!url.match('^https?\:\/\/')) throw `Invalid URL ${url}`; + if (!url.match('^https?\:\/\/')) throw new Error(`Invalid URL ${url}`); if (this.preLoad) { if (url in CONTEXTS) { @@ -126,7 +126,7 @@ class LdSignature { timeout: this.loderTimeout, }, { throwErrorWhenResponseNotOk: false }).then(res => { if (!res.ok) { - throw `${res.status} ${res.statusText}`; + throw new Error(`${res.status} ${res.statusText}`); } else { return res.json(); } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 87a9db405..76757f530 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -18,6 +18,7 @@ import { PollService } from '@/core/PollService.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { checkHttps } from '@/misc/check-https.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; // eslint-disable-next-line @typescript-eslint/consistent-type-imports import { ApLoggerService } from '../ApLoggerService.js'; @@ -32,7 +33,6 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; -import { checkHttps } from '@/misc/check-https.js'; @Injectable() export class ApNoteService { @@ -230,7 +230,7 @@ export class ApNoteService { quote = results.filter((x): x is { status: 'ok', res: Note | null } => x.status === 'ok').map(x => x.res).find(x => x); if (!quote) { if (results.some(x => x.status === 'temperror')) { - throw 'quote resolve failed'; + throw new Error('quote resolve failed'); } } } @@ -311,7 +311,7 @@ export class ApNoteService { // ブロックしてたら中断 const meta = await this.metaService.fetch(); - if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) throw { statusCode: 451 }; + if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) throw new StatusError('blocked host', 451); const unlock = await this.appLockService.getApLock(uri); diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index b717434e0..0a5b3184d 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -1,7 +1,11 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import Xev from 'xev'; +import * as Bull from 'bullmq'; import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { QUEUE, baseQueueOptions } from '@/queue/const.js'; import type { OnApplicationShutdown } from '@nestjs/common'; const ev = new Xev(); @@ -13,6 +17,9 @@ export class QueueStatsService implements OnApplicationShutdown { private intervalId: NodeJS.Timer; constructor( + @Inject(DI.config) + private config: Config, + private queueService: QueueService, ) { } @@ -31,11 +38,14 @@ export class QueueStatsService implements OnApplicationShutdown { let activeDeliverJobs = 0; let activeInboxJobs = 0; - this.queueService.deliverQueue.on('global:active', () => { + const deliverQueueEvents = new Bull.QueueEvents(QUEUE.DELIVER, baseQueueOptions(this.config, QUEUE.DELIVER)); + const inboxQueueEvents = new Bull.QueueEvents(QUEUE.INBOX, baseQueueOptions(this.config, QUEUE.INBOX)); + + deliverQueueEvents.on('active', () => { activeDeliverJobs++; }); - this.queueService.inboxQueue.on('global:active', () => { + inboxQueueEvents.on('active', () => { activeInboxJobs++; }); diff --git a/packages/backend/src/misc/id/aid.ts b/packages/backend/src/misc/id/aid.ts index 9e206ee98..f0cbc9900 100644 --- a/packages/backend/src/misc/id/aid.ts +++ b/packages/backend/src/misc/id/aid.ts @@ -21,7 +21,7 @@ function getNoise(): string { export function genAid(date: Date): string { const t = date.getTime(); - if (isNaN(t)) throw 'Failed to create AID: Invalid Date'; + if (isNaN(t)) throw new Error('Failed to create AID: Invalid Date'); counter++; return getTime(t) + getNoise(); } diff --git a/packages/backend/src/misc/prelude/time.ts b/packages/backend/src/misc/prelude/time.ts index 34e8b6b17..b21978b18 100644 --- a/packages/backend/src/misc/prelude/time.ts +++ b/packages/backend/src/misc/prelude/time.ts @@ -5,15 +5,16 @@ const dateTimeIntervals = { }; export function dateUTC(time: number[]): Date { - const d = time.length === 2 ? Date.UTC(time[0], time[1]) - : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) - : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) - : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) - : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) - : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) - : null; + const d = + time.length === 2 ? Date.UTC(time[0], time[1]) + : time.length === 3 ? Date.UTC(time[0], time[1], time[2]) + : time.length === 4 ? Date.UTC(time[0], time[1], time[2], time[3]) + : time.length === 5 ? Date.UTC(time[0], time[1], time[2], time[3], time[4]) + : time.length === 6 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5]) + : time.length === 7 ? Date.UTC(time[0], time[1], time[2], time[3], time[4], time[5], time[6]) + : null; - if (!d) throw 'wrong number of arguments'; + if (!d) throw new Error('wrong number of arguments'); return new Date(d); } diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index dc025f988..011082cd3 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -1,10 +1,9 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; -import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; -import { getJobInfo } from './get-job-info.js'; import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; @@ -35,6 +34,33 @@ import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMu import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; +import { QUEUE, baseQueueOptions } from './const.js'; + +// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 +function httpRelatedBackoff(attemptsMade: number) { + const baseDelay = 60 * 1000; // 1min + const maxBackoff = 8 * 60 * 60 * 1000; // 8hours + let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; + backoff = Math.min(backoff, maxBackoff); + backoff += Math.round(backoff * Math.random() * 0.2); + return backoff; +} + +function getJobInfo(job: Bull.Job | undefined, increment = false): string { + if (job == null) return '-'; + + const age = Date.now() - job.timestamp; + + const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` + : age > 10000 ? `${Math.floor(age / 1000)}s` + : `${age}ms`; + + // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする + const currentAttempts = job.attemptsMade + (increment ? 1 : 0); + const maxAttempts = job.opts ? job.opts.attempts : 0; + + return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; +} @Injectable() export class QueueProcessorService { @@ -45,7 +71,6 @@ export class QueueProcessorService { private config: Config, private queueLoggerService: QueueLoggerService, - private queueService: QueueService, private webhookDeliverProcessorService: WebhookDeliverProcessorService, private endedPollNotificationProcessorService: EndedPollNotificationProcessorService, private deliverProcessorService: DeliverProcessorService, @@ -97,146 +122,191 @@ export class QueueProcessorService { } } - const systemLogger = this.logger.createSubLogger('system'); - const deliverLogger = this.logger.createSubLogger('deliver'); - const webhookLogger = this.logger.createSubLogger('webhook'); - const inboxLogger = this.logger.createSubLogger('inbox'); - const dbLogger = this.logger.createSubLogger('db'); - const relationshipLogger = this.logger.createSubLogger('relationship'); - const objectStorageLogger = this.logger.createSubLogger('objectStorage'); + //#region system + const systemQueueWorker = new Bull.Worker(QUEUE.SYSTEM, (job) => { + switch (job.name) { + case 'tickCharts': return this.tickChartsProcessorService.process(); + case 'resyncCharts': return this.resyncChartsProcessorService.process(); + case 'cleanCharts': return this.cleanChartsProcessorService.process(); + case 'aggregateRetention': return this.aggregateRetentionProcessorService.process(); + case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process(); + case 'clean': return this.cleanProcessorService.process(); + default: throw new Error(`unrecognized job type ${job.name} for system`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.SYSTEM), + }); - this.queueService.systemQueue - .on('waiting', (jobId) => systemLogger.debug(`waiting id=${jobId}`)) + const systemLogger = this.logger.createSubLogger('system'); + + systemQueueWorker .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => systemLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => systemLogger.warn(`stalled id=${job.id}`)); + .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => systemLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.deliverQueue - .on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) - .on('error', (job: any, err: Error) => deliverLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); + //#region db + const dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { + switch (job.name) { + case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); + case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); + case 'exportNotes': return this.exportNotesProcessorService.process(job); + case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); + case 'exportFollowing': return this.exportFollowingProcessorService.process(job); + case 'exportMuting': return this.exportMutingProcessorService.process(job); + case 'exportBlocking': return this.exportBlockingProcessorService.process(job); + case 'exportUserLists': return this.exportUserListsProcessorService.process(job); + case 'exportAntennas': return this.exportAntennasProcessorService.process(job); + case 'importFollowing': return this.importFollowingProcessorService.process(job); + case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); + case 'importMuting': return this.importMutingProcessorService.process(job); + case 'importBlocking': return this.importBlockingProcessorService.process(job); + case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job); + case 'importUserLists': return this.importUserListsProcessorService.process(job); + case 'importCustomEmojis': return this.importCustomEmojisProcessorService.process(job); + case 'importAntennas': return this.importAntennasProcessorService.process(job); + case 'deleteAccount': return this.deleteAccountProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for db`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.DB), + }); - this.queueService.inboxQueue - .on('waiting', (jobId) => inboxLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) - .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) - .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => inboxLogger.warn(`stalled ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`)); + const dbLogger = this.logger.createSubLogger('db'); - this.queueService.dbQueue - .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`)) + dbQueueWorker .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); + .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => dbLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.relationshipQueue - .on('waiting', (jobId) => relationshipLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => relationshipLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => relationshipLogger.warn(`stalled id=${job.id}`)); + //#region deliver + const deliverQueueWorker = new Bull.Worker(QUEUE.DELIVER, (job) => this.deliverProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.DELIVER), + concurrency: this.config.deliverJobConcurrency ?? 128, + limiter: { + max: this.config.deliverJobPerSec ?? 128, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); - this.queueService.objectStorageQueue - .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`)) - .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) - .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) - .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) - .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`)); + const deliverLogger = this.logger.createSubLogger('deliver'); - this.queueService.webhookDeliverQueue - .on('waiting', (jobId) => webhookLogger.debug(`waiting id=${jobId}`)) + deliverQueueWorker + .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) + .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => deliverLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region inbox + const inboxQueueWorker = new Bull.Worker(QUEUE.INBOX, (job) => this.inboxProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.INBOX), + concurrency: this.config.inboxJobConcurrency ?? 16, + limiter: { + max: this.config.inboxJobPerSec ?? 16, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); + + const inboxLogger = this.logger.createSubLogger('inbox'); + + inboxQueueWorker + .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) + .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) + .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => inboxLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region webhook deliver + const webhookDeliverQueueWorker = new Bull.Worker(QUEUE.WEBHOOK_DELIVER, (job) => this.webhookDeliverProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.WEBHOOK_DELIVER), + concurrency: 64, + limiter: { + max: 64, + duration: 1000, + }, + settings: { + backoffStrategy: httpRelatedBackoff, + }, + }); + + const webhookLogger = this.logger.createSubLogger('webhook'); + + webhookDeliverQueueWorker .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) - .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) - .on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) })) - .on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); + .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`)) + .on('error', (err: Error) => webhookLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.systemQueue.add('tickCharts', { + //#region relationship + const relationshipQueueWorker = new Bull.Worker(QUEUE.RELATIONSHIP, (job) => { + switch (job.name) { + case 'follow': return this.relationshipProcessorService.processFollow(job); + case 'unfollow': return this.relationshipProcessorService.processUnfollow(job); + case 'block': return this.relationshipProcessorService.processBlock(job); + case 'unblock': return this.relationshipProcessorService.processUnblock(job); + default: throw new Error(`unrecognized job type ${job.name} for relationship`); + } }, { - repeat: { cron: '55 * * * *' }, - removeOnComplete: true, + ...baseQueueOptions(this.config, QUEUE.RELATIONSHIP), + concurrency: this.config.relashionshipJobConcurrency ?? 16, + limiter: { + max: this.config.relashionshipJobPerSec ?? 64, + duration: 1000, + }, }); - this.queueService.systemQueue.add('resyncCharts', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('cleanCharts', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('aggregateRetention', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('clean', { - }, { - repeat: { cron: '0 0 * * *' }, - removeOnComplete: true, - }); - - this.queueService.systemQueue.add('checkExpiredMutings', { - }, { - repeat: { cron: '*/5 * * * *' }, - removeOnComplete: true, - }); - - this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job)); - this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job)); - this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done)); - this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job)); - - this.queueService.dbQueue.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportUserLists', (job, done) => this.exportUserListsProcessorService.process(job, done)); - this.queueService.dbQueue.process('exportAntennas', (job, done) => this.exportAntennasProcessorService.process(job, done)); - this.queueService.dbQueue.process('importFollowing', (job, done) => this.importFollowingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importFollowingToDb', (job) => this.importFollowingProcessorService.processDb(job)); - this.queueService.dbQueue.process('importMuting', (job, done) => this.importMutingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importBlocking', (job, done) => this.importBlockingProcessorService.process(job, done)); - this.queueService.dbQueue.process('importBlockingToDb', (job) => this.importBlockingProcessorService.processDb(job)); - this.queueService.dbQueue.process('importUserLists', (job, done) => this.importUserListsProcessorService.process(job, done)); - this.queueService.dbQueue.process('importCustomEmojis', (job, done) => this.importCustomEmojisProcessorService.process(job, done)); - this.queueService.dbQueue.process('importAntennas', (job, done) => this.importAntennasProcessorService.process(job, done)); - this.queueService.dbQueue.process('deleteAccount', (job) => this.deleteAccountProcessorService.process(job)); - - this.queueService.objectStorageQueue.process('deleteFile', 16, (job) => this.deleteFileProcessorService.process(job)); - this.queueService.objectStorageQueue.process('cleanRemoteFiles', 16, (job, done) => this.cleanRemoteFilesProcessorService.process(job, done)); + const relationshipLogger = this.logger.createSubLogger('relationship'); - { - const maxJobs = this.config.relashionshipJobConcurrency ?? 16; - this.queueService.relationshipQueue.process('follow', maxJobs, (job) => this.relationshipProcessorService.processFollow(job)); - this.queueService.relationshipQueue.process('unfollow', maxJobs, (job) => this.relationshipProcessorService.processUnfollow(job)); - this.queueService.relationshipQueue.process('block', maxJobs, (job) => this.relationshipProcessorService.processBlock(job)); - this.queueService.relationshipQueue.process('unblock', maxJobs, (job) => this.relationshipProcessorService.processUnblock(job)); - } + relationshipQueueWorker + .on('active', (job) => relationshipLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => relationshipLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => relationshipLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`)); + //#endregion - this.queueService.systemQueue.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done)); - this.queueService.systemQueue.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done)); - this.queueService.systemQueue.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done)); - this.queueService.systemQueue.process('clean', (job, done) => this.cleanProcessorService.process(job, done)); + //#region object storage + const objectStorageQueueWorker = new Bull.Worker(QUEUE.OBJECT_STORAGE, (job) => { + switch (job.name) { + case 'deleteFile': return this.deleteFileProcessorService.process(job); + case 'cleanRemoteFiles': return this.cleanRemoteFilesProcessorService.process(job); + default: throw new Error(`unrecognized job type ${job.name} for objectStorage`); + } + }, { + ...baseQueueOptions(this.config, QUEUE.OBJECT_STORAGE), + concurrency: 16, + }); + + const objectStorageLogger = this.logger.createSubLogger('objectStorage'); + + objectStorageQueueWorker + .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job ? job.id : '-'}`, { job, e: renderError(err) })) + .on('error', (err: Error) => objectStorageLogger.error(`error ${err}`, { e: renderError(err) })) + .on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`)); + //#endregion + + //#region ended poll notification + const endedPollNotificationWorker = new Bull.Worker(QUEUE.ENDED_POLL_NOTIFICATION, (job) => this.endedPollNotificationProcessorService.process(job), { + ...baseQueueOptions(this.config, QUEUE.ENDED_POLL_NOTIFICATION), + }); + //#endregion } } diff --git a/packages/backend/src/queue/const.ts b/packages/backend/src/queue/const.ts new file mode 100644 index 000000000..d240fe70e --- /dev/null +++ b/packages/backend/src/queue/const.ts @@ -0,0 +1,26 @@ +import { Config } from '@/config.js'; +import type * as Bull from 'bullmq'; + +export const QUEUE = { + DELIVER: 'deliver', + INBOX: 'inbox', + SYSTEM: 'system', + ENDED_POLL_NOTIFICATION: 'endedPollNotification', + DB: 'db', + RELATIONSHIP: 'relationship', + OBJECT_STORAGE: 'objectStorage', + WEBHOOK_DELIVER: 'webhookDeliver', +}; + +export function baseQueueOptions(config: Config, queueName: typeof QUEUE[keyof typeof QUEUE]): Bull.QueueOptions { + return { + connection: { + port: config.redisForJobQueue.port, + host: config.redisForJobQueue.host, + family: config.redisForJobQueue.family == null ? 0 : config.redisForJobQueue.family, + password: config.redisForJobQueue.pass, + db: config.redisForJobQueue.db ?? 0, + }, + prefix: config.redisForJobQueue.prefix ? `${config.redisForJobQueue.prefix}:queue:${queueName}` : `queue:${queueName}`, + }; +} diff --git a/packages/backend/src/queue/get-job-info.ts b/packages/backend/src/queue/get-job-info.ts deleted file mode 100644 index d33e349c3..000000000 --- a/packages/backend/src/queue/get-job-info.ts +++ /dev/null @@ -1,15 +0,0 @@ -import Bull from 'bull'; - -export function getJobInfo(job: Bull.Job, increment = false) { - const age = Date.now() - job.timestamp; - - const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` - : age > 10000 ? `${Math.floor(age / 1000)}s` - : `${age}ms`; - - // onActiveとかonCompletedのattemptsMadeがなぜか0始まりなのでインクリメントする - const currentAttempts = job.attemptsMade + (increment ? 1 : 0); - const maxAttempts = job.opts ? job.opts.attempts : 0; - - return `id=${job.id} attempts=${currentAttempts}/${maxAttempts} age=${formated}`; -} diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts index e2720b4fe..600ce0828 100644 --- a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -9,7 +9,7 @@ import { deepClone } from '@/misc/clone.js'; import { IdService } from '@/core/IdService.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class AggregateRetentionProcessorService { @@ -32,7 +32,7 @@ export class AggregateRetentionProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Aggregating retention...'); const now = new Date(); @@ -62,7 +62,6 @@ export class AggregateRetentionProcessorService { } catch (err) { if (isDuplicateKeyValueError(err)) { this.logger.succ('Skip because it has already been processed by another worker.'); - done(); return; } throw err; @@ -88,6 +87,5 @@ export class AggregateRetentionProcessorService { } this.logger.succ('Retention aggregated.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts index 2476d71a5..c4ee212ba 100644 --- a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CheckExpiredMutingsProcessorService { @@ -27,7 +27,7 @@ export class CheckExpiredMutingsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Checking expired mutings...'); const expired = await this.mutingsRepository.createQueryBuilder('muting') @@ -41,6 +41,5 @@ export class CheckExpiredMutingsProcessorService { } this.logger.succ('All expired mutings checked.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts index b45816704..22d7c1b4f 100644 --- a/packages/backend/src/queue/processors/CleanChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanChartsProcessorService { @@ -45,7 +45,7 @@ export class CleanChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Clean charts...'); await Promise.all([ @@ -64,6 +64,5 @@ export class CleanChartsProcessorService { ]); this.logger.succ('All charts successfully cleaned.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index 1936e8df2..cefa6da5e 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -7,7 +7,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanProcessorService { @@ -36,7 +36,7 @@ export class CleanProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Cleaning...'); this.userIpsRepository.delete({ @@ -72,6 +72,5 @@ export class CleanProcessorService { } this.logger.succ('Cleaned.'); - done(); } } diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 5a33c2718..c54bf59ae 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -5,9 +5,9 @@ import type { DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; @Injectable() export class CleanRemoteFilesProcessorService { @@ -27,7 +27,7 @@ export class CleanRemoteFilesProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(job: Bull.Job>): Promise { this.logger.info('Deleting cached remote files...'); let deletedCount = 0; @@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService { }); if (files.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -62,10 +62,9 @@ export class CleanRemoteFilesProcessorService { isLink: false, }); - job.progress(deletedCount / total); + job.updateProgress(deletedCount / total); } this.logger.succ('All cached remote files has been deleted.'); - done(); } } diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index e36a78de6..39dd801af 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -8,10 +8,10 @@ import { DriveService } from '@/core/DriveService.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Note } from '@/models/entities/Note.js'; import { EmailService } from '@/core/EmailService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserDeleteJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserDeleteJobData } from '../types.js'; @Injectable() export class DeleteAccountProcessorService { diff --git a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts index 604497cf5..6772c5dc7 100644 --- a/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts @@ -5,10 +5,10 @@ import type { UsersRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class DeleteDriveFilesProcessorService { @@ -31,12 +31,11 @@ export class DeleteDriveFilesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Deleting drive files of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -56,7 +55,7 @@ export class DeleteDriveFilesProcessorService { }); if (files.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -71,10 +70,9 @@ export class DeleteDriveFilesProcessorService { userId: user.id, }); - job.progress(deletedCount / total); + job.updateProgress(deletedCount / total); } this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`); - done(); } } diff --git a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts index 2fb2f56f8..edf87bd92 100644 --- a/packages/backend/src/queue/processors/DeleteFileProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteFileProcessorService.ts @@ -3,10 +3,10 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { ObjectStorageFileJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { ObjectStorageFileJobData } from '../types.js'; @Injectable() export class DeleteFileProcessorService { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index f293bd4d7..406e9df85 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -1,4 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, InstancesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -16,7 +17,6 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { DeliverJobData } from '../types.js'; @Injectable() @@ -121,15 +121,13 @@ export class DeliverProcessorService { isSuspended: true, }); }); - return `${host} is gone`; + throw new Bull.UnrecoverableError(`${host} is gone`); } - // HTTPステータスコード4xxはクライアントエラーであり、それはつまり - // 何回再送しても成功することはないということなのでエラーにはしないでおく - return `${res.statusCode} ${res.statusMessage}`; + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } // 5xx etc. - throw `${res.statusCode} ${res.statusMessage}`; + throw new Error(`${res.statusCode} ${res.statusMessage}`); } else { // DNS error, socket error, timeout ... throw res; diff --git a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts index 501ed4090..21501592f 100644 --- a/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts +++ b/packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts @@ -6,7 +6,7 @@ import type Logger from '@/logger.js'; import { NotificationService } from '@/core/NotificationService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { EndedPollNotificationJobData } from '../types.js'; @Injectable() @@ -30,10 +30,9 @@ export class EndedPollNotificationProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const note = await this.notesRepository.findOneBy({ id: job.data.noteId }); if (note == null || !note.hasPoll) { - done(); return; } @@ -51,7 +50,5 @@ export class EndedPollNotificationProcessorService { noteId: note.id, }); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index 894903e79..ac52325c8 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -12,7 +12,7 @@ import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DBExportAntennasData } from '../types.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ExportAntennasProcessorService { @@ -39,10 +39,9 @@ export class ExportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } const [path, cleanup] = await createTemp(); @@ -96,7 +95,6 @@ export class ExportAntennasProcessorService { this.logger.succ('Exported to: ' + driveFile.id); } finally { cleanup(); - done(); } } } diff --git a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts index c7b54070d..eb758e162 100644 --- a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class ExportBlockingProcessorService { @@ -36,12 +36,11 @@ export class ExportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -69,7 +68,7 @@ export class ExportBlockingProcessorService { }); if (blockings.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -99,7 +98,7 @@ export class ExportBlockingProcessorService { blockerId: user.id, }); - job.progress(exportedCount / total); + job.updateProgress(exportedCount / total); } stream.end(); @@ -112,7 +111,5 @@ export class ExportBlockingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index b50f373ef..3203d9f3e 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -13,7 +13,7 @@ import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ExportCustomEmojisProcessorService { @@ -37,12 +37,11 @@ export class ExportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Exporting custom emojis ...'); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -117,24 +116,26 @@ export class ExportCustomEmojisProcessorService { metaStream.end(); // Create archive - const [archivePath, archiveCleanup] = await createTemp(); - const archiveStream = fs.createWriteStream(archivePath); - const archive = archiver('zip', { - zlib: { level: 0 }, - }); - archiveStream.on('close', async () => { - this.logger.succ(`Exported to: ${archivePath}`); + await new Promise(async (resolve) => { + const [archivePath, archiveCleanup] = await createTemp(); + const archiveStream = fs.createWriteStream(archivePath); + const archive = archiver('zip', { + zlib: { level: 0 }, + }); + archiveStream.on('close', async () => { + this.logger.succ(`Exported to: ${archivePath}`); - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; - const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; + const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); - this.logger.succ(`Exported to: ${driveFile.id}`); - cleanup(); - archiveCleanup(); - done(); + this.logger.succ(`Exported to: ${driveFile.id}`); + cleanup(); + archiveCleanup(); + resolve(); + }); + archive.pipe(archiveStream); + archive.directory(path, false); + archive.finalize(); }); - archive.pipe(archiveStream); - archive.directory(path, false); - archive.finalize(); } } diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts index f2f2383a8..76c38a6b8 100644 --- a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -42,12 +42,11 @@ export class ExportFavoritesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -91,7 +90,7 @@ export class ExportFavoritesProcessorService { }) as (NoteFavorite & { note: Note & { user: User } })[]; if (favorites.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -112,7 +111,7 @@ export class ExportFavoritesProcessorService { userId: user.id, }); - job.progress(exportedFavoritesCount / total); + job.updateProgress(exportedFavoritesCount / total); } await write(']'); @@ -127,8 +126,6 @@ export class ExportFavoritesProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts index fa9c1ac1e..8726cb140 100644 --- a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts @@ -10,10 +10,10 @@ import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import type { Following } from '@/models/entities/Following.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbExportFollowingData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbExportFollowingData } from '../types.js'; @Injectable() export class ExportFollowingProcessorService { @@ -40,12 +40,11 @@ export class ExportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -116,7 +115,5 @@ export class ExportFollowingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts index b14bf5f5b..0f11a9e84 100644 --- a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class ExportMutingProcessorService { @@ -39,12 +39,11 @@ export class ExportMutingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -73,7 +72,7 @@ export class ExportMutingProcessorService { }); if (mutes.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -103,7 +102,7 @@ export class ExportMutingProcessorService { muterId: user.id, }); - job.progress(exportedCount / total); + job.updateProgress(exportedCount / total); } stream.end(); @@ -116,7 +115,5 @@ export class ExportMutingProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index e4f12ad10..24fb33188 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js'; import type { Note } from '@/models/entities/Note.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @Injectable() @@ -39,12 +39,11 @@ export class ExportNotesProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting notes of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -87,7 +86,7 @@ export class ExportNotesProcessorService { }) as Note[]; if (notes.length === 0) { - job.progress(100); + job.updateProgress(100); break; } @@ -108,7 +107,7 @@ export class ExportNotesProcessorService { userId: user.id, }); - job.progress(exportedNotesCount / total); + job.updateProgress(exportedNotesCount / total); } await write(']'); @@ -123,8 +122,6 @@ export class ExportNotesProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts index 54bde4404..ec6335805 100644 --- a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts @@ -9,10 +9,10 @@ import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbJobDataWithUser } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbJobDataWithUser } from '../types.js'; @Injectable() export class ExportUserListsProcessorService { @@ -39,12 +39,11 @@ export class ExportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Exporting user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -92,7 +91,5 @@ export class ExportUserListsProcessorService { } finally { cleanup(); } - - done(); } } diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index d06131b8c..575cad69d 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import { DBAntennaImportJobData } from '../types.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; const validate = new Ajv().compile({ type: 'object', @@ -59,7 +59,7 @@ export class ImportAntennasProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { const now = new Date(); try { for (const antenna of job.data.antenna) { @@ -89,8 +89,6 @@ export class ImportAntennasProcessorService { } } catch (err: any) { this.logger.error(err); - } finally { - done(); } } } diff --git a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts index 3f075b02d..2f1a9e5b0 100644 --- a/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportBlockingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; @Injectable() export class ImportBlockingProcessorService { @@ -34,12 +34,11 @@ export class ImportBlockingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing blocking of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -47,7 +46,6 @@ export class ImportBlockingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -56,7 +54,6 @@ export class ImportBlockingProcessorService { this.queueService.createImportBlockingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); - done(); } @bindThis @@ -85,7 +82,7 @@ export class ImportBlockingProcessorService { } if (target == null) { - throw `Unable to resolve user: @${username}@${host}`; + throw new Error(`Unable to resolve user: @${username}@${host}`); } // skip myself diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 600468a28..d86256787 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -12,7 +12,7 @@ import { DriveService } from '@/core/DriveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; // TODO: 名前衝突時の動作を選べるようにする @@ -45,14 +45,13 @@ export class ImportCustomEmojisProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Importing custom emojis ...'); const file = await this.driveFilesRepository.findOneBy({ id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -116,7 +115,6 @@ export class ImportCustomEmojisProcessorService { cleanup(); this.logger.succ('Imported'); - done(); }); unzipStream.pipe(extractor); this.logger.succ(`Unzipping to ${outputPath}`); diff --git a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts index aa5cf12c5..15bee9672 100644 --- a/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportFollowingProcessorService.ts @@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js'; @Injectable() export class ImportFollowingProcessorService { @@ -34,12 +34,11 @@ export class ImportFollowingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing following of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -47,7 +46,6 @@ export class ImportFollowingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -56,7 +54,6 @@ export class ImportFollowingProcessorService { this.queueService.createImportFollowingToDbJob({ id: user.id }, targets); this.logger.succ('Import jobs created'); - done(); } @bindThis @@ -85,7 +82,7 @@ export class ImportFollowingProcessorService { } if (target == null) { - throw `Unable to resolve user: @${username}@${host}`; + throw new Error(`Unable to resolve user: @${username}@${host}`); } // skip myself diff --git a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts index 379994ee7..723935cd3 100644 --- a/packages/backend/src/queue/processors/ImportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportMutingProcessorService.ts @@ -9,10 +9,10 @@ import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { UserMutingService } from '@/core/UserMutingService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; -import type { DbUserImportJobData } from '../types.js'; import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbUserImportJobData } from '../types.js'; @Injectable() export class ImportMutingProcessorService { @@ -38,12 +38,11 @@ export class ImportMutingProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing muting of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -51,7 +50,6 @@ export class ImportMutingProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -83,7 +81,7 @@ export class ImportMutingProcessorService { } if (target == null) { - throw `cannot resolve user: @${username}@${host}`; + throw new Error(`cannot resolve user: @${username}@${host}`); } // skip myself @@ -98,6 +96,5 @@ export class ImportMutingProcessorService { } this.logger.succ('Imported'); - done(); } } diff --git a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts index c42386341..824ee8157 100644 --- a/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportUserListsProcessorService.ts @@ -12,7 +12,7 @@ import { IdService } from '@/core/IdService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import type { DbUserImportJobData } from '../types.js'; @Injectable() @@ -46,12 +46,11 @@ export class ImportUserListsProcessorService { } @bindThis - public async process(job: Bull.Job, done: () => void): Promise { + public async process(job: Bull.Job): Promise { this.logger.info(`Importing user lists of ${job.data.user.id} ...`); const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); if (user == null) { - done(); return; } @@ -59,7 +58,6 @@ export class ImportUserListsProcessorService { id: job.data.fileId, }); if (file == null) { - done(); return; } @@ -109,6 +107,5 @@ export class ImportUserListsProcessorService { } this.logger.succ('Imported'); - done(); } } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index ab8b1e9e2..ce1d7aaa1 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -1,8 +1,8 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import httpSignature from '@peertube/http-signature'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; -import type { InstancesRepository, DriveFilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { MetaService } from '@/core/MetaService.js'; @@ -23,10 +23,8 @@ import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { InboxJobData } from '../types.js'; -// ユーザーのinboxにアクティビティが届いた時の処理 @Injectable() export class InboxProcessorService { private logger: Logger; @@ -35,12 +33,6 @@ export class InboxProcessorService { @Inject(DI.config) private config: Config, - @Inject(DI.instancesRepository) - private instancesRepository: InstancesRepository, - - @Inject(DI.driveFilesRepository) - private driveFilesRepository: DriveFilesRepository, - private utilityService: UtilityService, private metaService: MetaService, private apInboxService: ApInboxService, @@ -93,24 +85,24 @@ export class InboxProcessorService { try { authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor)); } catch (err) { - // 対象が4xxならスキップ + // 対象が4xxならスキップ if (err instanceof StatusError) { if (err.isClientError) { - return `skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`; + throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); } - throw `Error in actor ${activity.actor} - ${err.statusCode ?? err}`; + throw new Error(`Error in actor ${activity.actor} - ${err.statusCode ?? err}`); } } } // それでもわからなければ終了 if (authUser == null) { - return 'skip: failed to resolve user'; + throw new Bull.UnrecoverableError('skip: failed to resolve user'); } // publicKey がなくても終了 if (authUser.key == null) { - return 'skip: failed to resolve user publicKey'; + throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey'); } // HTTP-Signatureの検証 @@ -118,10 +110,10 @@ export class InboxProcessorService { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { - // 一致しなくても、でもLD-Signatureがありそうならそっちも見る + // 一致しなくても、でもLD-Signatureがありそうならそっちも見る if (activity.signature) { if (activity.signature.type !== 'RsaSignature2017') { - return `skip: unsupported LD-signature type ${activity.signature.type}`; + throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`); } // activity.signature.creator: https://example.oom/users/user#main-key @@ -134,32 +126,32 @@ export class InboxProcessorService { // keyIdからLD-Signatureのユーザーを取得 authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator); if (authUser == null) { - return 'skip: LD-Signatureのユーザーが取得できませんでした'; + throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); } if (authUser.key == null) { - return 'skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'; + throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } // LD-Signature検証 const ldSignature = this.ldSignatureService.use(); const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { - return 'skip: LD-Signatureの検証に失敗しました'; + throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } // もう一度actorチェック if (authUser.user.uri !== activity.actor) { - return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`; + throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); } // ブロックしてたら中断 const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) { - return `Blocked request: ${ldHost}`; + throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); } } else { - return `skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`; + throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); } } @@ -168,7 +160,7 @@ export class InboxProcessorService { const signerHost = this.utilityService.extractDbHost(authUser.user.uri!); const activityIdHost = this.utilityService.extractDbHost(activity.id); if (signerHost !== activityIdHost) { - return `skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`; + throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`); } } diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index ff454df45..722260d94 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; diff --git a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts index e5840f3da..eab8e1e68 100644 --- a/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/ResyncChartsProcessorService.ts @@ -15,7 +15,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class ResyncChartsProcessorService { @@ -43,7 +43,7 @@ export class ResyncChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Resync charts...'); // TODO: ユーザーごとのチャートも更新する @@ -55,6 +55,5 @@ export class ResyncChartsProcessorService { ]); this.logger.succ('All charts successfully resynced.'); - done(); } } diff --git a/packages/backend/src/queue/processors/TickChartsProcessorService.ts b/packages/backend/src/queue/processors/TickChartsProcessorService.ts index 7ff84c15a..f1696bf56 100644 --- a/packages/backend/src/queue/processors/TickChartsProcessorService.ts +++ b/packages/backend/src/queue/processors/TickChartsProcessorService.ts @@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; +import type * as Bull from 'bullmq'; @Injectable() export class TickChartsProcessorService { @@ -45,7 +45,7 @@ export class TickChartsProcessorService { } @bindThis - public async process(job: Bull.Job>, done: () => void): Promise { + public async process(): Promise { this.logger.info('Tick charts...'); await Promise.all([ @@ -64,6 +64,5 @@ export class TickChartsProcessorService { ]); this.logger.succ('All charts successfully ticked.'); - done(); } } diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index 84a5c21c4..8b40c1674 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -1,4 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; +import * as Bull from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { WebhooksRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -7,7 +8,6 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; -import type Bull from 'bull'; import type { WebhookDeliverJobData } from '../types.js'; @Injectable() @@ -66,11 +66,11 @@ export class WebhookDeliverProcessorService { if (res instanceof StatusError) { // 4xx if (res.isClientError) { - return `${res.statusCode} ${res.statusMessage}`; + throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`); } // 5xx etc. - throw `${res.statusCode} ${res.statusMessage}`; + throw new Error(`${res.statusCode} ${res.statusMessage}`); } else { // DNS error, socket error, timeout ... throw res; diff --git a/packages/backend/src/server/api/endpoints/admin/relays/add.ts b/packages/backend/src/server/api/endpoints/admin/relays/add.ts index f12738bd3..f2d4aa899 100644 --- a/packages/backend/src/server/api/endpoints/admin/relays/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/relays/add.ts @@ -62,7 +62,7 @@ export default class extends Endpoint { ) { super(meta, paramDef, async (ps, me) => { try { - if (new URL(ps.inbox).protocol !== 'https:') throw 'https only'; + if (new URL(ps.inbox).protocol !== 'https:') throw new Error('https only'); } catch { throw new ApiError(meta.errors.invalidUrl); } diff --git a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts index 2956bf1cb..742df0ca9 100644 --- a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts +++ b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts @@ -82,14 +82,14 @@ export default class extends Endpoint { try { if (ps.tag) { - if (!safeForSql(normalizeForSearch(ps.tag))) throw 'Injection'; + if (!safeForSql(normalizeForSearch(ps.tag))) throw new Error('Injection'); query.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`); } else { query.andWhere(new Brackets(qb => { for (const tags of ps.query!) { qb.orWhere(new Brackets(qb => { for (const tag of tags) { - if (!safeForSql(normalizeForSearch(tag))) throw 'Injection'; + if (!safeForSql(normalizeForSearch(tag))) throw new Error('Injection'); qb.andWhere(`'{"${normalizeForSearch(tag)}"}' <@ note.tags`); } })); diff --git a/packages/backend/src/server/api/endpoints/reset-db.ts b/packages/backend/src/server/api/endpoints/reset-db.ts index 4ced6d3ff..1d4825f81 100644 --- a/packages/backend/src/server/api/endpoints/reset-db.ts +++ b/packages/backend/src/server/api/endpoints/reset-db.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { private redisClient: Redis.Redis, ) { super(meta, paramDef, async (ps, me) => { - if (process.env.NODE_ENV !== 'test') throw 'NODE_ENV is not a test'; + if (process.env.NODE_ENV !== 'test') throw new Error('NODE_ENV is not a test'); await redisClient.flushdb(); await resetDb(this.db); diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index 6b31e6861..a7bcd859a 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -52,11 +52,7 @@ export class MockResolver extends Resolver { const r = this._rs.get(value); if (!r) { - throw { - name: 'StatusError', - statusCode: 404, - message: 'Not registed for mock', - }; + throw new Error('Not registed for mock'); } const object = JSON.parse(r.content); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 1a4a0b354..22f7d81e4 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -478,7 +478,7 @@ export async function testPaginationConsistency=12'} + /bullmq@3.14.1: + resolution: {integrity: sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==} dependencies: - cron-parser: 4.7.1 - debuglog: 1.0.1 - get-port: 5.1.1 + cron-parser: 4.8.1 + glob: 8.1.0 ioredis: 5.3.2 lodash: 4.17.21 - msgpackr: 1.8.1 + msgpackr: 1.9.2 semver: 7.5.1 - uuid: 8.3.2 + tslib: 2.5.2 + uuid: 9.0.0 transitivePeerDependencies: - supports-color + dev: false /busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} @@ -9306,6 +9302,7 @@ packages: /cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} + dev: false /co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} @@ -9607,11 +9604,12 @@ packages: readable-stream: 3.6.0 dev: false - /cron-parser@4.7.1: - resolution: {integrity: sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==} + /cron-parser@4.8.1: + resolution: {integrity: sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==} engines: {node: '>=12.0.0'} dependencies: - luxon: 3.2.1 + luxon: 3.3.0 + dev: false /cropperjs@2.0.0-beta.2: resolution: {integrity: sha512-jDRSODDGKmi9vp3p/+WXkxMqV/AE+GpSld1U3cHZDRdLy9UykRzurSe8k1dR0TExn45ygCMrv31qkg+K3EeXXw==} @@ -9890,9 +9888,6 @@ packages: ms: 2.1.2 supports-color: 8.1.1 - /debuglog@1.0.1: - resolution: {integrity: sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==} - /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -10084,6 +10079,7 @@ packages: /denque@2.1.0: resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} engines: {node: '>=0.10'} + dev: false /depd@1.1.2: resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} @@ -11753,6 +11749,7 @@ packages: /get-port@5.1.1: resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} engines: {node: '>=8'} + dev: true /get-stream@3.0.0: resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==} @@ -12606,6 +12603,7 @@ packages: standard-as-callback: 2.1.0 transitivePeerDependencies: - supports-color + dev: false /iota-array@1.0.0: resolution: {integrity: sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==} @@ -14288,6 +14286,7 @@ packages: /lodash.defaults@4.2.0: resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + dev: false /lodash.difference@4.5.0: resolution: {integrity: sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==} @@ -14319,6 +14318,7 @@ packages: /lodash.isarguments@3.1.0: resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + dev: false /lodash.isempty@4.4.0: resolution: {integrity: sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==} @@ -14472,9 +14472,10 @@ packages: engines: {node: '>=16.14'} dev: true - /luxon@3.2.1: - resolution: {integrity: sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==} + /luxon@3.3.0: + resolution: {integrity: sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==} engines: {node: '>=12'} + dev: false /lz-string@1.5.0: resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} @@ -14950,24 +14951,27 @@ packages: engines: {node: '>=12.13'} dev: false - /msgpackr-extract@2.2.0: - resolution: {integrity: sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==} + /msgpackr-extract@3.0.2: + resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==} + hasBin: true requiresBuild: true dependencies: - node-gyp-build-optional-packages: 5.0.3 + node-gyp-build-optional-packages: 5.0.7 optionalDependencies: - '@msgpackr-extract/msgpackr-extract-darwin-arm64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-darwin-x64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-arm': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-arm64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-linux-x64': 2.2.0 - '@msgpackr-extract/msgpackr-extract-win32-x64': 2.2.0 + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.2 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2 + dev: false optional: true - /msgpackr@1.8.1: - resolution: {integrity: sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==} + /msgpackr@1.9.2: + resolution: {integrity: sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==} optionalDependencies: - msgpackr-extract: 2.2.0 + msgpackr-extract: 3.0.2 + dev: false /msw-storybook-addon@1.8.0(msw@1.2.1): resolution: {integrity: sha512-dw3vZwqjixmiur0vouRSOax7wPSu9Og2Hspy9JZFHf49bZRjwDiLF0Pfn2NXEkGviYJOJiGxS1ejoTiUwoSg4A==} @@ -15219,8 +15223,10 @@ packages: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 - /node-gyp-build-optional-packages@5.0.3: - resolution: {integrity: sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==} + /node-gyp-build-optional-packages@5.0.7: + resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==} + hasBin: true + dev: false optional: true /node-gyp-build@4.6.0: @@ -17252,6 +17258,7 @@ packages: /redis-errors@1.2.0: resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} engines: {node: '>=4'} + dev: false /redis-info@3.1.0: resolution: {integrity: sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==} @@ -17269,6 +17276,7 @@ packages: engines: {node: '>=4'} dependencies: redis-errors: 1.2.0 + dev: false /redis@4.5.1: resolution: {integrity: sha512-oxXSoIqMJCQVBTfxP6BNTCtDMyh9G6Vi5wjdPdV/sRKkufyZslDqCScSGcOr6XGR/reAWZefz7E4leM31RgdBA==} @@ -18395,6 +18403,7 @@ packages: /standard-as-callback@2.1.0: resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false /start-server-and-test@2.0.0: resolution: {integrity: sha512-UqKLw0mJbfrsG1jcRLTUlvuRi9sjNuUiDOLI42r7R5fA9dsFoywAy9DoLXNYys9B886E4RCKb+qM1Gzu96h7DQ==}