diff --git a/CHANGELOG.md b/CHANGELOG.md index ed7168ace..902322287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ - nsfwjs のモデルロードを排他することで、重複ロードによってメモリ使用量が増加しないように - 連合の配送ジョブのパフォーマンスを向上（ロック機構の見直し、Redisキャッシュの活用） - featuredノートのsignedGet回数を減らしました +- リモートサーバーからのNSFW映像のキャッシュだけを無効化できるオプションを追加 - MeilisearchにIndexするノートの範囲を設定できるように - Fix: Remove Meilisearch index when notes are deleted - Fix: 非英語環境でのPostgreSQLのエラーハンドリングを修正 diff --git a/locales/index.d.ts b/locales/index.d.ts index e3ad4ed00..082cde078 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -159,6 +159,8 @@ export interface Locale { "settingGuide": string; "cacheRemoteFiles": string; "cacheRemoteFilesDescription": string; + "cacheRemoteSensitiveFiles": string; + "cacheRemoteSensitiveFilesDescription": string; "flagAsBot": string; "flagAsBotDescription": string; "flagAsCat": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index c66b42284..ceff2a7cf 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -156,6 +156,8 @@ addEmoji: "絵文字を追加" settingGuide: "おすすめ設定" cacheRemoteFiles: "リモートのファイルをキャッシュする" cacheRemoteFilesDescription: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。サーバーのストレージを節約できますが、サムネイルが生成されないので通信量が増加します。" +cacheRemoteSensitiveFiles: "リモートのセンシティブなファイルをキャッシュする" +cacheRemoteSensitiveFilesDescription: "この設定を無効にすると、リモートのセンシティブなファイルはキャッシュせず直リンクするようになります。" flagAsBot: "Botとして設定" flagAsBotDescription: "このアカウントがプログラムによって運用される場合は、このフラグをオンにします。オンにすると、反応の連鎖を防ぐためのフラグとして他の開発者に役立ったり、Misskeyのシステム上での扱いがBotに合ったものになります。" flagAsCat: "にゃああああああああああああああ！！！！！！！！！！！！" diff --git a/packages/backend/migration/1689102832143-nsfw-cache.js b/packages/backend/migration/1689102832143-nsfw-cache.js new file mode 100644 index 000000000..cdce0dae0 --- /dev/null +++ b/packages/backend/migration/1689102832143-nsfw-cache.js @@ -0,0 +1,11 @@ +export class NsfwCache1689102832143 { + name = 'NsfwCache1689102832143' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "cacheRemoteSensitiveFiles" boolean NOT NULL DEFAULT true`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "cacheRemoteSensitiveFiles"`); + } +} diff --git a/packages/backend/src/core/activitypub/models/ApImageService.ts b/packages/backend/src/core/activitypub/models/ApImageService.ts index 0da312241..1f2984894 100644 --- a/packages/backend/src/core/activitypub/models/ApImageService.ts +++ b/packages/backend/src/core/activitypub/models/ApImageService.ts @@ -1,7 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository } from '@/models/index.js'; -import type { Config } from '@/config.js'; import type { RemoteUser } from '@/models/entities/User.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import { MetaService } from '@/core/MetaService.js'; @@ -20,9 +19,6 @@ export class ApImageService { private logger: Logger; constructor( - @Inject(DI.config) - private config: Config, - @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, @@ -47,7 +43,7 @@ export class ApImageService { const image = await this.apResolverService.createResolver().resolve(value); if (image.url == null) { - throw new Error('invalid image: url not privided'); + throw new Error('invalid image: url not provided'); } if (typeof image.url !== 'string') { @@ -62,12 +58,17 @@ export class ApImageService { const instance = await this.metaService.fetch(); + // Cache if remote file cache is on AND either + // 1. remote sensitive file is also on + // 2. or the image is not sensitive + const shouldBeCached = instance.cacheRemoteFiles && (instance.cacheRemoteSensitiveFiles || !image.sensitive); + const file = await this.driveService.uploadFromUrl({ url: image.url, user: actor, uri: image.url, sensitive: image.sensitive, - isLink: !instance.cacheRemoteFiles, + isLink: !shouldBeCached, comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH), }); if (!file.isLink || file.url === image.url) return file; diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index a251c0b31..7bb1b6771 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -1,7 +1,6 @@ import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm'; import { id } from '../id.js'; import { User } from './User.js'; -import type { Clip } from './Clip.js'; @Entity() export class Meta { @@ -126,6 +125,11 @@ export class Meta { }) public cacheRemoteFiles: boolean; + @Column('boolean', { + default: true, + }) + public cacheRemoteSensitiveFiles: boolean; + @Column({ ...id(), nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 28aec7a09..084bdb598 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -1,5 +1,4 @@ import { Inject, Injectable } from '@nestjs/common'; -import { MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { MetaService } from '@/core/MetaService.js'; import type { Config } from '@/config.js'; @@ -20,6 +19,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + cacheRemoteSensitiveFiles: { + type: 'boolean', + optional: false, nullable: false, + }, emailRequiredForSignup: { type: 'boolean', optional: false, nullable: false, @@ -332,6 +335,7 @@ export default class extends Endpoint { enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, cacheRemoteFiles: instance.cacheRemoteFiles, + cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, pinnedUsers: instance.pinnedUsers, hiddenTags: instance.hiddenTags, blockedHosts: instance.blockedHosts, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 0b20b058f..144360a92 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -43,6 +43,7 @@ export const paramDef = { defaultLightTheme: { type: 'string', nullable: true }, defaultDarkTheme: { type: 'string', nullable: true }, cacheRemoteFiles: { type: 'boolean' }, + cacheRemoteSensitiveFiles: { type: 'boolean' }, emailRequiredForSignup: { type: 'boolean' }, enableHcaptcha: { type: 'boolean' }, hcaptchaSiteKey: { type: 'string', nullable: true }, @@ -193,6 +194,10 @@ export default class extends Endpoint { set.cacheRemoteFiles = ps.cacheRemoteFiles; } + if (ps.cacheRemoteSensitiveFiles !== undefined) { + set.cacheRemoteSensitiveFiles = ps.cacheRemoteSensitiveFiles; + } + if (ps.emailRequiredForSignup !== undefined) { set.emailRequiredForSignup = ps.emailRequiredForSignup; } diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 915a1e54f..3d0146e31 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -83,6 +83,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + cacheRemoteSensitiveFiles: { + type: 'boolean', + optional: false, nullable: false, + }, emailRequiredForSignup: { type: 'boolean', optional: false, nullable: false, @@ -272,7 +276,7 @@ export default class extends Endpoint { .orWhere('ads.dayOfWeek = 0'); })) .getMany(); - + const response: any = { maintainerName: instance.maintainerName, maintainerEmail: instance.maintainerEmail, @@ -329,6 +333,7 @@ export default class extends Endpoint { ...(ps.detail ? { cacheRemoteFiles: instance.cacheRemoteFiles, + cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, requireSetup: (await this.usersRepository.countBy({ host: IsNull(), })) === 0, diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 02b900da9..78b916c11 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -4,6 +4,7 @@ import * as assert from 'assert'; import { Test } from '@nestjs/testing'; import { jest } from '@jest/globals'; +import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; @@ -11,9 +12,12 @@ import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; -import type { IActor, ICollection, IPost } from '@/core/activitypub/type.js'; -import { Note } from '@/models/index.js'; +import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js'; +import { Meta, Note } from '@/models/index.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; +import { DownloadService } from '@/core/DownloadService.js'; +import { MetaService } from '@/core/MetaService.js'; +import type { RemoteUser } from '@/models/entities/User.js'; import { MockResolver } from '../misc/mock-resolver.js'; const host = 'https://host1.test'; @@ -63,16 +67,47 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe }; } +async function createRandomRemoteUser( + resolver: MockResolver, + personService: ApPersonService, +): Promise { + const actor = createRandomActor(); + resolver.register(actor.id, actor); + + return await personService.createPerson(actor.id, resolver); +} + describe('ActivityPub', () => { + let imageService: ApImageService; let noteService: ApNoteService; let personService: ApPersonService; let rendererService: ApRendererService; let resolver: MockResolver; + const metaInitial = { + cacheRemoteFiles: true, + cacheRemoteSensitiveFiles: true, + blockedHosts: [] as string[], + sensitiveWords: [] as string[], + } as Meta; + let meta = metaInitial; + beforeAll(async () => { const app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], - }).compile(); + }) + .overrideProvider(DownloadService).useValue({ + async downloadUrl(): Promise<{ filename: string }> { + return { + filename: 'dummy.tmp', + }; + }, + }) + .overrideProvider(MetaService).useValue({ + async fetch(): Promise { + return meta; + }, + }).compile(); await app.init(); app.enableShutdownHooks(); @@ -80,6 +115,7 @@ describe('ActivityPub', () => { noteService = app.get(ApNoteService); personService = app.get(ApPersonService); rendererService = app.get(ApRendererService); + imageService = app.get(ApImageService); resolver = new MockResolver(await app.resolve(LoggerService)); // Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error @@ -219,4 +255,91 @@ describe('ActivityPub', () => { assert.strictEqual(note.uri, actor2Note.id); }); }); + + describe('Images', () => { + test('Create images', async () => { + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(!driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(!sensitiveDriveFile.isLink); + }); + + test('cacheRemoteFiles=false disables caching', async () => { + meta = { ...metaInitial, cacheRemoteFiles: false }; + + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(sensitiveDriveFile.isLink); + }); + + test('cacheRemoteSensitiveFiles=false only affects sensitive files', async () => { + meta = { ...metaInitial, cacheRemoteSensitiveFiles: false }; + + const imageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/foo.png', + name: '', + }; + const driveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + imageObject, + ); + assert.ok(!driveFile.isLink); + + const sensitiveImageObject: IApDocument = { + type: 'Document', + mediaType: 'image/png', + url: 'http://host1.test/bar.png', + name: '', + sensitive: true, + }; + const sensitiveDriveFile = await imageService.createImage( + await createRandomRemoteUser(resolver, personService), + sensitiveImageObject, + ); + assert.ok(sensitiveDriveFile.isLink); + }); + }); }); diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index 4c2fe46f2..bd57c0618 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -37,6 +37,13 @@ + + @@ -104,7 +111,6 @@ import { fetchInstance } from '@/instance'; import { i18n } from '@/i18n'; import { definePageMetadata } from '@/scripts/page-metadata'; import MkButton from '@/components/MkButton.vue'; -import MkColorInput from '@/components/MkColorInput.vue'; let name: string | null = $ref(null); let description: string | null = $ref(null); @@ -112,13 +118,14 @@ let maintainerName: string | null = $ref(null); let maintainerEmail: string | null = $ref(null); let pinnedUsers: string = $ref(''); let cacheRemoteFiles: boolean = $ref(false); +let cacheRemoteSensitiveFiles: boolean = $ref(false); let enableServiceWorker: boolean = $ref(false); let swPublicKey: any = $ref(null); let swPrivateKey: any = $ref(null); let deeplAuthKey: string = $ref(''); let deeplIsPro: boolean = $ref(false); -async function init() { +async function init(): Promise { const meta = await os.api('admin/meta'); name = meta.name; description = meta.description; @@ -126,6 +133,7 @@ async function init() { maintainerEmail = meta.maintainerEmail; pinnedUsers = meta.pinnedUsers.join('\n'); cacheRemoteFiles = meta.cacheRemoteFiles; + cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles; enableServiceWorker = meta.enableServiceWorker; swPublicKey = meta.swPublickey; swPrivateKey = meta.swPrivateKey; @@ -133,7 +141,7 @@ async function init() { deeplIsPro = meta.deeplIsPro; } -function save() { +function save(): void { os.apiWithDialog('admin/update-meta', { name, description, @@ -141,6 +149,7 @@ function save() { maintainerEmail, pinnedUsers: pinnedUsers.split('\n'), cacheRemoteFiles, + cacheRemoteSensitiveFiles, enableServiceWorker, swPublicKey, swPrivateKey, diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index 851b21abc..65d6e2e0b 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -271,6 +271,7 @@ type DetailedInstanceMetadata = LiteInstanceMetadata & { pinnedPages: string[]; pinnedClipId: string | null; cacheRemoteFiles: boolean; + cacheRemoteSensitiveFiles: boolean; requireSetup: boolean; proxyAccountName: string | null; features: Record; @@ -327,6 +328,10 @@ export type Endpoints = { req: TODO; res: TODO; }; + 'admin/meta': { + req: TODO; + res: TODO; + }; 'admin/reset-password': { req: TODO; res: TODO; @@ -2805,7 +2810,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u // // src/api.types.ts:16:32 - (ae-forgotten-export) The symbol "TODO" needs to be exported by the entry point index.d.ts // src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts -// src/api.types.ts:628:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts +// src/api.types.ts:629:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts // src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/packages/misskey-js/src/api.types.ts b/packages/misskey-js/src/api.types.ts index ed12be4a0..343977f4b 100644 --- a/packages/misskey-js/src/api.types.ts +++ b/packages/misskey-js/src/api.types.ts @@ -20,6 +20,7 @@ export type Endpoints = { 'admin/get-table-stats': { req: TODO; res: TODO; }; 'admin/invite': { req: TODO; res: TODO; }; 'admin/logs': { req: TODO; res: TODO; }; + 'admin/meta': { req: TODO; res: TODO; }; 'admin/reset-password': { req: TODO; res: TODO; }; 'admin/resolve-abuse-user-report': { req: TODO; res: TODO; }; 'admin/resync-chart': { req: TODO; res: TODO; }; diff --git a/packages/misskey-js/src/entities.ts b/packages/misskey-js/src/entities.ts index 9d8fd6a49..ea85bedf6 100644 --- a/packages/misskey-js/src/entities.ts +++ b/packages/misskey-js/src/entities.ts @@ -338,6 +338,7 @@ export type DetailedInstanceMetadata = LiteInstanceMetadata & { pinnedPages: string[]; pinnedClipId: string | null; cacheRemoteFiles: boolean; + cacheRemoteSensitiveFiles: boolean; requireSetup: boolean; proxyAccountName: string | null; features: Record;