From 1cd59c1ee3ae0166d073b461c106a5cd2f280269 Mon Sep 17 00:00:00 2001 From: ShittyKopper Date: Thu, 4 Jan 2024 18:16:50 +0300 Subject: [PATCH 1/4] feat: initial builtin video thumbnail generator implementation when you disable "cache external media", video thumbnails off of remote instances do not get generated. misskey has a videoThumbnailGenerator config option to point to an external service to make that happen, but they do not provide any kind of implementation (or any documentation beyond a comment on the config file) this provides a video thumbnail generator that uses the same thumbnail generation code path used in local files, providing a quick and dirty solution to instances that want video thumbnails without the need to store external media permanently the eventual goal of this is to be the fallback implementation when that config option is unset. the current implementation is extremely bare bones and performs no caching or any other optimizations whatsoever --- .../backend/src/server/FileServerService.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index e82ef64dc..714dddb16 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -81,6 +81,13 @@ export class FileServerService { .catch(err => this.errorHandler(request, reply, err)); }); + fastify.get<{ + Querystring: { url: string; }; + }>('/proxy/thumbnail.webp', async (request, reply) => { + return await this.videoThumbnailHandler(request, reply) + .catch(err => this.errorHandler(request, reply, err)); + }); + fastify.get<{ Params: { url: string; }; Querystring: { url?: string; }; @@ -371,6 +378,51 @@ export class FileServerService { } } + @bindThis + private async videoThumbnailHandler(request: FastifyRequest<{ Querystring: { url: string; }; }>, reply: FastifyReply) { + const file = await this.getStreamAndTypeFromUrl(request.query.url); + + if (file === '404') { + reply.code(404); + reply.header('Cache-Control', 'max-age=86400'); + return reply.sendFile('/dummy.png', assets); // TODO: return webp + } + + if (file === '204') { + reply.code(204); + reply.header('Cache-Control', 'max-age=86400'); + return; + } + + if (file.file?.thumbnailUrl) { + return await reply.redirect(301, file.file.thumbnailUrl); + } + + if (!file.mime.startsWith('video/')) { + if ('cleanup' in file) { + file.cleanup(); + } + + reply.code(400); + return; + } + + try { + const image = await this.videoProcessingService.generateVideoThumbnail(file.path); + + if ('cleanup' in file) { + file.cleanup(); + } + + reply.header('Content-Type', image.type); + reply.header('Cache-Control', 'max-age=31536000, immutable'); + return image.data; + } catch (e) { + if ('cleanup' in file) file.cleanup(); + throw e; + } + } + @bindThis private async getStreamAndTypeFromUrl(url: string): Promise< { state: 'remote'; fileRole?: 'thumbnail' | 'webpublic' | 'original'; file?: MiDriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; filename: string; } From bc24e6a2942551283d3f93d21a98473f33e76617 Mon Sep 17 00:00:00 2001 From: ShittyKopper Date: Thu, 4 Jan 2024 19:10:20 +0300 Subject: [PATCH 2/4] upd: use builtin thumbnail generator if no thumbnail generator is set --- packages/backend/src/core/VideoProcessingService.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/VideoProcessingService.ts b/packages/backend/src/core/VideoProcessingService.ts index ffb757335..e157160b4 100644 --- a/packages/backend/src/core/VideoProcessingService.ts +++ b/packages/backend/src/core/VideoProcessingService.ts @@ -50,7 +50,14 @@ export class VideoProcessingService { @bindThis public getExternalVideoThumbnailUrl(url: string): string | null { - if (this.config.videoThumbnailGenerator == null) return null; + if (this.config.videoThumbnailGenerator == null) { + return appendQuery( + `${this.config.url}/proxy/thumbnail.webp`, + query({ + url, + }), + ); + } return appendQuery( `${this.config.videoThumbnailGenerator}/thumbnail.webp`, From a3c302e7565bf612535dd61af00fdc9831a85378 Mon Sep 17 00:00:00 2001 From: ShittyKopper Date: Thu, 4 Jan 2024 19:55:02 +0300 Subject: [PATCH 3/4] upd: introduce a separate ./cache dir and cache video thumbnails there --- .gitignore | 1 + docker-compose_example.yml | 1 + .../src/core/InternalStorageService.ts | 27 +++++++++++++++++++ .../backend/src/server/FileServerService.ts | 12 +++++++++ 4 files changed, 41 insertions(+) diff --git a/.gitignore b/.gitignore index 11e69b262..4816375a4 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ api-docs.json *.code-workspace .DS_Store /files +/cache ormconfig.json temp /packages/frontend/src/**/*.stories.ts diff --git a/docker-compose_example.yml b/docker-compose_example.yml index 0a422a2a9..b61daa5eb 100644 --- a/docker-compose_example.yml +++ b/docker-compose_example.yml @@ -20,6 +20,7 @@ services: - shonk volumes: - ./files:/sharkey/files + - ./cache:/sharkey/cache - ./.config:/sharkey/.config:ro redis: diff --git a/packages/backend/src/core/InternalStorageService.ts b/packages/backend/src/core/InternalStorageService.ts index 22129bb34..96d7be2ac 100644 --- a/packages/backend/src/core/InternalStorageService.ts +++ b/packages/backend/src/core/InternalStorageService.ts @@ -16,6 +16,7 @@ const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); const path = Path.resolve(_dirname, '../../../../files'); +const cachePath = Path.resolve(_dirname, '../../../../cache'); @Injectable() export class InternalStorageService { @@ -30,11 +31,26 @@ export class InternalStorageService { return Path.resolve(path, key); } + @bindThis + public resolveCachePath(key: string) { + return Path.resolve(cachePath, key); + } + + @bindThis + public existsCache(key: string) { + return fs.existsSync(this.resolveCachePath(key)); + } + @bindThis public read(key: string) { return fs.createReadStream(this.resolvePath(key)); } + @bindThis + public readCache(key: string) { + return fs.createReadStream(this.resolveCachePath(key)); + } + @bindThis public saveFromPath(key: string, srcPath: string) { fs.mkdirSync(path, { recursive: true }); @@ -49,8 +65,19 @@ export class InternalStorageService { return `${this.config.url}/files/${key}`; } + @bindThis + public saveCacheFromBuffer(key: string, data: Buffer) { + fs.mkdirSync(cachePath, { recursive: true }); + fs.writeFileSync(this.resolveCachePath(key), data); + } + @bindThis public del(key: string) { fs.unlink(this.resolvePath(key), () => {}); } + + @bindThis + public delCache(key: string) { + fs.unlink(this.resolveCachePath(key), () => {}); + } } diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 714dddb16..6659932ed 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -4,6 +4,7 @@ */ import * as fs from 'node:fs'; +import * as crypto from 'node:crypto'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; import { Inject, Injectable } from '@nestjs/common'; @@ -33,6 +34,7 @@ const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); const assets = `${_dirname}/../../server/file/assets/`; +const cacheDir = `${_dirname}/../../../../cache/`; @Injectable() export class FileServerService { @@ -380,6 +382,14 @@ export class FileServerService { @bindThis private async videoThumbnailHandler(request: FastifyRequest<{ Querystring: { url: string; }; }>, reply: FastifyReply) { + const cacheKey = crypto.createHash('md5').update(request.query.url).digest('base64url'); + const cacheFile = `videoThumbnail-${cacheKey}.webp`; + if (this.internalStorageService.existsCache(cacheFile)) { + reply.header('Content-Type', 'image/webp'); + reply.header('Cache-Control', 'max-age=31536000, immutable'); + return reply.sendFile(cacheFile, cacheDir); + } + const file = await this.getStreamAndTypeFromUrl(request.query.url); if (file === '404') { @@ -414,6 +424,8 @@ export class FileServerService { file.cleanup(); } + this.internalStorageService.saveCacheFromBuffer(cacheFile, image.data); + reply.header('Content-Type', image.type); reply.header('Cache-Control', 'max-age=31536000, immutable'); return image.data; From 634259d600fb1a75f88d69a382e34b2ed78a6a5e Mon Sep 17 00:00:00 2001 From: ShittyKopper Date: Thu, 4 Jan 2024 20:52:50 +0300 Subject: [PATCH 4/4] upd: add config option to enable builtin video thumbnail generator --- .config/docker_example.yml | 9 +++++++-- .config/example.yml | 9 +++++++-- packages/backend/src/config.ts | 3 +++ .../backend/src/core/VideoProcessingService.ts | 14 ++++++++------ packages/backend/src/server/FileServerService.ts | 16 +++++++++------- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/.config/docker_example.yml b/.config/docker_example.yml index c6c83a98b..6d82fdcc7 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -106,7 +106,7 @@ redis: # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── -# You can set scope to local (default value) or global +# You can set scope to local (default value) or global # (include notes from remote). #meilisearch: @@ -198,13 +198,18 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com +# Enables the built-in thumbnail generator for remote videos. (default: false) +# Only useful if "Cache remote files" is disabled, and "videoThumbnailGenerator" is unset. +# Without it, remote video files that are not cached will not have any thumbnails. +#enableBuiltinVideoThumbnailGenerator: false + # Sign to ActivityPub GET request (default: true) signToActivityPubGet: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false # For security reasons, uploading attachments from the intranet is prohibited, -# but exceptions can be made from the following settings. Default value is "undefined". +# but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). #allowedPrivateNetworks: [ # '127.0.0.1/32' diff --git a/.config/example.yml b/.config/example.yml index 4aa7757c6..183331ef6 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -118,7 +118,7 @@ redis: # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── -# You can set scope to local (default value) or global +# You can set scope to local (default value) or global # (include notes from remote). #meilisearch: @@ -213,13 +213,18 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com +# Enables the built-in thumbnail generator for remote videos. (default: false) +# Only useful if "Cache remote files" is disabled, and "videoThumbnailGenerator" is unset. +# Without it, remote video files that are not cached will not have any thumbnails. +#enableBuiltinVideoThumbnailGenerator: false + # Sign to ActivityPub GET request (default: true) signToActivityPubGet: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false # For security reasons, uploading attachments from the intranet is prohibited, -# but exceptions can be made from the following settings. Default value is "undefined". +# but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). #allowedPrivateNetworks: [ # '127.0.0.1/32' diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index a550fdc36..137506bb6 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -86,6 +86,7 @@ type Source = { mediaProxy?: string; proxyRemoteFiles?: boolean; videoThumbnailGenerator?: string; + enableBuiltinVideoThumbnailGenerator?: boolean; customMOTD?: string[]; @@ -167,6 +168,7 @@ export type Config = { mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; + enableBuiltinVideoThumbnailGenerator: boolean; redis: RedisOptions & RedisOptionsSource; redisForPubsub: RedisOptions & RedisOptionsSource; redisForJobQueue: RedisOptions & RedisOptionsSource; @@ -272,6 +274,7 @@ export function loadConfig(): Config { videoThumbnailGenerator: config.videoThumbnailGenerator ? config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator : null, + enableBuiltinVideoThumbnailGenerator: config.enableBuiltinVideoThumbnailGenerator ?? false, userAgent: `Misskey/${version} (${config.url})`, clientEntry: clientManifest['src/_boot_.ts'], clientManifestExists: clientManifestExists, diff --git a/packages/backend/src/core/VideoProcessingService.ts b/packages/backend/src/core/VideoProcessingService.ts index e157160b4..c9f706e5c 100644 --- a/packages/backend/src/core/VideoProcessingService.ts +++ b/packages/backend/src/core/VideoProcessingService.ts @@ -51,12 +51,14 @@ export class VideoProcessingService { @bindThis public getExternalVideoThumbnailUrl(url: string): string | null { if (this.config.videoThumbnailGenerator == null) { - return appendQuery( - `${this.config.url}/proxy/thumbnail.webp`, - query({ - url, - }), - ); + if (this.config.enableBuiltinVideoThumbnailGenerator) { + return appendQuery( + `${this.config.url}/proxy/thumbnail.webp`, + query({ url }), + ); + } + + return null; } return appendQuery( diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 6659932ed..3614ea21b 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -83,12 +83,14 @@ export class FileServerService { .catch(err => this.errorHandler(request, reply, err)); }); - fastify.get<{ - Querystring: { url: string; }; - }>('/proxy/thumbnail.webp', async (request, reply) => { - return await this.videoThumbnailHandler(request, reply) - .catch(err => this.errorHandler(request, reply, err)); - }); + if (this.config.enableBuiltinVideoThumbnailGenerator) { + fastify.get<{ + Querystring: { url: string; }; + }>('/proxy/thumbnail.webp', async (request, reply) => { + return await this.videoThumbnailHandler(request, reply) + .catch(err => this.errorHandler(request, reply, err)); + }); + } fastify.get<{ Params: { url: string; }; @@ -395,7 +397,7 @@ export class FileServerService { if (file === '404') { reply.code(404); reply.header('Cache-Control', 'max-age=86400'); - return reply.sendFile('/dummy.png', assets); // TODO: return webp + return reply.sendFile('/dummy.png', assets); } if (file === '204') {