merge: Built-in video thumbnail generator (for remote video thumbnails without needing "Cache remote media" enabled) (!310)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/310

.config/docker_example.yml

@@ -163,7 +163,7 @@ redis:
 #   ┌───────────────────────────┐
 #───┘ MeiliSearch configuration └─────────────────────────────
 
-# You can set scope to local (default value) or global 
+# You can set scope to local (default value) or global
 # (include notes from remote).
 
 #meilisearch:
@@ -255,13 +255,18 @@ proxyRemoteFiles: true
 #   https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4
 #videoThumbnailGenerator: https://example.com
 
+# Enables the built-in thumbnail generator for remote videos. (default: false)
+# Only useful if "Cache remote files" is disabled, and "videoThumbnailGenerator" is unset.
+# Without it, remote video files that are not cached will not have any thumbnails.
+#enableBuiltinVideoThumbnailGenerator: false
+
 # Sign to ActivityPub GET request (default: true)
 signToActivityPubGet: true
 # check that inbound ActivityPub GET requests are signed ("authorized fetch")
 checkActivityPubGetSignature: false
 
 # For security reasons, uploading attachments from the intranet is prohibited,
-# but exceptions can be made from the following settings. Default value is "undefined". 
+# but exceptions can be made from the following settings. Default value is "undefined".
 # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)).
 #allowedPrivateNetworks: [
 #  '127.0.0.1/32'

.config/example.yml

@@ -270,6 +270,11 @@ proxyRemoteFiles: true
 #   https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4
 #videoThumbnailGenerator: https://example.com
 
+# Enables the built-in thumbnail generator for remote videos. (default: false)
+# Only useful if "Cache remote files" is disabled, and "videoThumbnailGenerator" is unset.
+# Without it, remote video files that are not cached will not have any thumbnails.
+#enableBuiltinVideoThumbnailGenerator: false
+
 # Sign to ActivityPub GET request (default: true)
 signToActivityPubGet: true
 # check that inbound ActivityPub GET requests are signed ("authorized fetch")

.gitignore

@@ -55,6 +55,7 @@ api-docs.json
 *.code-workspace
 .DS_Store
 /files
+/cache
 ormconfig.json
 temp
 /packages/frontend/src/**/*.stories.ts

.gitlab-ci.yml

@@ -11,7 +11,7 @@ testCommit:
   variables:
     POSTGRES_PASSWORD: ci
   script:
-    - apt-get update && apt-get install -y git wget curl build-essential python3 
+    - apt-get update && apt-get install -y git wget curl build-essential python3
     - cp .config/ci.yml .config/default.yml
     - corepack enable
     - corepack prepare pnpm@latest --activate
@@ -55,6 +55,8 @@ getImageTag:
   only:
     - stable
     - develop
+    - tags
+
 buildDocker:
   stage: deploy
   needs:
@@ -78,6 +80,8 @@ buildDocker:
   only:
     - stable
     - develop
+    - tags
+
 mergeManifests:
   stage: deploy
   needs:
@@ -103,3 +107,4 @@ mergeManifests:
   only:
     - stable
     - develop
+    - tags

docker-compose_example.yml

@@ -21,6 +21,7 @@ services:
       - shonk
     volumes:
       - ./files:/sharkey/files
+      - ./cache:/sharkey/cache
       - ./.config:/sharkey/.config:ro
 
   redis:

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "sharkey",
-	"version": "2024.3.1",
+	"version": "2024.3.2-devel",
 	"codename": "shonk",
 	"repository": {
 		"type": "git",

packages/backend/package.json

@@ -172,7 +172,7 @@
 		"stringz": "2.1.0",
 		"systeminformation": "5.22.0",
 		"tinycolor2": "1.6.0",
-		"tmp": "0.2.2",
+		"tmp": "0.2.3",
 		"tsc-alias": "1.8.8",
 		"tsconfig-paths": "4.2.0",
 		"typeorm": "0.3.20",

packages/backend/src/config.ts

@@ -88,6 +88,7 @@ type Source = {
 	mediaProxy?: string;
 	proxyRemoteFiles?: boolean;
 	videoThumbnailGenerator?: string;
+	enableBuiltinVideoThumbnailGenerator?: boolean;
 
 	customMOTD?: string[];
 
@@ -170,6 +171,7 @@ export type Config = {
 	mediaProxy: string;
 	externalMediaProxyEnabled: boolean;
 	videoThumbnailGenerator: string | null;
+	enableBuiltinVideoThumbnailGenerator: boolean;
 	redis: RedisOptions & RedisOptionsSource;
 	redisForPubsub: RedisOptions & RedisOptionsSource;
 	redisForJobQueue: RedisOptions & RedisOptionsSource;
@@ -276,6 +278,7 @@ export function loadConfig(): Config {
 		videoThumbnailGenerator: config.videoThumbnailGenerator ?
 			config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator
 			: null,
+		enableBuiltinVideoThumbnailGenerator: config.enableBuiltinVideoThumbnailGenerator ?? false,
 		userAgent: `Misskey/${version} (${config.url})`,
 		clientEntry: clientManifest['src/_boot_.ts'],
 		clientManifestExists: clientManifestExists,

packages/backend/src/core/InternalStorageService.ts

@@ -16,6 +16,7 @@ const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
 
 const path = Path.resolve(_dirname, '../../../../files');
+const cachePath = Path.resolve(_dirname, '../../../../cache');
 
 @Injectable()
 export class InternalStorageService {
@@ -30,11 +31,26 @@ export class InternalStorageService {
 		return Path.resolve(path, key);
 	}
 
+	@bindThis
+	public resolveCachePath(key: string) {
+		return Path.resolve(cachePath, key);
+	}
+
+	@bindThis
+	public existsCache(key: string) {
+		return fs.existsSync(this.resolveCachePath(key));
+	}
+
 	@bindThis
 	public read(key: string) {
 		return fs.createReadStream(this.resolvePath(key));
 	}
 
+	@bindThis
+	public readCache(key: string) {
+		return fs.createReadStream(this.resolveCachePath(key));
+	}
+
 	@bindThis
 	public saveFromPath(key: string, srcPath: string) {
 		fs.mkdirSync(path, { recursive: true });
@@ -49,8 +65,19 @@ export class InternalStorageService {
 		return `${this.config.url}/files/${key}`;
 	}
 
+	@bindThis
+	public saveCacheFromBuffer(key: string, data: Buffer) {
+		fs.mkdirSync(cachePath, { recursive: true });
+		fs.writeFileSync(this.resolveCachePath(key), data);
+	}
+
 	@bindThis
 	public del(key: string) {
 		fs.unlink(this.resolvePath(key), () => {});
 	}
+
+	@bindThis
+	public delCache(key: string) {
+		fs.unlink(this.resolveCachePath(key), () => {});
+	}
 }

packages/backend/src/core/VideoProcessingService.ts

@@ -50,7 +50,16 @@ export class VideoProcessingService {
 
 	@bindThis
 	public getExternalVideoThumbnailUrl(url: string): string | null {
-		if (this.config.videoThumbnailGenerator == null) return null;
+		if (this.config.videoThumbnailGenerator == null) {
+			if (this.config.enableBuiltinVideoThumbnailGenerator) {
+				return appendQuery(
+					`${this.config.url}/proxy/thumbnail.webp`,
+					query({ url }),
+				);
+			}
+
+			return null;
+		}
 
 		return appendQuery(
 			`${this.config.videoThumbnailGenerator}/thumbnail.webp`,

packages/backend/src/queue/processors/ImportNotesProcessorService.ts

@@ -421,7 +421,7 @@ export class ImportNotesProcessorService {
 					if (file.name) {
 						this.driveService.updateFile(exists, { comment: file.name }, user);
 					}
-					
+
 					files.push(exists);
 				}
 			}

packages/backend/src/server/FileServerService.ts

@@ -4,6 +4,7 @@
  */
 
 import * as fs from 'node:fs';
+import * as crypto from 'node:crypto';
 import { fileURLToPath } from 'node:url';
 import { dirname } from 'node:path';
 import { Inject, Injectable } from '@nestjs/common';
@@ -34,6 +35,7 @@ const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
 
 const assets = `${_dirname}/../../server/file/assets/`;
+const cacheDir = `${_dirname}/../../../../cache/`;
 
 @Injectable()
 export class FileServerService {
@@ -85,6 +87,15 @@ export class FileServerService {
 			done();
 		});
 
+		if (this.config.enableBuiltinVideoThumbnailGenerator) {
+			fastify.get<{
+				Querystring: { url: string; };
+			}>('/proxy/thumbnail.webp', async (request, reply) => {
+				return await this.videoThumbnailHandler(request, reply)
+					.catch(err => this.errorHandler(request, reply, err));
+			});
+		}
+
 		fastify.get<{
 			Params: { url: string; };
 			Querystring: { url?: string; };
@@ -192,6 +203,7 @@ export class FileServerService {
 						reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
 						reply.header('Accept-Ranges', 'bytes');
 						reply.header('Content-Length', chunksize);
+						reply.code(206);
 					} else {
 						image = {
 							data: fs.createReadStream(file.path),
@@ -261,7 +273,6 @@ export class FileServerService {
 					const parts = range.replace(/bytes=/, '').split('-');
 					const start = parseInt(parts[0], 10);
 					let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
-					console.log(end);
 					if (end > file.file.size) {
 						end = file.file.size - 1;
 					}
@@ -431,6 +442,7 @@ export class FileServerService {
 					reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
 					reply.header('Accept-Ranges', 'bytes');
 					reply.header('Content-Length', chunksize);
+					reply.code(206);
 				} else {
 					image = {
 						data: fs.createReadStream(file.path),
@@ -466,6 +478,61 @@ export class FileServerService {
 		}
 	}
 
+	@bindThis
+	private async videoThumbnailHandler(request: FastifyRequest<{ Querystring: { url: string; }; }>, reply: FastifyReply) {
+		const cacheKey = crypto.createHash('md5').update(request.query.url).digest('base64url');
+		const cacheFile = `videoThumbnail-${cacheKey}.webp`;
+		if (this.internalStorageService.existsCache(cacheFile)) {
+			reply.header('Content-Type', 'image/webp');
+			reply.header('Cache-Control', 'max-age=31536000, immutable');
+			return reply.sendFile(cacheFile, cacheDir);
+		}
+
+		const file = await this.getStreamAndTypeFromUrl(request.query.url);
+
+		if (file === '404') {
+			reply.code(404);
+			reply.header('Cache-Control', 'max-age=86400');
+			return reply.sendFile('/dummy.png', assets);
+		}
+
+		if (file === '204') {
+			reply.code(204);
+			reply.header('Cache-Control', 'max-age=86400');
+			return;
+		}
+
+		if (file.file?.thumbnailUrl) {
+			return await reply.redirect(301, file.file.thumbnailUrl);
+		}
+
+		if (!file.mime.startsWith('video/')) {
+			if ('cleanup' in file) {
+				file.cleanup();
+			}
+
+			reply.code(400);
+			return;
+		}
+
+		try {
+			const image = await this.videoProcessingService.generateVideoThumbnail(file.path);
+
+			if ('cleanup' in file) {
+				file.cleanup();
+			}
+
+			this.internalStorageService.saveCacheFromBuffer(cacheFile, image.data);
+
+			reply.header('Content-Type', image.type);
+			reply.header('Cache-Control', 'max-age=31536000, immutable');
+			return image.data;
+		} catch (e) {
+			if ('cleanup' in file) file.cleanup();
+			throw e;
+		}
+	}
+
 	@bindThis
 	private async getStreamAndTypeFromUrl(url: string): Promise<
 		{ state: 'remote'; fileRole?: 'thumbnail' | 'webpublic' | 'original'; file?: MiDriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; filename: string; }
@@ -527,6 +594,9 @@ export class FileServerService {
 		if (!file.storedInternal) {
 			if (!(file.isLink && file.uri)) return '204';
 			const result = await this.downloadAndDetectTypeFromUrl(file.uri);
+			if (!file.size) {
+				file.size = (await fs.promises.stat(result.path)).size;
+			}
 			return {
 				...result,
 				url: file.uri,

packages/backend/src/server/web/views/note.pug

@@ -5,8 +5,8 @@ block vars
 	- const title = user.name ? `${user.name} (@${user.username})` : `@${user.username}`;
 	- const url = `${config.url}/notes/${note.id}`;
 	- const isRenote = note.renote && note.text == null && note.fileIds.length == 0 && note.poll == null;
-	- const images = (note.files || []).filter(file => file.type.startsWith('image/') && !file.isSensitive)
-	- const videos = (note.files || []).filter(file => file.type.startsWith('video/') && !file.isSensitive)
+	- const images = note.cw ? [] : (note.files || []).filter(file => file.type.startsWith('image/') && !file.isSensitive)
+	- const videos = note.cw ? [] : (note.files || []).filter(file => file.type.startsWith('video/') && !file.isSensitive)
 
 block title
 	= `${title} | ${instanceName}`

pnpm-lock.yaml

@@ -392,8 +392,8 @@ importers:
         specifier: 1.6.0
         version: 1.6.0
       tmp:
-        specifier: 0.2.2
-        version: 0.2.2
+        specifier: 0.2.3
+        version: 0.2.3
       tsc-alias:
         specifier: 1.8.8
         version: 1.8.8
@@ -18813,6 +18813,12 @@ packages:
     engines: {node: '>=14'}
     dependencies:
       rimraf: 5.0.5
+    dev: true
+
+  /tmp@0.2.3:
+    resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==}
+    engines: {node: '>=14.14'}
+    dev: false
 
   /tmpl@1.0.5:
     resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==}