merge: Built-in video thumbnail generator (for remote video thumbnails without needing "Cache remote media" enabled) (!310)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/310

.config/docker_example.yml

@@ -255,6 +255,11 @@ proxyRemoteFiles: true
 #   https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4
 #videoThumbnailGenerator: https://example.com
 
+# Enables the built-in thumbnail generator for remote videos. (default: false)
+# Only useful if "Cache remote files" is disabled, and "videoThumbnailGenerator" is unset.
+# Without it, remote video files that are not cached will not have any thumbnails.
+#enableBuiltinVideoThumbnailGenerator: false
+
 # Sign to ActivityPub GET request (default: true)
 signToActivityPubGet: true
 # check that inbound ActivityPub GET requests are signed ("authorized fetch")

.config/example.yml

@@ -270,6 +270,11 @@ proxyRemoteFiles: true
 #   https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4
 #videoThumbnailGenerator: https://example.com
 
+# Enables the built-in thumbnail generator for remote videos. (default: false)
+# Only useful if "Cache remote files" is disabled, and "videoThumbnailGenerator" is unset.
+# Without it, remote video files that are not cached will not have any thumbnails.
+#enableBuiltinVideoThumbnailGenerator: false
+
 # Sign to ActivityPub GET request (default: true)
 signToActivityPubGet: true
 # check that inbound ActivityPub GET requests are signed ("authorized fetch")

.gitignore

@@ -55,6 +55,7 @@ api-docs.json
 *.code-workspace
 .DS_Store
 /files
+/cache
 ormconfig.json
 temp
 /packages/frontend/src/**/*.stories.ts

docker-compose_example.yml

@@ -21,6 +21,7 @@ services:
       - shonk
     volumes:
       - ./files:/sharkey/files
+      - ./cache:/sharkey/cache
       - ./.config:/sharkey/.config:ro
 
   redis:

packages/backend/src/config.ts

@@ -88,6 +88,7 @@ type Source = {
 	mediaProxy?: string;
 	proxyRemoteFiles?: boolean;
 	videoThumbnailGenerator?: string;
+	enableBuiltinVideoThumbnailGenerator?: boolean;
 
 	customMOTD?: string[];
 
@@ -170,6 +171,7 @@ export type Config = {
 	mediaProxy: string;
 	externalMediaProxyEnabled: boolean;
 	videoThumbnailGenerator: string | null;
+	enableBuiltinVideoThumbnailGenerator: boolean;
 	redis: RedisOptions & RedisOptionsSource;
 	redisForPubsub: RedisOptions & RedisOptionsSource;
 	redisForJobQueue: RedisOptions & RedisOptionsSource;
@@ -276,6 +278,7 @@ export function loadConfig(): Config {
 		videoThumbnailGenerator: config.videoThumbnailGenerator ?
 			config.videoThumbnailGenerator.endsWith('/') ? config.videoThumbnailGenerator.substring(0, config.videoThumbnailGenerator.length - 1) : config.videoThumbnailGenerator
 			: null,
+		enableBuiltinVideoThumbnailGenerator: config.enableBuiltinVideoThumbnailGenerator ?? false,
 		userAgent: `Misskey/${version} (${config.url})`,
 		clientEntry: clientManifest['src/_boot_.ts'],
 		clientManifestExists: clientManifestExists,

packages/backend/src/core/InternalStorageService.ts

@@ -16,6 +16,7 @@ const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
 
 const path = Path.resolve(_dirname, '../../../../files');
+const cachePath = Path.resolve(_dirname, '../../../../cache');
 
 @Injectable()
 export class InternalStorageService {
@@ -30,11 +31,26 @@ export class InternalStorageService {
 		return Path.resolve(path, key);
 	}
 
+	@bindThis
+	public resolveCachePath(key: string) {
+		return Path.resolve(cachePath, key);
+	}
+
+	@bindThis
+	public existsCache(key: string) {
+		return fs.existsSync(this.resolveCachePath(key));
+	}
+
 	@bindThis
 	public read(key: string) {
 		return fs.createReadStream(this.resolvePath(key));
 	}
 
+	@bindThis
+	public readCache(key: string) {
+		return fs.createReadStream(this.resolveCachePath(key));
+	}
+
 	@bindThis
 	public saveFromPath(key: string, srcPath: string) {
 		fs.mkdirSync(path, { recursive: true });
@@ -49,8 +65,19 @@ export class InternalStorageService {
 		return `${this.config.url}/files/${key}`;
 	}
 
+	@bindThis
+	public saveCacheFromBuffer(key: string, data: Buffer) {
+		fs.mkdirSync(cachePath, { recursive: true });
+		fs.writeFileSync(this.resolveCachePath(key), data);
+	}
+
 	@bindThis
 	public del(key: string) {
 		fs.unlink(this.resolvePath(key), () => {});
 	}
+
+	@bindThis
+	public delCache(key: string) {
+		fs.unlink(this.resolveCachePath(key), () => {});
+	}
 }

packages/backend/src/core/VideoProcessingService.ts

@@ -50,7 +50,16 @@ export class VideoProcessingService {
 
 	@bindThis
 	public getExternalVideoThumbnailUrl(url: string): string | null {
-		if (this.config.videoThumbnailGenerator == null) return null;
+		if (this.config.videoThumbnailGenerator == null) {
+			if (this.config.enableBuiltinVideoThumbnailGenerator) {
+				return appendQuery(
+					`${this.config.url}/proxy/thumbnail.webp`,
+					query({ url }),
+				);
+			}
+
+			return null;
+		}
 
 		return appendQuery(
 			`${this.config.videoThumbnailGenerator}/thumbnail.webp`,

packages/backend/src/server/FileServerService.ts

@@ -4,6 +4,7 @@
  */
 
 import * as fs from 'node:fs';
+import * as crypto from 'node:crypto';
 import { fileURLToPath } from 'node:url';
 import { dirname } from 'node:path';
 import { Inject, Injectable } from '@nestjs/common';
@@ -34,6 +35,7 @@ const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
 
 const assets = `${_dirname}/../../server/file/assets/`;
+const cacheDir = `${_dirname}/../../../../cache/`;
 
 @Injectable()
 export class FileServerService {
@@ -85,6 +87,15 @@ export class FileServerService {
 			done();
 		});
 
+		if (this.config.enableBuiltinVideoThumbnailGenerator) {
+			fastify.get<{
+				Querystring: { url: string; };
+			}>('/proxy/thumbnail.webp', async (request, reply) => {
+				return await this.videoThumbnailHandler(request, reply)
+					.catch(err => this.errorHandler(request, reply, err));
+			});
+		}
+
 		fastify.get<{
 			Params: { url: string; };
 			Querystring: { url?: string; };
@@ -466,6 +477,61 @@ export class FileServerService {
 		}
 	}
 
+	@bindThis
+	private async videoThumbnailHandler(request: FastifyRequest<{ Querystring: { url: string; }; }>, reply: FastifyReply) {
+		const cacheKey = crypto.createHash('md5').update(request.query.url).digest('base64url');
+		const cacheFile = `videoThumbnail-${cacheKey}.webp`;
+		if (this.internalStorageService.existsCache(cacheFile)) {
+			reply.header('Content-Type', 'image/webp');
+			reply.header('Cache-Control', 'max-age=31536000, immutable');
+			return reply.sendFile(cacheFile, cacheDir);
+		}
+
+		const file = await this.getStreamAndTypeFromUrl(request.query.url);
+
+		if (file === '404') {
+			reply.code(404);
+			reply.header('Cache-Control', 'max-age=86400');
+			return reply.sendFile('/dummy.png', assets);
+		}
+
+		if (file === '204') {
+			reply.code(204);
+			reply.header('Cache-Control', 'max-age=86400');
+			return;
+		}
+
+		if (file.file?.thumbnailUrl) {
+			return await reply.redirect(301, file.file.thumbnailUrl);
+		}
+
+		if (!file.mime.startsWith('video/')) {
+			if ('cleanup' in file) {
+				file.cleanup();
+			}
+
+			reply.code(400);
+			return;
+		}
+
+		try {
+			const image = await this.videoProcessingService.generateVideoThumbnail(file.path);
+
+			if ('cleanup' in file) {
+				file.cleanup();
+			}
+
+			this.internalStorageService.saveCacheFromBuffer(cacheFile, image.data);
+
+			reply.header('Content-Type', image.type);
+			reply.header('Cache-Control', 'max-age=31536000, immutable');
+			return image.data;
+		} catch (e) {
+			if ('cleanup' in file) file.cleanup();
+			throw e;
+		}
+	}
+
 	@bindThis
 	private async getStreamAndTypeFromUrl(url: string): Promise<
 		{ state: 'remote'; fileRole?: 'thumbnail' | 'webpublic' | 'original'; file?: MiDriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; filename: string; }

packages/backend/src/server/api/mastodon/converters.ts

@@ -279,7 +279,8 @@ export class MastoConverters {
 			emoji_reactions: status.emoji_reactions,
 			bookmarked: false,
 			quote: isQuote ? await this.convertReblog(status.reblog) : false,
-			edited_at: note.updatedAt?.toISOString(),
+			// optional chaining cannot be used, as it evaluates to undefined, not null
+			edited_at: note.updatedAt ? note.updatedAt.toISOString() : null,
 		});
 	}
 }

packages/backend/src/server/web/views/base.pug

@@ -43,7 +43,6 @@ html
 		link(rel='stylesheet' href='/assets/phosphor-icons/bold/style.css')
 		link(rel='stylesheet' href='/static-assets/fonts/sharkey-icons/style.css')
 		link(rel='modulepreload' href=`/vite/${clientEntry.file}`)
-		script(src='/client-assets/libopenmpt.js')
 
 		if !config.clientManifestExists
 				script(type="module" src="/vite/@vite/client")
@@ -73,7 +72,6 @@ html
 		script.
 			var VERSION = "#{version}";
 			var CLIENT_ENTRY = "#{clientEntry.file}";
-			window.libopenmpt = window.Module;
 
 		script(type='application/json' id='misskey_meta' data-generated-at=now)
 			!= metaJson

packages/frontend/src/scripts/chiptune2.ts

@@ -1,9 +1,12 @@
-/* global libopenmpt UTF8ToString writeAsciiToMemory */
+// @ts-nocheck
 /* eslint-disable */
 
 const ChiptuneAudioContext = window.AudioContext || window.webkitAudioContext;
 
-export function ChiptuneJsConfig (repeatCount: number, context: AudioContext) {
+let libopenmpt
+let libopenmptLoadPromise
+
+export function ChiptuneJsConfig (repeatCount?: number, context?: AudioContext) {
 	this.repeatCount = repeatCount;
 	this.context = context;
 }
@@ -20,6 +23,28 @@ export function ChiptuneJsPlayer (config: object) {
 	this.volume = 1;
 }
 
+ChiptuneJsPlayer.prototype.initialize = function() {
+	if (libopenmptLoadPromise) return libopenmptLoadPromise;
+	if (libopenmpt) return Promise.resolve();
+
+	libopenmptLoadPromise = new Promise(async (resolve, reject) => {
+		try {
+			const { Module } = await import('./libopenmpt/libopenmpt.js');
+			await new Promise((resolve) => {
+				Module['onRuntimeInitialized'] = resolve;
+			})
+			libopenmpt = Module;
+			resolve()
+		} catch (e) {
+			reject(e)
+		} finally {
+			libopenmptLoadPromise = undefined;
+		}
+	})
+
+	return libopenmptLoadPromise;
+}
+
 ChiptuneJsPlayer.prototype.constructor = ChiptuneJsPlayer;
 
 ChiptuneJsPlayer.prototype.fireEvent = function (eventName: string, response) {
@@ -61,12 +86,12 @@ ChiptuneJsPlayer.prototype.seek = function (position: number) {
 
 ChiptuneJsPlayer.prototype.metadata = function () {
 	const data = {};
-	const keys = UTF8ToString(libopenmpt._openmpt_module_get_metadata_keys(this.currentPlayingNode.modulePtr)).split(';');
+	const keys = libopenmpt.UTF8ToString(libopenmpt._openmpt_module_get_metadata_keys(this.currentPlayingNode.modulePtr)).split(';');
 	let keyNameBuffer = 0;
 	for (const key of keys) {
 		keyNameBuffer = libopenmpt._malloc(key.length + 1);
-		writeAsciiToMemory(key, keyNameBuffer);
-		data[key] = UTF8ToString(libopenmpt._openmpt_module_get_metadata(this.currentPlayingNode.modulePtr, keyNameBuffer));
+		libopenmpt.writeAsciiToMemory(key, keyNameBuffer);
+		data[key] = libopenmpt.UTF8ToString(libopenmpt._openmpt_module_get_metadata(this.currentPlayingNode.modulePtr, keyNameBuffer));
 		libopenmpt._free(keyNameBuffer);
 	}
 	return data;
@@ -84,7 +109,7 @@ ChiptuneJsPlayer.prototype.unlock = function () {
 };
 
 ChiptuneJsPlayer.prototype.load = function (input) {
-	return new Promise((resolve, reject) => {
+	return this.initialize().then(() => new Promise((resolve, reject) => {
 		if(this.touchLocked) {
 			this.unlock();
 		}
@@ -106,7 +131,7 @@ ChiptuneJsPlayer.prototype.load = function (input) {
 				reject(error);
 			});
 		}
-	});
+	}));
 };
 
 ChiptuneJsPlayer.prototype.play = function (buffer: ArrayBuffer) {
@@ -180,7 +205,7 @@ ChiptuneJsPlayer.prototype.getPatternNumRows = function (pattern: number) {
 
 ChiptuneJsPlayer.prototype.getPatternRowChannel = function (pattern: number, row: number, channel: number) {
 	if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) {
-		return UTF8ToString(libopenmpt._openmpt_module_format_pattern_row_channel(this.currentPlayingNode.modulePtr, pattern, row, channel, 0, true));
+		return libopenmpt.UTF8ToString(libopenmpt._openmpt_module_format_pattern_row_channel(this.currentPlayingNode.modulePtr, pattern, row, channel, 0, true));
 	}
 	return '';
 };

packages/frontend/src/scripts/libopenmpt/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2004-2024, OpenMPT Project Developers and Contributors
+Copyright (c) 1997-2003, Olivier Lapicque
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the OpenMPT project nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/frontend/src/scripts/libopenmpt/readme.md

@@ -0,0 +1,23 @@
+modifications made to `libopenmpt.js` (can be taken from https://lib.openmpt.org/libopenmpt/download/):
+
+at the beginning of the file:
+```js
+// @ts-nocheck
+/* eslint-disable */
+```
+
+at the end of the file:
+```js
+Module.UTF8ToString = UTF8ToString;
+Module.writeAsciiToMemory = writeAsciiToMemory;
+export { Module }
+```
+
+replace
+```
+wasmBinaryFile="libopenmpt.wasm"
+```
+with
+```
+wasmBinaryFile=new URL("./libopenmpt.wasm", import.meta.url).href
+```

packages/frontend/vite.config.ts

@@ -8,7 +8,7 @@ import meta from '../../package.json';
 import pluginUnwindCssModuleClassName from './lib/rollup-plugin-unwind-css-module-class-name.js';
 import pluginJson5 from './vite.json5.js';
 
-const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.json', '.json5', '.svg', '.sass', '.scss', '.css', '.vue'];
+const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.json', '.json5', '.svg', '.sass', '.scss', '.css', '.vue', '.wasm'];
 
 const hash = (str: string, seed = 0): number => {
 	let h1 = 0xdeadbeef ^ seed,

packages/megalodon/src/entities/status.ts

@@ -19,6 +19,7 @@ namespace Entity {
     content: string
     plain_content?: string | null
     created_at: string
+    edited_at: string | null
     emojis: Emoji[]
     replies_count: number
     reblogs_count: number

packages/megalodon/src/friendica/api_client.ts

@@ -725,6 +725,7 @@ namespace FriendicaAPI {
       content: s.content,
       plain_content: null,
       created_at: s.created_at,
+      edited_at: s.edited_at || null,
       emojis: Array.isArray(s.emojis) ? s.emojis.map(e => emoji(e)) : [],
       replies_count: s.replies_count,
       reblogs_count: s.reblogs_count,

packages/megalodon/src/friendica/entities/status.ts

@@ -17,6 +17,7 @@ namespace FriendicaEntity {
     reblog: Status | null
     content: string
     created_at: string
+    edited_at?: string | null
     emojis: Emoji[]
     replies_count: number
     reblogs_count: number

packages/megalodon/src/mastodon/api_client.ts

@@ -628,6 +628,7 @@ namespace MastodonAPI {
       content: s.content,
       plain_content: null,
       created_at: s.created_at,
+      edited_at: s.edited_at || null,
       emojis: Array.isArray(s.emojis) ? s.emojis.map(e => emoji(e)) : [],
       replies_count: s.replies_count,
       reblogs_count: s.reblogs_count,

packages/megalodon/src/mastodon/entities/status.ts

@@ -18,6 +18,7 @@ namespace MastodonEntity {
     reblog: Status | null
     content: string
     created_at: string
+    edited_at?: string | null
     emojis: Emoji[]
     replies_count: number
     reblogs_count: number

packages/megalodon/src/misskey/api_client.ts

@@ -283,6 +283,7 @@ namespace MisskeyAPI {
           : '',
         plain_content: n.text ? n.text : null,
         created_at: n.createdAt,
+        edited_at: n.updatedAt || null,
         emojis: mapEmojis(n.emojis).concat(mapReactionEmojis(n.reactionEmojis)),
         replies_count: n.repliesCount,
         reblogs_count: n.renoteCount,

packages/megalodon/src/misskey/entities/note.ts

@@ -7,6 +7,7 @@ namespace MisskeyEntity {
   export type Note = {
     id: string
     createdAt: string
+    updatedAt?: string | null
     userId: string
     user: User
     text: string | null

packages/megalodon/src/pleroma/api_client.ts

@@ -357,6 +357,7 @@ namespace PleromaAPI {
       content: s.content,
       plain_content: s.pleroma.content?.['text/plain'] ? s.pleroma.content['text/plain'] : null,
       created_at: s.created_at,
+      edited_at: s.edited_at || null,
       emojis: Array.isArray(s.emojis) ? s.emojis.map(e => emoji(e)) : [],
       replies_count: s.replies_count,
       reblogs_count: s.reblogs_count,

packages/megalodon/src/pleroma/entities/status.ts

@@ -18,6 +18,7 @@ namespace PleromaEntity {
     reblog: Status | null
     content: string
     created_at: string
+    edited_at?: string | null
     emojis: Emoji[]
     replies_count: number
     reblogs_count: number

packages/megalodon/test/integration/mastodon/api_client.spec.ts

@@ -49,6 +49,7 @@ const status: Entity.Status = {
   content: 'hoge',
   plain_content: null,
   created_at: '2019-03-26T21:40:32',
+  edited_at: null,
   emojis: [],
   replies_count: 0,
   reblogs_count: 0,

packages/megalodon/test/unit/parser.spec.ts

@@ -38,6 +38,7 @@ const status: Entity.Status = {
   content: 'hoge',
   plain_content: 'hoge',
   created_at: '2019-03-26T21:40:32',
+  edited_at: null,
   emojis: [],
   replies_count: 0,
   reblogs_count: 0,

packages/megalodon/test/unit/webo_socket.spec.ts

@@ -37,6 +37,7 @@ const status: Entity.Status = {
   content: 'hoge',
   plain_content: 'hoge',
   created_at: '2019-03-26T21:40:32',
+  edited_at: null,
   emojis: [],
   replies_count: 0,
   reblogs_count: 0,