merge: handle non-ASCII emoji names (!464)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/464

packages/backend/src/core/HttpRequestService.ts

@@ -15,6 +15,7 @@ import type { Config } from '@/config.js';
 import { StatusError } from '@/misc/status-error.js';
 import { bindThis } from '@/decorators.js';
 import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
+import { assertActivityMatchesUrls } from '@/core/activitypub/misc/check-against-url.js';
 import type { IObject } from '@/core/activitypub/type.js';
 import type { Response } from 'node-fetch';
 import type { URL } from 'node:url';
@@ -125,7 +126,12 @@ export class HttpRequestService {
 			validators: [validateContentTypeSetAsActivityPub],
 		});
 
-		return await res.json() as IObject;
+		const finalUrl = res.url; // redirects may have been involved
+		const activity = await res.json() as IObject;
+
+		assertActivityMatchesUrls(activity, [url, finalUrl]);
+
+		return activity;
 	}
 
 	@bindThis

packages/backend/src/core/UtilityService.ts

@@ -86,7 +86,7 @@ export class UtilityService {
 	@bindThis
 	public extractDbHost(uri: string): string {
 		const url = new URL(uri);
-		return this.toPuny(url.hostname);
+		return this.toPuny(url.host);
 	}
 
 	@bindThis
@@ -99,4 +99,11 @@ export class UtilityService {
 		if (host == null) return null;
 		return toASCII(host.toLowerCase());
 	}
+
+	@bindThis
+	public punyHost(url: string): string {
+		const urlObj = new URL(url);
+		const host = `${this.toPuny(urlObj.hostname)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`;
+		return host;
+	}
 }

packages/backend/src/core/activitypub/ApRequestService.ts

@@ -14,7 +14,9 @@ import { HttpRequestService } from '@/core/HttpRequestService.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import { bindThis } from '@/decorators.js';
 import type Logger from '@/logger.js';
+import type { IObject } from './type.js';
 import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
+import { assertActivityMatchesUrls } from '@/core/activitypub/misc/check-against-url.js';
 
 type Request = {
 	url: string;
@@ -201,6 +203,11 @@ export class ApRequestService {
 			validators: [validateContentTypeSetAsActivityPub],
 		});
 
-		return await res.json();
+		const finalUrl = res.url; // redirects may have been involved
+		const activity = await res.json() as IObject;
+
+		assertActivityMatchesUrls(activity, [url, finalUrl]);
+
+		return activity;
 	}
 }

packages/backend/src/core/activitypub/ApResolverService.ts

@@ -115,6 +115,14 @@ export class Resolver {
 			throw new Error('invalid response');
 		}
 
+		// HttpRequestService / ApRequestService have already checked that
+		// `object.id` or `object.url` matches the URL used to fetch the
+		// object after redirects; here we double-check that no redirects
+		// bounced between hosts
+		if (object.id && (this.utilityService.punyHost(object.id) !== this.utilityService.punyHost(value))) {
+			throw new Error(`invalid AP object ${value}: id ${object.id} has different host`);
+		}
+
 		return object;
 	}
 

packages/backend/src/core/activitypub/misc/check-against-url.ts

@@ -0,0 +1,19 @@
+/*
+ * SPDX-FileCopyrightText: dakkar and sharkey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+import type { IObject } from '../type.js';
+
+export function assertActivityMatchesUrls(activity: IObject, urls: string[]) {
+	const idOk = activity.id !== undefined && urls.includes(activity.id);
+
+	// technically `activity.url` could be an `ApObject = IObject |
+	// string | (IObject | string)[]`, but if it's a complicated thing
+	// and the `activity.id` doesn't match, I think we're fine
+	// rejecting the activity
+	const urlOk = typeof(activity.url) === 'string' && urls.includes(activity.url);
+
+	if (!idOk && !urlOk) {
+		throw new Error(`bad Activity: neither id(${activity?.id}) nor url(${activity?.url}) match location(${urls})`);
+	}
+}

packages/backend/src/core/activitypub/models/ApPersonService.ts

@@ -127,12 +127,6 @@ export class ApPersonService implements OnModuleInit {
 		this.logger = this.apLoggerService.logger;
 	}
 
-	private punyHost(url: string): string {
-		const urlObj = new URL(url);
-		const host = `${this.utilityService.toPuny(urlObj.hostname)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`;
-		return host;
-	}
-
 	/**
 	 * Validate and convert to actor object
 	 * @param x Fetched object
@@ -140,7 +134,7 @@ export class ApPersonService implements OnModuleInit {
 	 */
 	@bindThis
 	private validateActor(x: IObject, uri: string): IActor {
-		const expectHost = this.punyHost(uri);
+		const expectHost = this.utilityService.punyHost(uri);
 
 		if (!isActor(x)) {
 			throw new Error(`invalid Actor type '${x.type}'`);
@@ -154,6 +148,19 @@ export class ApPersonService implements OnModuleInit {
 			throw new Error('invalid Actor: wrong inbox');
 		}
 
+		if (this.utilityService.punyHost(x.inbox) !== expectHost) {
+			throw new Error('invalid Actor: inbox has different host');
+		}
+
+		for (const collection of ['outbox', 'followers', 'following'] as (keyof IActor)[]) {
+			const collectionUri = (x as IActor)[collection];
+			if (typeof collectionUri === 'string' && collectionUri.length > 0) {
+				if (this.utilityService.punyHost(collectionUri) !== expectHost) {
+					throw new Error(`invalid Actor: ${collection} has different host`);
+				}
+			}
+		}
+
 		if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) {
 			throw new Error('invalid Actor: wrong username');
 		}
@@ -177,7 +184,7 @@ export class ApPersonService implements OnModuleInit {
 			x.summary = truncate(x.summary, summaryLength);
 		}
 
-		const idHost = this.punyHost(x.id);
+		const idHost = this.utilityService.punyHost(x.id);
 		if (idHost !== expectHost) {
 			throw new Error('invalid Actor: id has different host');
 		}
@@ -187,7 +194,7 @@ export class ApPersonService implements OnModuleInit {
 				throw new Error('invalid Actor: publicKey.id is not a string');
 			}
 
-			const publicKeyIdHost = this.punyHost(x.publicKey.id);
+			const publicKeyIdHost = this.utilityService.punyHost(x.publicKey.id);
 			if (publicKeyIdHost !== expectHost) {
 				throw new Error('invalid Actor: publicKey.id has different host');
 			}
@@ -286,7 +293,7 @@ export class ApPersonService implements OnModuleInit {
 
 		this.logger.info(`Creating the Person: ${person.id}`);
 
-		const host = this.punyHost(object.id);
+		const host = this.utilityService.punyHost(object.id);
 
 		const fields = this.analyzeAttachments(person.attachment ?? []);
 

packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts

@@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
 		});
 
 		for (const emoji of customEmojis) {
-			if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) {
+			if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
 				this.logger.error(`invalid emoji name: ${emoji.name}`);
 				continue;
 			}

packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts

@@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
 					continue;
 				}
 				const emojiInfo = record.emoji;
-				if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) {
-					this.logger.error(`invalid emojiname: ${emojiInfo.name}`);
+				const nameNfc = emojiInfo.name.normalize('NFC');
+				if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
+					this.logger.error(`invalid emojiname: ${nameNfc}`);
 					continue;
 				}
 				const emojiPath = outputPath + '/' + record.fileName;
 				await this.emojisRepository.delete({
-					name: emojiInfo.name,
+					name: nameNfc,
 				});
 				const driveFile = await this.driveService.addFile({
 					user: null,
@@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
 					force: true,
 				});
 				await this.customEmojiService.add({
-					name: emojiInfo.name,
-					category: emojiInfo.category,
+					name: nameNfc,
+					category: emojiInfo.category?.normalize('NFC'),
 					host: null,
-					aliases: emojiInfo.aliases,
+					aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
 					driveFile,
 					license: emojiInfo.license,
 					isSensitive: emojiInfo.isSensitive,

packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts

@@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases);
+			await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/add.ts

@@ -40,7 +40,7 @@ export const meta = {
 export const paramDef = {
 	type: 'object',
 	properties: {
-		name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
+		name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
 		fileId: { type: 'string', format: 'misskey:id' },
 		category: {
 			type: 'string',
@@ -73,18 +73,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private emojiEntityService: EmojiEntityService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
+			const nameNfc = ps.name.normalize('NFC');
 			const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
 			if (driveFile == null) throw new ApiError(meta.errors.noSuchFile);
-			const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
+			const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
 			if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
 
 			if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null });
 
 			const emoji = await this.customEmojiService.add({
 				driveFile,
-				name: ps.name,
-				category: ps.category ?? null,
-				aliases: ps.aliases ?? [],
+				name: nameNfc,
+				category: ps.category?.normalize('NFC') ?? null,
+				aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [],
 				host: null,
 				license: ps.license ?? null,
 				isSensitive: ps.isSensitive ?? false,

packages/backend/src/server/api/endpoints/admin/emoji/copy.ts

@@ -82,15 +82,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				throw new ApiError();
 			}
 
+			const nameNfc = emoji.name.normalize('NFC');
 			// Duplication Check
-			const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name);
+			const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
 			if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
 
 			const addedEmoji = await this.customEmojiService.add({
 				driveFile,
-				name: emoji.name,
-				category: emoji.category,
-				aliases: emoji.aliases,
+				name: nameNfc,
+				category: emoji.category?.normalize('NFC'),
+				aliases: emoji.aliases?.map(a => a.normalize('NFC')),
 				host: null,
 				license: emoji.license,
 				isSensitive: emoji.isSensitive,

packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts

@@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			}
 
 			if (ps.query) {
-				q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' })
+				q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
 					.orderBy('length(emoji.name)', 'ASC');
 			}
 

packages/backend/src/server/api/endpoints/admin/emoji/list.ts

@@ -92,17 +92,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				//const emojis = await q.limit(ps.limit).getMany();
 
 				emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
-				const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g);
+				const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
 
 				if (queryarry) {
 					emojis = emojis.filter(emoji =>
-						queryarry.includes(`:${emoji.name}:`),
+						queryarry.includes(`:${emoji.name.normalize('NFC')}:`),
 					);
 				} else {
+					const queryNfc = ps.query!.normalize('NFC');
 					emojis = emojis.filter(emoji =>
-						emoji.name.includes(ps.query!) ||
-						emoji.aliases.some(a => a.includes(ps.query!)) ||
-						emoji.category?.includes(ps.query!));
+						emoji.name.includes(queryNfc) ||
+						emoji.aliases.some(a => a.includes(queryNfc)) ||
+						emoji.category?.includes(queryNfc));
 				}
 				emojis.splice(ps.limit + 1);
 			} else {

packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts

@@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases);
+			await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts

@@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases);
+			await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts

@@ -36,7 +36,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null);
+			await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null);
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/update.ts

@@ -40,7 +40,7 @@ export const paramDef = {
 	type: 'object',
 	properties: {
 		id: { type: 'string', format: 'misskey:id' },
-		name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
+		name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
 		fileId: { type: 'string', format: 'misskey:id' },
 		category: {
 			type: 'string',
@@ -72,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
+			const nameNfc = ps.name?.normalize('NFC');
 			let driveFile;
 			if (ps.fileId) {
 				driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
@@ -83,22 +84,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				emojiId = ps.id;
 				const emoji = await this.customEmojiService.getEmojiById(ps.id);
 				if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
-				if (ps.name && (ps.name !== emoji.name)) {
-					const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
+				if (nameNfc && (nameNfc !== emoji.name)) {
+					const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
 					if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists);
 				}
 			} else {
-				if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
-				const emoji = await this.customEmojiService.getEmojiByName(ps.name);
+				if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
+				const emoji = await this.customEmojiService.getEmojiByName(nameNfc);
 				if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
 				emojiId = emoji.id;
 			}
 
 			await this.customEmojiService.update(emojiId, {
 				driveFile,
-				name: ps.name,
-				category: ps.category,
-				aliases: ps.aliases,
+				name: nameNfc,
+				category: ps.category?.normalize('NFC'),
+				aliases: ps.aliases?.map(a => a.normalize('NFC')),
 				license: ps.license,
 				isSensitive: ps.isSensitive,
 				localOnly: ps.localOnly,

packages/backend/src/server/api/endpoints/ap/show.ts

@@ -113,8 +113,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 	@bindThis
 	private async fetchAny(uri: string, me: MiLocalUser | null | undefined): Promise<SchemaType<typeof meta['res']> | null> {
 	// ブロックしてたら中断
+		const host = this.utilityService.extractDbHost(uri);
 		const fetchedMeta = await this.metaService.fetch();
-		if (this.utilityService.isBlockedHost(fetchedMeta.blockedHosts, this.utilityService.extractDbHost(uri))) return null;
+		if (this.utilityService.isBlockedHost(fetchedMeta.blockedHosts, host)) return null;
 
 		let local = await this.mergePack(me, ...await Promise.all([
 			this.apDbResolverService.getUserFromApId(uri),
@@ -122,6 +123,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		]));
 		if (local != null) return local;
 
+		// local object, not found in db? fail
+		if (this.utilityService.isSelfHost(host)) return null;
+
 		// リモートから一旦オブジェクトフェッチ
 		const resolver = this.apResolverService.createResolver();
 		const object = await resolver.resolve(uri) as any;

packages/frontend/src/components/MkAutocomplete.vue

@@ -238,7 +238,7 @@ function exec() {
 			return;
 		}
 
-		emojis.value = searchEmoji(props.q.toLowerCase(), emojiDb.value);
+		emojis.value = searchEmoji(props.q.normalize('NFC').toLowerCase(), emojiDb.value);
 	} else if (props.type === 'mfmTag') {
 		if (!props.q || props.q === '') {
 			mfmTags.value = MFM_TAGS;

packages/frontend/src/components/MkEmojiPicker.vue

@@ -205,7 +205,7 @@ watch(q, () => {
 		return;
 	}
 
-	const newQ = q.value.replace(/:/g, '').toLowerCase();
+	const newQ = q.value.replace(/:/g, '').normalize('NFC').toLowerCase();
 
 	const searchCustom = () => {
 		const max = 100;

packages/frontend/src/pages/emoji-edit-dialog.vue

@@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 					</div>
 				</div>
 				<MkButton rounded style="margin: 0 auto;" @click="changeImage">{{ i18n.ts.selectFile }}</MkButton>
-				<MkInput v-model="name" pattern="[a-z0-9_]" autocapitalize="off">
+				<MkInput v-model="name" autocapitalize="off">
 					<template #label>{{ i18n.ts.name }}</template>
 				</MkInput>
 				<MkInput v-model="category" :datalist="customEmojiCategories">

packages/frontend/src/scripts/autocomplete.ts

@@ -99,7 +99,7 @@ export class Autocomplete {
 		const isHashtag = hashtagIndex !== -1;
 		const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' ');
 		const isMfmTag = mfmTagIndex !== -1 && !isMfmParam;
-		const isEmoji = emojiIndex !== -1 && text.split(/:[a-z0-9_+\-]+:/).pop()!.includes(':');
+		const isEmoji = emojiIndex !== -1 && text.split(/:[\p{Letter}\p{Number}\p{Mark}_+-]+:/u).pop()!.includes(':');
 
 		let opened = false;
 
@@ -125,7 +125,7 @@ export class Autocomplete {
 		if (isEmoji && !opened && this.onlyType.includes('emoji')) {
 			const emoji = text.substring(emojiIndex + 1);
 			if (!emoji.includes(' ')) {
-				this.open('emoji', emoji);
+				this.open('emoji', emoji.normalize('NFC'));
 				opened = true;
 			}
 		}