merge: handle non-ASCII emoji names (!464)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/464

IMPORTANT_NOTES.md

@@ -6,8 +6,11 @@ When using a service with Sharkey, there are several important points to keep in
 
 2. Even for posts made in private, there is no guarantee that the recipient's server will treat them as private in the same way. Please exercise caution when posting personal or confidential information. (Again, this applies to the internet in general.)
 
-3. Account deletion can be a resource-intensive process and may take a long time. In cases with a lot of uploaded data, it may even be impossible to delete an account.
+3. The "Drive" feature is NOT secure cloud storage. This feature exists for easier managing of your uploaded files.
+Any data uploaded, whether shared via post or not, will be publicly accessible. Please use 3rd party cloud storage providers if you need to upload data with sensitive information of any kind. 
 
-4. Please disable ad blockers. Some servers may rely on advertising revenue to cover operating costs. Additionally, ad blockers can mistakenly block content and features unrelated to ads, potentially causing issues with the client's functionality and preventing normal use of Sharkey. Therefore, we recommend turning off ad blockers and similar features when using Sharkey.
+4. Account deletion can be a resource-intensive process and may take a long time. In cases with a lot of uploaded data, it may even be impossible to delete an account.
 
-Please understand these points and enjoy using the service.
+5. Please disable ad blockers. Some servers may rely on advertising revenue to cover operating costs. Additionally, ad blockers can mistakenly block content and features unrelated to ads, potentially causing issues with the client's functionality and preventing normal use of Sharkey. Therefore, we recommend turning off ad blockers and similar features when using Sharkey.
+
+Please understand these points and enjoy using the service.

packages/backend/ormconfig.js

@@ -11,7 +11,11 @@ export default new DataSource({
 	username: config.db.user,
 	password: config.db.pass,
 	database: config.db.db,
-	extra: config.db.extra,
+	extra: {
+		...config.db.extra,
+		// migrations may be very slow, give them longer to run (that 10*1000 comes from postgres.ts)
+		statement_timeout: (config.db.extra?.statement_timeout ?? 1000 * 10) * 10,
+	},
 	entities: entities,
 	migrations: ['migration/*.js'],
 });

packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts

@@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
 		});
 
 		for (const emoji of customEmojis) {
-			if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) {
+			if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
 				this.logger.error(`invalid emoji name: ${emoji.name}`);
 				continue;
 			}

packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts

@@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
 					continue;
 				}
 				const emojiInfo = record.emoji;
-				if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) {
-					this.logger.error(`invalid emojiname: ${emojiInfo.name}`);
+				const nameNfc = emojiInfo.name.normalize('NFC');
+				if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
+					this.logger.error(`invalid emojiname: ${nameNfc}`);
 					continue;
 				}
 				const emojiPath = outputPath + '/' + record.fileName;
 				await this.emojisRepository.delete({
-					name: emojiInfo.name,
+					name: nameNfc,
 				});
 				const driveFile = await this.driveService.addFile({
 					user: null,
@@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
 					force: true,
 				});
 				await this.customEmojiService.add({
-					name: emojiInfo.name,
-					category: emojiInfo.category,
+					name: nameNfc,
+					category: emojiInfo.category?.normalize('NFC'),
 					host: null,
-					aliases: emojiInfo.aliases,
+					aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
 					driveFile,
 					license: emojiInfo.license,
 					isSensitive: emojiInfo.isSensitive,

packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts

@@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases);
+			await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/add.ts

@@ -40,7 +40,7 @@ export const meta = {
 export const paramDef = {
 	type: 'object',
 	properties: {
-		name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
+		name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
 		fileId: { type: 'string', format: 'misskey:id' },
 		category: {
 			type: 'string',
@@ -73,18 +73,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private emojiEntityService: EmojiEntityService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
+			const nameNfc = ps.name.normalize('NFC');
 			const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
 			if (driveFile == null) throw new ApiError(meta.errors.noSuchFile);
-			const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
+			const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
 			if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
 
 			if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null });
 
 			const emoji = await this.customEmojiService.add({
 				driveFile,
-				name: ps.name,
-				category: ps.category ?? null,
-				aliases: ps.aliases ?? [],
+				name: nameNfc,
+				category: ps.category?.normalize('NFC') ?? null,
+				aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [],
 				host: null,
 				license: ps.license ?? null,
 				isSensitive: ps.isSensitive ?? false,

packages/backend/src/server/api/endpoints/admin/emoji/copy.ts

@@ -82,15 +82,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				throw new ApiError();
 			}
 
+			const nameNfc = emoji.name.normalize('NFC');
 			// Duplication Check
-			const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name);
+			const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
 			if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
 
 			const addedEmoji = await this.customEmojiService.add({
 				driveFile,
-				name: emoji.name,
-				category: emoji.category,
-				aliases: emoji.aliases,
+				name: nameNfc,
+				category: emoji.category?.normalize('NFC'),
+				aliases: emoji.aliases?.map(a => a.normalize('NFC')),
 				host: null,
 				license: emoji.license,
 				isSensitive: emoji.isSensitive,

packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts

@@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			}
 
 			if (ps.query) {
-				q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' })
+				q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
 					.orderBy('length(emoji.name)', 'ASC');
 			}
 

packages/backend/src/server/api/endpoints/admin/emoji/list.ts

@@ -92,17 +92,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				//const emojis = await q.limit(ps.limit).getMany();
 
 				emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
-				const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g);
+				const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
 
 				if (queryarry) {
 					emojis = emojis.filter(emoji =>
-						queryarry.includes(`:${emoji.name}:`),
+						queryarry.includes(`:${emoji.name.normalize('NFC')}:`),
 					);
 				} else {
+					const queryNfc = ps.query!.normalize('NFC');
 					emojis = emojis.filter(emoji =>
-						emoji.name.includes(ps.query!) ||
-						emoji.aliases.some(a => a.includes(ps.query!)) ||
-						emoji.category?.includes(ps.query!));
+						emoji.name.includes(queryNfc) ||
+						emoji.aliases.some(a => a.includes(queryNfc)) ||
+						emoji.category?.includes(queryNfc));
 				}
 				emojis.splice(ps.limit + 1);
 			} else {

packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts

@@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases);
+			await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts

@@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases);
+			await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts

@@ -36,7 +36,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null);
+			await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null);
 		});
 	}
 }

packages/backend/src/server/api/endpoints/admin/emoji/update.ts

@@ -40,7 +40,7 @@ export const paramDef = {
 	type: 'object',
 	properties: {
 		id: { type: 'string', format: 'misskey:id' },
-		name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
+		name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
 		fileId: { type: 'string', format: 'misskey:id' },
 		category: {
 			type: 'string',
@@ -72,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private customEmojiService: CustomEmojiService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
+			const nameNfc = ps.name?.normalize('NFC');
 			let driveFile;
 			if (ps.fileId) {
 				driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
@@ -83,22 +84,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				emojiId = ps.id;
 				const emoji = await this.customEmojiService.getEmojiById(ps.id);
 				if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
-				if (ps.name && (ps.name !== emoji.name)) {
-					const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
+				if (nameNfc && (nameNfc !== emoji.name)) {
+					const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
 					if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists);
 				}
 			} else {
-				if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
-				const emoji = await this.customEmojiService.getEmojiByName(ps.name);
+				if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
+				const emoji = await this.customEmojiService.getEmojiByName(nameNfc);
 				if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
 				emojiId = emoji.id;
 			}
 
 			await this.customEmojiService.update(emojiId, {
 				driveFile,
-				name: ps.name,
-				category: ps.category,
-				aliases: ps.aliases,
+				name: nameNfc,
+				category: ps.category?.normalize('NFC'),
+				aliases: ps.aliases?.map(a => a.normalize('NFC')),
 				license: ps.license,
 				isSensitive: ps.isSensitive,
 				localOnly: ps.localOnly,

packages/frontend/src/account.ts

@@ -43,6 +43,7 @@ export async function signout() {
 	waiting();
 	miLocalStorage.removeItem('account');
 	await removeAccount($i.id);
+	document.cookie = `token=; path=/; max-age=0${ location.protocol === 'https:' ? '; Secure' : ''}`;
 	const accounts = await getAccounts();
 
 	//#region Remove service worker registration
@@ -200,7 +201,7 @@ export async function login(token: Account['token'], redirect?: string) {
 			throw reason;
 		});
 	miLocalStorage.setItem('account', JSON.stringify(me));
-	document.cookie = `token=${token}; path=/; max-age=31536000`; // bull dashboardの認証とかで使う
+	document.cookie = `token=${token}; path=/; max-age=31536000${ location.protocol === 'https:' ? '; Secure' : ''}`; // bull dashboardの認証とかで使う
 	await addAccount(me.id, token);
 
 	if (redirect) {

packages/frontend/src/components/MkAutocomplete.vue

@@ -238,7 +238,7 @@ function exec() {
 			return;
 		}
 
-		emojis.value = searchEmoji(props.q.toLowerCase(), emojiDb.value);
+		emojis.value = searchEmoji(props.q.normalize('NFC').toLowerCase(), emojiDb.value);
 	} else if (props.type === 'mfmTag') {
 		if (!props.q || props.q === '') {
 			mfmTags.value = MFM_TAGS;

packages/frontend/src/components/MkCode.core.vue

@@ -72,6 +72,10 @@ watch(() => props.lang, (to) => {
 </script>
 
 <style module lang="scss">
+.codeBlockRoot {
+  text-align: left;
+}
+
 .codeBlockRoot :global(.shiki) > code {
   counter-reset: step;
   counter-increment: step 0;

packages/frontend/src/components/MkEmojiPicker.vue

@@ -205,7 +205,7 @@ watch(q, () => {
 		return;
 	}
 
-	const newQ = q.value.replace(/:/g, '').toLowerCase();
+	const newQ = q.value.replace(/:/g, '').normalize('NFC').toLowerCase();
 
 	const searchCustom = () => {
 		const max = 100;

packages/frontend/src/pages/emoji-edit-dialog.vue

@@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 					</div>
 				</div>
 				<MkButton rounded style="margin: 0 auto;" @click="changeImage">{{ i18n.ts.selectFile }}</MkButton>
-				<MkInput v-model="name" pattern="[a-z0-9_]" autocapitalize="off">
+				<MkInput v-model="name" autocapitalize="off">
 					<template #label>{{ i18n.ts.name }}</template>
 				</MkInput>
 				<MkInput v-model="category" :datalist="customEmojiCategories">

packages/frontend/src/pages/welcome.timeline.vue

@@ -40,7 +40,7 @@ const isScrolling = ref(false);
 const scrollEl = shallowRef<HTMLElement>();
 
 misskeyApiGet('notes/featured').then(_notes => {
-	notes.value = _notes;
+	notes.value = _notes.filter(n => n.cw == null);
 });
 
 onUpdated(() => {

packages/frontend/src/scripts/autocomplete.ts

@@ -99,7 +99,7 @@ export class Autocomplete {
 		const isHashtag = hashtagIndex !== -1;
 		const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' ');
 		const isMfmTag = mfmTagIndex !== -1 && !isMfmParam;
-		const isEmoji = emojiIndex !== -1 && text.split(/:[a-z0-9_+\-]+:/).pop()!.includes(':');
+		const isEmoji = emojiIndex !== -1 && text.split(/:[\p{Letter}\p{Number}\p{Mark}_+-]+:/u).pop()!.includes(':');
 
 		let opened = false;
 
@@ -125,7 +125,7 @@ export class Autocomplete {
 		if (isEmoji && !opened && this.onlyType.includes('emoji')) {
 			const emoji = text.substring(emojiIndex + 1);
 			if (!emoji.includes(' ')) {
-				this.open('emoji', emoji);
+				this.open('emoji', emoji.normalize('NFC'));
 				opened = true;
 			}
 		}