From 354cb2a6754b55fd3ad01388a4a17d3a76d4a09b Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 9 Mar 2024 12:17:48 +0000 Subject: [PATCH 1/2] handle non-ASCII emoji names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use the more inclusive regexp for validating emoji names * always normalize emoji names, aliases, categories the latter point is necessary to allow matching, for example, `รค` against `a`+combining diaeresis this will also need to bump the version of `sfm-js` once we merge https://activitypub.software/TransFem-org/sfm-js/-/merge_requests/2 --- .../ExportCustomEmojisProcessorService.ts | 2 +- .../ImportCustomEmojisProcessorService.ts | 13 +++++++------ .../endpoints/admin/emoji/add-aliases-bulk.ts | 2 +- .../src/server/api/endpoints/admin/emoji/add.ts | 11 ++++++----- .../server/api/endpoints/admin/emoji/copy.ts | 9 +++++---- .../api/endpoints/admin/emoji/list-remote.ts | 2 +- .../server/api/endpoints/admin/emoji/list.ts | 11 ++++++----- .../admin/emoji/remove-aliases-bulk.ts | 2 +- .../endpoints/admin/emoji/set-aliases-bulk.ts | 2 +- .../endpoints/admin/emoji/set-category-bulk.ts | 2 +- .../server/api/endpoints/admin/emoji/update.ts | 17 +++++++++-------- .../frontend/src/components/MkAutocomplete.vue | 2 +- .../frontend/src/components/MkEmojiPicker.vue | 2 +- .../frontend/src/pages/emoji-edit-dialog.vue | 2 +- packages/frontend/src/scripts/autocomplete.ts | 4 ++-- 15 files changed, 44 insertions(+), 39 deletions(-) diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index e4eb4791b..45087927a 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService { }); for (const emoji of customEmojis) { - if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) { + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) { this.logger.error(`invalid emoji name: ${emoji.name}`); continue; } diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 171809d25..04ad74ee0 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService { continue; } const emojiInfo = record.emoji; - if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) { - this.logger.error(`invalid emojiname: ${emojiInfo.name}`); + const nameNfc = emojiInfo.name.normalize('NFC'); + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) { + this.logger.error(`invalid emojiname: ${nameNfc}`); continue; } const emojiPath = outputPath + '/' + record.fileName; await this.emojisRepository.delete({ - name: emojiInfo.name, + name: nameNfc, }); const driveFile = await this.driveService.addFile({ user: null, @@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService { force: true, }); await this.customEmojiService.add({ - name: emojiInfo.name, - category: emojiInfo.category, + name: nameNfc, + category: emojiInfo.category?.normalize('NFC'), host: null, - aliases: emojiInfo.aliases, + aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')), driveFile, license: emojiInfo.license, isSensitive: emojiInfo.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts index a30a080e5..f4fc79bdb 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index 767e517b8..b45a3c715 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -40,7 +40,7 @@ export const meta = { export const paramDef = { type: 'object', properties: { - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -73,18 +73,19 @@ export default class extends Endpoint { // eslint- private emojiEntityService: EmojiEntityService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name.normalize('NFC'); const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); if (driveFile == null) throw new ApiError(meta.errors.noSuchFile); - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null }); const emoji = await this.customEmojiService.add({ driveFile, - name: ps.name, - category: ps.category ?? null, - aliases: ps.aliases ?? [], + name: nameNfc, + category: ps.category?.normalize('NFC') ?? null, + aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [], host: null, license: ps.license ?? null, isSensitive: ps.isSensitive ?? false, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index 29af7598e..f96881319 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -82,15 +82,16 @@ export default class extends Endpoint { // eslint- throw new ApiError(); } + const nameNfc = emoji.name.normalize('NFC'); // Duplication Check - const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); const addedEmoji = await this.customEmojiService.add({ driveFile, - name: emoji.name, - category: emoji.category, - aliases: emoji.aliases, + name: nameNfc, + category: emoji.category?.normalize('NFC'), + aliases: emoji.aliases?.map(a => a.normalize('NFC')), host: null, license: emoji.license, isSensitive: emoji.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index e423f440d..1182918ea 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -98,7 +98,7 @@ export default class extends Endpoint { // eslint- } if (ps.query) { - q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }) + q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' }) .orderBy('length(emoji.name)', 'ASC'); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts index 53810d1d1..5e21111f9 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts @@ -92,17 +92,18 @@ export default class extends Endpoint { // eslint- //const emojis = await q.limit(ps.limit).getMany(); emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany(); - const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g); + const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug); if (queryarry) { emojis = emojis.filter(emoji => - queryarry.includes(`:${emoji.name}:`), + queryarry.includes(`:${emoji.name.normalize('NFC')}:`), ); } else { + const queryNfc = ps.query!.normalize('NFC'); emojis = emojis.filter(emoji => - emoji.name.includes(ps.query!) || - emoji.aliases.some(a => a.includes(ps.query!)) || - emoji.category?.includes(ps.query!)); + emoji.name.includes(queryNfc) || + emoji.aliases.some(a => a.includes(queryNfc)) || + emoji.category?.includes(queryNfc)); } emojis.splice(ps.limit + 1); } else { diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts index 0fa119eab..e78620eac 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts index d9ee18699..69fc8e0cb 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts index dc25df276..679a9f9c4 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts @@ -36,7 +36,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null); + await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts index 22609a16a..3caa0f84a 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts @@ -40,7 +40,7 @@ export const paramDef = { type: 'object', properties: { id: { type: 'string', format: 'misskey:id' }, - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -72,6 +72,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name?.normalize('NFC'); let driveFile; if (ps.fileId) { driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -83,22 +84,22 @@ export default class extends Endpoint { // eslint- emojiId = ps.id; const emoji = await this.customEmojiService.getEmojiById(ps.id); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); - if (ps.name && (ps.name !== emoji.name)) { - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + if (nameNfc && (nameNfc !== emoji.name)) { + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists); } } else { - if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); - const emoji = await this.customEmojiService.getEmojiByName(ps.name); + if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); + const emoji = await this.customEmojiService.getEmojiByName(nameNfc); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); emojiId = emoji.id; } await this.customEmojiService.update(emojiId, { driveFile, - name: ps.name, - category: ps.category, - aliases: ps.aliases, + name: nameNfc, + category: ps.category?.normalize('NFC'), + aliases: ps.aliases?.map(a => a.normalize('NFC')), license: ps.license, isSensitive: ps.isSensitive, localOnly: ps.localOnly, diff --git a/packages/frontend/src/components/MkAutocomplete.vue b/packages/frontend/src/components/MkAutocomplete.vue index 8b665bfac..b04a1c92e 100644 --- a/packages/frontend/src/components/MkAutocomplete.vue +++ b/packages/frontend/src/components/MkAutocomplete.vue @@ -238,7 +238,7 @@ function exec() { return; } - emojis.value = searchEmoji(props.q.toLowerCase(), emojiDb.value); + emojis.value = searchEmoji(props.q.normalize('NFC').toLowerCase(), emojiDb.value); } else if (props.type === 'mfmTag') { if (!props.q || props.q === '') { mfmTags.value = MFM_TAGS; diff --git a/packages/frontend/src/components/MkEmojiPicker.vue b/packages/frontend/src/components/MkEmojiPicker.vue index 1219a29d8..53fca97fc 100644 --- a/packages/frontend/src/components/MkEmojiPicker.vue +++ b/packages/frontend/src/components/MkEmojiPicker.vue @@ -205,7 +205,7 @@ watch(q, () => { return; } - const newQ = q.value.replace(/:/g, '').toLowerCase(); + const newQ = q.value.replace(/:/g, '').normalize('NFC').toLowerCase(); const searchCustom = () => { const max = 100; diff --git a/packages/frontend/src/pages/emoji-edit-dialog.vue b/packages/frontend/src/pages/emoji-edit-dialog.vue index 64960fd06..d03d3d912 100644 --- a/packages/frontend/src/pages/emoji-edit-dialog.vue +++ b/packages/frontend/src/pages/emoji-edit-dialog.vue @@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only {{ i18n.ts.selectFile }} - + diff --git a/packages/frontend/src/scripts/autocomplete.ts b/packages/frontend/src/scripts/autocomplete.ts index 9fc8f7843..d942402ff 100644 --- a/packages/frontend/src/scripts/autocomplete.ts +++ b/packages/frontend/src/scripts/autocomplete.ts @@ -99,7 +99,7 @@ export class Autocomplete { const isHashtag = hashtagIndex !== -1; const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' '); const isMfmTag = mfmTagIndex !== -1 && !isMfmParam; - const isEmoji = emojiIndex !== -1 && text.split(/:[a-z0-9_+\-]+:/).pop()!.includes(':'); + const isEmoji = emojiIndex !== -1 && text.split(/:[\p{Letter}\p{Number}\p{Mark}_+-]+:/u).pop()!.includes(':'); let opened = false; @@ -125,7 +125,7 @@ export class Autocomplete { if (isEmoji && !opened && this.onlyType.includes('emoji')) { const emoji = text.substring(emojiIndex + 1); if (!emoji.includes(' ')) { - this.open('emoji', emoji); + this.open('emoji', emoji.normalize('NFC')); opened = true; } } From b6f41a28edccf4d5e08457886ce31827a7810ffc Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 7 Apr 2024 16:37:31 +0100 Subject: [PATCH 2/2] pull in sfm-js that supports non-ascii in emoji names --- packages/backend/package.json | 2 +- packages/frontend/package.json | 4 ++-- pnpm-lock.yaml | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 2ddb067af..d973d527c 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -88,7 +88,7 @@ "@smithy/node-http-handler": "2.1.10", "@swc/cli": "0.1.63", "@swc/core": "1.3.107", - "@transfem-org/sfm-js": "0.24.4", + "@transfem-org/sfm-js": "0.24.5", "@twemoji/parser": "15.0.0", "accepts": "1.3.8", "ajv": "8.12.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 72b26961c..d3249ca81 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -21,12 +21,12 @@ "@github/webauthn-json": "2.1.1", "@mcaptcha/vanilla-glue": "0.1.0-alpha-3", "@misskey-dev/browser-image-resizer": "2024.1.0", + "@phosphor-icons/web": "^2.0.3", "@rollup/plugin-json": "6.1.0", "@rollup/plugin-replace": "5.0.5", "@rollup/pluginutils": "5.1.0", - "@transfem-org/sfm-js": "0.24.4", "@syuilo/aiscript": "0.17.0", - "@phosphor-icons/web": "^2.0.3", + "@transfem-org/sfm-js": "0.24.5", "@twemoji/parser": "15.0.0", "@vitejs/plugin-vue": "5.0.4", "@vue/compiler-sfc": "3.4.21", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b54383a88..eaa60b2a6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -140,8 +140,8 @@ importers: specifier: 1.3.107 version: 1.3.107 '@transfem-org/sfm-js': - specifier: 0.24.4 - version: 0.24.4 + specifier: 0.24.5 + version: 0.24.5 '@twemoji/parser': specifier: 15.0.0 version: 15.0.0 @@ -709,8 +709,8 @@ importers: specifier: 0.17.0 version: 0.17.0 '@transfem-org/sfm-js': - specifier: 0.24.4 - version: 0.24.4 + specifier: 0.24.5 + version: 0.24.5 '@twemoji/parser': specifier: 15.0.0 version: 15.0.0 @@ -7436,8 +7436,8 @@ packages: resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} dev: false - /@transfem-org/sfm-js@0.24.4: - resolution: {integrity: sha1-0wEXqL5UJseGFO4GGFRrES6NCDk=, tarball: https://activitypub.software/api/v4/projects/2/packages/npm/@transfem-org/sfm-js/-/@transfem-org/sfm-js-0.24.4.tgz} + /@transfem-org/sfm-js@0.24.5: + resolution: {integrity: sha1-c9qJO12lIG+kovDGKjZmK2qPqcw=, tarball: https://activitypub.software/api/v4/projects/2/packages/npm/@transfem-org/sfm-js/-/@transfem-org/sfm-js-0.24.5.tgz} dependencies: '@twemoji/parser': 15.0.0 dev: false