graphite/Sharkey
1
0
Fork 0
mirror of https://git.joinsharkey.org/Sharkey/Sharkey.git synced 2024-09-23 02:01:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

handle non-ASCII emoji names

Browse source 
* use the more inclusive regexp for validating emoji names
* always normalize emoji names, aliases, categories

the latter point is necessary to allow matching, for example, `ä`
against `a`+combining diaeresis

this will also need to bump the version of `sfm-js` once we merge
https://activitypub.software/TransFem-org/sfm-js/-/merge_requests/2
This commit is contained in:
dakkar 2024-03-09 12:17:48 +00:00
parent d1f0fc6d5d
commit 354cb2a675
15 changed files with 44 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts
View file

@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
}); });
for (const emoji of customEmojis) { for (const emoji of customEmojis) {
if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) { if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
this.logger.error(`invalid emoji name: ${emoji.name}`); this.logger.error(`invalid emoji name: ${emoji.name}`);
continue; continue;
} }

13
packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts
View file

@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
continue; continue;
} }
const emojiInfo = record.emoji; const emojiInfo = record.emoji;
if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) { const nameNfc = emojiInfo.name.normalize('NFC');
this.logger.error(`invalid emojiname: ${emojiInfo.name}`); if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
this.logger.error(`invalid emojiname: ${nameNfc}`);
continue; continue;
} }
const emojiPath = outputPath + '/' + record.fileName; const emojiPath = outputPath + '/' + record.fileName;
await this.emojisRepository.delete({ await this.emojisRepository.delete({
name: emojiInfo.name, name: nameNfc,
}); });
const driveFile = await this.driveService.addFile({ const driveFile = await this.driveService.addFile({
user: null, user: null,
@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
force: true, force: true,
}); });
await this.customEmojiService.add({ await this.customEmojiService.add({
name: emojiInfo.name, name: nameNfc,
category: emojiInfo.category, category: emojiInfo.category?.normalize('NFC'),
host: null, host: null,
aliases: emojiInfo.aliases, aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
driveFile, driveFile,
license: emojiInfo.license, license: emojiInfo.license,
isSensitive: emojiInfo.isSensitive, isSensitive: emojiInfo.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases); await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
}); });
} }
} }

11
packages/backend/src/server/api/endpoints/admin/emoji/add.ts
View file

@ -40,7 +40,7 @@ export const meta = {
export const paramDef = { export const paramDef = {
type: 'object', type: 'object',
properties: { properties: {
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' }, fileId: { type: 'string', format: 'misskey:id' },
category: { category: {
type: 'string', type: 'string',
@ -73,18 +73,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private emojiEntityService: EmojiEntityService, private emojiEntityService: EmojiEntityService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name.normalize('NFC');
const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
if (driveFile == null) throw new ApiError(meta.errors.noSuchFile); if (driveFile == null) throw new ApiError(meta.errors.noSuchFile);
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null }); if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null });
const emoji = await this.customEmojiService.add({ const emoji = await this.customEmojiService.add({
driveFile, driveFile,
name: ps.name, name: nameNfc,
category: ps.category ?? null, category: ps.category?.normalize('NFC') ?? null,
aliases: ps.aliases ?? [], aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [],
host: null, host: null,
license: ps.license ?? null, license: ps.license ?? null,
isSensitive: ps.isSensitive ?? false, isSensitive: ps.isSensitive ?? false,

9
packages/backend/src/server/api/endpoints/admin/emoji/copy.ts
View file

@ -82,15 +82,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError(); throw new ApiError();
} }
const nameNfc = emoji.name.normalize('NFC');
// Duplication Check // Duplication Check
const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name); const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
const addedEmoji = await this.customEmojiService.add({ const addedEmoji = await this.customEmojiService.add({
driveFile, driveFile,
name: emoji.name, name: nameNfc,
category: emoji.category, category: emoji.category?.normalize('NFC'),
aliases: emoji.aliases, aliases: emoji.aliases?.map(a => a.normalize('NFC')),
host: null, host: null,
license: emoji.license, license: emoji.license,
isSensitive: emoji.isSensitive, isSensitive: emoji.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts
View file

@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
} }
if (ps.query) { if (ps.query) {
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }) q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
.orderBy('length(emoji.name)', 'ASC'); .orderBy('length(emoji.name)', 'ASC');
} }

11
packages/backend/src/server/api/endpoints/admin/emoji/list.ts
View file

@ -92,17 +92,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//const emojis = await q.limit(ps.limit).getMany(); //const emojis = await q.limit(ps.limit).getMany();
emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany(); emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g); const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
if (queryarry) { if (queryarry) {
emojis = emojis.filter(emoji => emojis = emojis.filter(emoji =>
queryarry.includes(`:${emoji.name}:`), queryarry.includes(`:${emoji.name.normalize('NFC')}:`),
); );
} else { } else {
const queryNfc = ps.query!.normalize('NFC');
emojis = emojis.filter(emoji => emojis = emojis.filter(emoji =>
emoji.name.includes(ps.query!) || emoji.name.includes(queryNfc) ||
emoji.aliases.some(a => a.includes(ps.query!)) || emoji.aliases.some(a => a.includes(queryNfc)) ||
emoji.category?.includes(ps.query!)); emoji.category?.includes(queryNfc));
} }
emojis.splice(ps.limit + 1); emojis.splice(ps.limit + 1);
} else { } else {

2
packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases); await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
}); });
} }
} }

2
packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases); await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
}); });
} }
} }

2
packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts
View file

@ -36,7 +36,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null); await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null);
}); });
} }
} }

17
packages/backend/src/server/api/endpoints/admin/emoji/update.ts
View file

@ -40,7 +40,7 @@ export const paramDef = {
type: 'object', type: 'object',
properties: { properties: {
id: { type: 'string', format: 'misskey:id' }, id: { type: 'string', format: 'misskey:id' },
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' }, fileId: { type: 'string', format: 'misskey:id' },
category: { category: {
type: 'string', type: 'string',
@ -72,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name?.normalize('NFC');
let driveFile; let driveFile;
if (ps.fileId) { if (ps.fileId) {
driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
@ -83,22 +84,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
emojiId = ps.id; emojiId = ps.id;
const emoji = await this.customEmojiService.getEmojiById(ps.id); const emoji = await this.customEmojiService.getEmojiById(ps.id);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
if (ps.name && (ps.name !== emoji.name)) { if (nameNfc && (nameNfc !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists); if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists);
} }
} else { } else {
if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(ps.name); const emoji = await this.customEmojiService.getEmojiByName(nameNfc);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
emojiId = emoji.id; emojiId = emoji.id;
} }
await this.customEmojiService.update(emojiId, { await this.customEmojiService.update(emojiId, {
driveFile, driveFile,
name: ps.name, name: nameNfc,
category: ps.category, category: ps.category?.normalize('NFC'),
aliases: ps.aliases, aliases: ps.aliases?.map(a => a.normalize('NFC')),
license: ps.license, license: ps.license,
isSensitive: ps.isSensitive, isSensitive: ps.isSensitive,
localOnly: ps.localOnly, localOnly: ps.localOnly,

2
packages/frontend/src/components/MkAutocomplete.vue
View file

@ -238,7 +238,7 @@ function exec() {
return; return;
} }
emojis.value = searchEmoji(props.q.toLowerCase(), emojiDb.value); emojis.value = searchEmoji(props.q.normalize('NFC').toLowerCase(), emojiDb.value);
} else if (props.type === 'mfmTag') { } else if (props.type === 'mfmTag') {
if (!props.q || props.q === '') { if (!props.q || props.q === '') {
mfmTags.value = MFM_TAGS; mfmTags.value = MFM_TAGS;

2
packages/frontend/src/components/MkEmojiPicker.vue
View file

@ -205,7 +205,7 @@ watch(q, () => {
return; return;
} }
const newQ = q.value.replace(/:/g, '').toLowerCase(); const newQ = q.value.replace(/:/g, '').normalize('NFC').toLowerCase();
const searchCustom = () => { const searchCustom = () => {
const max = 100; const max = 100;

2
packages/frontend/src/pages/emoji-edit-dialog.vue
View file

@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only
</div> </div>
</div> </div>
<MkButton rounded style="margin: 0 auto;" @click="changeImage">{{ i18n.ts.selectFile }}</MkButton> <MkButton rounded style="margin: 0 auto;" @click="changeImage">{{ i18n.ts.selectFile }}</MkButton>
<MkInput v-model="name" pattern="[a-z0-9_]" autocapitalize="off"> <MkInput v-model="name" autocapitalize="off">
<template #label>{{ i18n.ts.name }}</template> <template #label>{{ i18n.ts.name }}</template>
</MkInput> </MkInput>
<MkInput v-model="category" :datalist="customEmojiCategories"> <MkInput v-model="category" :datalist="customEmojiCategories">

4
packages/frontend/src/scripts/autocomplete.ts
View file

@ -99,7 +99,7 @@ export class Autocomplete {
const isHashtag = hashtagIndex !== -1; const isHashtag = hashtagIndex !== -1;
const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' '); const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' ');
const isMfmTag = mfmTagIndex !== -1 && !isMfmParam; const isMfmTag = mfmTagIndex !== -1 && !isMfmParam;
const isEmoji = emojiIndex !== -1 && text.split(/:[a-z0-9_+\-]+:/).pop()!.includes(':'); const isEmoji = emojiIndex !== -1 && text.split(/:[\p{Letter}\p{Number}\p{Mark}_+-]+:/u).pop()!.includes(':');
let opened = false; let opened = false;
@ -125,7 +125,7 @@ export class Autocomplete {
if (isEmoji && !opened && this.onlyType.includes('emoji')) { if (isEmoji && !opened && this.onlyType.includes('emoji')) {
const emoji = text.substring(emojiIndex + 1); const emoji = text.substring(emojiIndex + 1);
if (!emoji.includes(' ')) { if (!emoji.includes(' ')) {
this.open('emoji', emoji); this.open('emoji', emoji.normalize('NFC'));
opened = true; opened = true;
} }
} }