graphite/Sharkey
1
0
Fork 0
mirror of https://git.joinsharkey.org/Sharkey/Sharkey.git synced 2024-11-22 21:53:08 +02:00
Code Issues Projects Releases Packages Wiki Activity

merge: handle non-ASCII emoji names (!464)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/464
This commit is contained in:
dakkar 2024-04-07 15:37:22 +00:00
commit 70199a8fab
18 changed files with 53 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/package.json
View file

@ -88,7 +88,7 @@
"@smithy/node-http-handler": "2.1.10", "@smithy/node-http-handler": "2.1.10",
"@swc/cli": "0.1.63", "@swc/cli": "0.1.63",
"@swc/core": "1.3.107", "@swc/core": "1.3.107",
"@transfem-org/sfm-js": "0.24.4", "@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.0.0", "@twemoji/parser": "15.0.0",
"accepts": "1.3.8", "accepts": "1.3.8",
"ajv": "8.12.0", "ajv": "8.12.0",

2
packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts
View file

@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
}); });
for (const emoji of customEmojis) { for (const emoji of customEmojis) {
if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) { if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
this.logger.error(`invalid emoji name: ${emoji.name}`); this.logger.error(`invalid emoji name: ${emoji.name}`);
continue; continue;
} }

13
packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts
View file

@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
continue; continue;
} }
const emojiInfo = record.emoji; const emojiInfo = record.emoji;
if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) { const nameNfc = emojiInfo.name.normalize('NFC');
this.logger.error(`invalid emojiname: ${emojiInfo.name}`); if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
this.logger.error(`invalid emojiname: ${nameNfc}`);
continue; continue;
} }
const emojiPath = outputPath + '/' + record.fileName; const emojiPath = outputPath + '/' + record.fileName;
await this.emojisRepository.delete({ await this.emojisRepository.delete({
name: emojiInfo.name, name: nameNfc,
}); });
const driveFile = await this.driveService.addFile({ const driveFile = await this.driveService.addFile({
user: null, user: null,
@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
force: true, force: true,
}); });
await this.customEmojiService.add({ await this.customEmojiService.add({
name: emojiInfo.name, name: nameNfc,
category: emojiInfo.category, category: emojiInfo.category?.normalize('NFC'),
host: null, host: null,
aliases: emojiInfo.aliases, aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
driveFile, driveFile,
license: emojiInfo.license, license: emojiInfo.license,
isSensitive: emojiInfo.isSensitive, isSensitive: emojiInfo.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases); await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
}); });
} }
} }

11
packages/backend/src/server/api/endpoints/admin/emoji/add.ts
View file

@ -40,7 +40,7 @@ export const meta = {
export const paramDef = { export const paramDef = {
type: 'object', type: 'object',
properties: { properties: {
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' }, fileId: { type: 'string', format: 'misskey:id' },
category: { category: {
type: 'string', type: 'string',
@ -73,18 +73,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private emojiEntityService: EmojiEntityService, private emojiEntityService: EmojiEntityService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name.normalize('NFC');
const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
if (driveFile == null) throw new ApiError(meta.errors.noSuchFile); if (driveFile == null) throw new ApiError(meta.errors.noSuchFile);
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null }); if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null });
const emoji = await this.customEmojiService.add({ const emoji = await this.customEmojiService.add({
driveFile, driveFile,
name: ps.name, name: nameNfc,
category: ps.category ?? null, category: ps.category?.normalize('NFC') ?? null,
aliases: ps.aliases ?? [], aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [],
host: null, host: null,
license: ps.license ?? null, license: ps.license ?? null,
isSensitive: ps.isSensitive ?? false, isSensitive: ps.isSensitive ?? false,

9
packages/backend/src/server/api/endpoints/admin/emoji/copy.ts
View file

@ -82,15 +82,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError(); throw new ApiError();
} }
const nameNfc = emoji.name.normalize('NFC');
// Duplication Check // Duplication Check
const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name); const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
const addedEmoji = await this.customEmojiService.add({ const addedEmoji = await this.customEmojiService.add({
driveFile, driveFile,
name: emoji.name, name: nameNfc,
category: emoji.category, category: emoji.category?.normalize('NFC'),
aliases: emoji.aliases, aliases: emoji.aliases?.map(a => a.normalize('NFC')),
host: null, host: null,
license: emoji.license, license: emoji.license,
isSensitive: emoji.isSensitive, isSensitive: emoji.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts
View file

@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
} }
if (ps.query) { if (ps.query) {
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }) q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
.orderBy('length(emoji.name)', 'ASC'); .orderBy('length(emoji.name)', 'ASC');
} }

11
packages/backend/src/server/api/endpoints/admin/emoji/list.ts
View file

@ -92,17 +92,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//const emojis = await q.limit(ps.limit).getMany(); //const emojis = await q.limit(ps.limit).getMany();
emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany(); emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g); const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
if (queryarry) { if (queryarry) {
emojis = emojis.filter(emoji => emojis = emojis.filter(emoji =>
queryarry.includes(`:${emoji.name}:`), queryarry.includes(`:${emoji.name.normalize('NFC')}:`),
); );
} else { } else {
const queryNfc = ps.query!.normalize('NFC');
emojis = emojis.filter(emoji => emojis = emojis.filter(emoji =>
emoji.name.includes(ps.query!) || emoji.name.includes(queryNfc) ||
emoji.aliases.some(a => a.includes(ps.query!)) || emoji.aliases.some(a => a.includes(queryNfc)) ||
emoji.category?.includes(ps.query!)); emoji.category?.includes(queryNfc));
} }
emojis.splice(ps.limit + 1); emojis.splice(ps.limit + 1);
} else { } else {

2
packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases); await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
}); });
} }
} }

2
packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases); await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
}); });
} }
} }

2
packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts
View file

@ -36,7 +36,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null); await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null);
}); });
} }
} }

17
packages/backend/src/server/api/endpoints/admin/emoji/update.ts
View file

@ -40,7 +40,7 @@ export const paramDef = {
type: 'object', type: 'object',
properties: { properties: {
id: { type: 'string', format: 'misskey:id' }, id: { type: 'string', format: 'misskey:id' },
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' }, fileId: { type: 'string', format: 'misskey:id' },
category: { category: {
type: 'string', type: 'string',
@ -72,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService, private customEmojiService: CustomEmojiService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name?.normalize('NFC');
let driveFile; let driveFile;
if (ps.fileId) { if (ps.fileId) {
driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
@ -83,22 +84,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
emojiId = ps.id; emojiId = ps.id;
const emoji = await this.customEmojiService.getEmojiById(ps.id); const emoji = await this.customEmojiService.getEmojiById(ps.id);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
if (ps.name && (ps.name !== emoji.name)) { if (nameNfc && (nameNfc !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists); if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists);
} }
} else { } else {
if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(ps.name); const emoji = await this.customEmojiService.getEmojiByName(nameNfc);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
emojiId = emoji.id; emojiId = emoji.id;
} }
await this.customEmojiService.update(emojiId, { await this.customEmojiService.update(emojiId, {
driveFile, driveFile,
name: ps.name, name: nameNfc,
category: ps.category, category: ps.category?.normalize('NFC'),
aliases: ps.aliases, aliases: ps.aliases?.map(a => a.normalize('NFC')),
license: ps.license, license: ps.license,
isSensitive: ps.isSensitive, isSensitive: ps.isSensitive,
localOnly: ps.localOnly, localOnly: ps.localOnly,

4
packages/frontend/package.json
View file

@ -21,12 +21,12 @@
"@github/webauthn-json": "2.1.1", "@github/webauthn-json": "2.1.1",
"@mcaptcha/vanilla-glue": "0.1.0-alpha-3", "@mcaptcha/vanilla-glue": "0.1.0-alpha-3",
"@misskey-dev/browser-image-resizer": "2024.1.0", "@misskey-dev/browser-image-resizer": "2024.1.0",
"@phosphor-icons/web": "^2.0.3",
"@rollup/plugin-json": "6.1.0", "@rollup/plugin-json": "6.1.0",
"@rollup/plugin-replace": "5.0.5", "@rollup/plugin-replace": "5.0.5",
"@rollup/pluginutils": "5.1.0", "@rollup/pluginutils": "5.1.0",
"@transfem-org/sfm-js": "0.24.4",
"@syuilo/aiscript": "0.17.0", "@syuilo/aiscript": "0.17.0",
"@phosphor-icons/web": "^2.0.3", "@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.0.0", "@twemoji/parser": "15.0.0",
"@vitejs/plugin-vue": "5.0.4", "@vitejs/plugin-vue": "5.0.4",
"@vue/compiler-sfc": "3.4.21", "@vue/compiler-sfc": "3.4.21",

2
packages/frontend/src/components/MkAutocomplete.vue
View file

@ -238,7 +238,7 @@ function exec() {
return; return;
} }
emojis.value = searchEmoji(props.q.toLowerCase(), emojiDb.value); emojis.value = searchEmoji(props.q.normalize('NFC').toLowerCase(), emojiDb.value);
} else if (props.type === 'mfmTag') { } else if (props.type === 'mfmTag') {
if (!props.q || props.q === '') { if (!props.q || props.q === '') {
mfmTags.value = MFM_TAGS; mfmTags.value = MFM_TAGS;

2
packages/frontend/src/components/MkEmojiPicker.vue
View file

@ -205,7 +205,7 @@ watch(q, () => {
return; return;
} }
const newQ = q.value.replace(/:/g, '').toLowerCase(); const newQ = q.value.replace(/:/g, '').normalize('NFC').toLowerCase();
const searchCustom = () => { const searchCustom = () => {
const max = 100; const max = 100;

2
packages/frontend/src/pages/emoji-edit-dialog.vue
View file

@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only
</div> </div>
</div> </div>
<MkButton rounded style="margin: 0 auto;" @click="changeImage">{{ i18n.ts.selectFile }}</MkButton> <MkButton rounded style="margin: 0 auto;" @click="changeImage">{{ i18n.ts.selectFile }}</MkButton>
<MkInput v-model="name" pattern="[a-z0-9_]" autocapitalize="off"> <MkInput v-model="name" autocapitalize="off">
<template #label>{{ i18n.ts.name }}</template> <template #label>{{ i18n.ts.name }}</template>
</MkInput> </MkInput>
<MkInput v-model="category" :datalist="customEmojiCategories"> <MkInput v-model="category" :datalist="customEmojiCategories">

4
packages/frontend/src/scripts/autocomplete.ts
View file

@ -99,7 +99,7 @@ export class Autocomplete {
const isHashtag = hashtagIndex !== -1; const isHashtag = hashtagIndex !== -1;
const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' '); const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' ');
const isMfmTag = mfmTagIndex !== -1 && !isMfmParam; const isMfmTag = mfmTagIndex !== -1 && !isMfmParam;
const isEmoji = emojiIndex !== -1 && text.split(/:[a-z0-9_+\-]+:/).pop()!.includes(':'); const isEmoji = emojiIndex !== -1 && text.split(/:[\p{Letter}\p{Number}\p{Mark}_+-]+:/u).pop()!.includes(':');
let opened = false; let opened = false;
@ -125,7 +125,7 @@ export class Autocomplete {
if (isEmoji && !opened && this.onlyType.includes('emoji')) { if (isEmoji && !opened && this.onlyType.includes('emoji')) {
const emoji = text.substring(emojiIndex + 1); const emoji = text.substring(emojiIndex + 1);
if (!emoji.includes(' ')) { if (!emoji.includes(' ')) {
this.open('emoji', emoji); this.open('emoji', emoji.normalize('NFC'));
opened = true; opened = true;
} }
} }

12
pnpm-lock.yaml
View file

@ -140,8 +140,8 @@ importers:
specifier: 1.3.107 specifier: 1.3.107
version: 1.3.107 version: 1.3.107
'@transfem-org/sfm-js': '@transfem-org/sfm-js':
specifier: 0.24.4 specifier: 0.24.5
version: 0.24.4 version: 0.24.5
'@twemoji/parser': '@twemoji/parser':
specifier: 15.0.0 specifier: 15.0.0
version: 15.0.0 version: 15.0.0
@ -709,8 +709,8 @@ importers:
specifier: 0.17.0 specifier: 0.17.0
version: 0.17.0 version: 0.17.0
'@transfem-org/sfm-js': '@transfem-org/sfm-js':
specifier: 0.24.4 specifier: 0.24.5
version: 0.24.4 version: 0.24.5
'@twemoji/parser': '@twemoji/parser':
specifier: 15.0.0 specifier: 15.0.0
version: 15.0.0 version: 15.0.0
@ -7436,8 +7436,8 @@ packages:
resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
dev: false dev: false
/@transfem-org/sfm-js@0.24.4: /@transfem-org/sfm-js@0.24.5:
resolution: {integrity: sha1-0wEXqL5UJseGFO4GGFRrES6NCDk=, tarball: https://activitypub.software/api/v4/projects/2/packages/npm/@transfem-org/sfm-js/-/@transfem-org/sfm-js-0.24.4.tgz} resolution: {integrity: sha1-c9qJO12lIG+kovDGKjZmK2qPqcw=, tarball: https://activitypub.software/api/v4/projects/2/packages/npm/@transfem-org/sfm-js/-/@transfem-org/sfm-js-0.24.5.tgz}
dependencies: dependencies:
'@twemoji/parser': 15.0.0 '@twemoji/parser': 15.0.0
dev: false dev: false