From 8a416cd30268a82b3d55b8f6c07bd20ce8602926 Mon Sep 17 00:00:00 2001 From: Essem Date: Sat, 3 Feb 2024 16:45:38 -0600 Subject: [PATCH] feat: Add langPref config option --- .config/ci.yml | 8 ++- .config/docker_example.yml | 8 ++- .config/example.yml | 4 ++ packages/backend/src/config.ts | 4 ++ .../core/activitypub/models/ApNoteService.ts | 62 +++++++++++++------ packages/backend/src/misc/langmap.ts | 1 + .../backend/src/models/json-schema/note.ts | 4 +- .../src/server/api/endpoints/i/update.ts | 6 +- .../src/server/api/endpoints/notes/create.ts | 4 +- .../src/server/api/endpoints/notes/edit.ts | 4 +- .../frontend/src/components/MkPostForm.vue | 8 +-- .../frontend/src/pages/settings/profile.vue | 4 +- packages/frontend/src/scripts/langmap.ts | 1 + 13 files changed, 78 insertions(+), 40 deletions(-) diff --git a/.config/ci.yml b/.config/ci.yml index c48fca49b..5fc2c787a 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -106,7 +106,7 @@ redis: # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── -# You can set scope to local (default value) or global +# You can set scope to local (default value) or global # (include notes from remote). #meilisearch: @@ -204,7 +204,7 @@ signToActivityPubGet: true checkActivityPubGetSignature: false # For security reasons, uploading attachments from the intranet is prohibited, -# but exceptions can be made from the following settings. Default value is "undefined". +# but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). #allowedPrivateNetworks: [ # '127.0.0.1/32' @@ -212,5 +212,9 @@ checkActivityPubGetSignature: false #customMOTD: ['Hello World', 'The sharks rule all', 'Shonks'] +# Prefer these languages for remote notes with language-specific content +# Must be valid language codes according to BCP 47 +#langPref: ['en', 'ja'] + # Upload or download file size limits (bytes) #maxFileSize: 262144000 diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 296237c95..4aa2dc9a1 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -163,7 +163,7 @@ redis: # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── -# You can set scope to local (default value) or global +# You can set scope to local (default value) or global # (include notes from remote). #meilisearch: @@ -261,7 +261,7 @@ signToActivityPubGet: true checkActivityPubGetSignature: false # For security reasons, uploading attachments from the intranet is prohibited, -# but exceptions can be made from the following settings. Default value is "undefined". +# but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). #allowedPrivateNetworks: [ # '127.0.0.1/32' @@ -269,5 +269,9 @@ checkActivityPubGetSignature: false #customMOTD: ['Hello World', 'The sharks rule all', 'Shonks'] +# Prefer these languages for remote notes with language-specific content +# Must be valid language codes according to BCP 47 +#langPref: ['en', 'ja'] + # Upload or download file size limits (bytes) #maxFileSize: 262144000 diff --git a/.config/example.yml b/.config/example.yml index c037a280b..9af68272d 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -284,6 +284,10 @@ checkActivityPubGetSignature: false #customMOTD: ['Hello World', 'The sharks rule all', 'Shonks'] +# Prefer these languages for remote notes with language-specific content +# Must be valid language codes according to BCP 47 +#langPref: ['en', 'ja'] + # Upload or download file size limits (bytes) #maxFileSize: 262144000 diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index c99bc7ae0..2fa736951 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -91,6 +91,8 @@ type Source = { customMOTD?: string[]; + langPref?: string[]; + signToActivityPubGet?: boolean; checkActivityPubGetSignature?: boolean; @@ -153,6 +155,7 @@ export type Config = { customMOTD: string[] | undefined; signToActivityPubGet: boolean; checkActivityPubGetSignature: boolean | undefined; + langPref: string[]; version: string; publishTarballInsteadOfProvideRepositoryUrl: boolean; @@ -269,6 +272,7 @@ export function loadConfig(): Config { inboxJobMaxAttempts: config.inboxJobMaxAttempts, proxyRemoteFiles: config.proxyRemoteFiles, customMOTD: config.customMOTD, + langPref: config.langPref ?? ['en', 'ja'], signToActivityPubGet: config.signToActivityPubGet ?? true, checkActivityPubGetSignature: config.checkActivityPubGetSignature, mediaProxy: externalMediaProxy ?? internalMediaProxy, diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 204846f7d..ded331e75 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -25,7 +25,7 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { checkHttps } from '@/misc/check-https.js'; -import { langmap } from '@/misc/langmap.js'; +import { langs } from '@/misc/langmap.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; @@ -241,25 +241,37 @@ export class ApNoteService { const cw = note.summary === '' ? null : note.summary; + let lang: string | null = null; + if (note.contentMap != null) { + for (const preferredLang of this.config.langPref) { + if (note.contentMap[preferredLang]) { + lang = preferredLang; + break; + } + } + + if (!lang) lang = Object.keys(note.contentMap)[0]; + if (!langs.includes(lang)) lang = null; + } + // テキストのパース let text: string | null = null; if (note.source?.mediaType === 'text/x.misskeymarkdown' && typeof note.source.content === 'string') { text = note.source.content; - } else if (note.contentMap != null) { - const entry = Object.entries(note.contentMap)[0]; - text = this.apMfmService.htmlToMfm(entry[1], note.tag); + } else if (note.contentMap != null && Object.keys(note.contentMap).length !== 0) { + let content: string; + if (lang) { + content = note.contentMap[lang]; + } else { + content = Object.values(note.contentMap)[0]; + } + text = this.apMfmService.htmlToMfm(content, note.tag); } else if (typeof note._misskey_content !== 'undefined') { text = note._misskey_content; } else if (typeof note.content === 'string') { text = this.apMfmService.htmlToMfm(note.content, note.tag); } - let lang: string | null = null; - if (note.contentMap != null) { - const key = Object.keys(note.contentMap)[0]; - lang = Object.keys(langmap).includes(key) ? key : null; - } - // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); @@ -459,25 +471,37 @@ export class ApNoteService { const cw = note.summary === '' ? null : note.summary; + let lang: string | null = null; + if (note.contentMap != null) { + for (const preferredLang of this.config.langPref) { + if (note.contentMap[preferredLang]) { + lang = preferredLang; + break; + } + } + + if (!lang) lang = Object.keys(note.contentMap)[0]; + if (!langs.includes(lang)) lang = null; + } + // テキストのパース let text: string | null = null; if (note.source?.mediaType === 'text/x.misskeymarkdown' && typeof note.source.content === 'string') { text = note.source.content; - } else if (note.contentMap != null) { - const entry = Object.entries(note.contentMap)[0]; - text = this.apMfmService.htmlToMfm(entry[1], note.tag); + } else if (note.contentMap != null && Object.keys(note.contentMap).length !== 0) { + let content: string; + if (lang) { + content = note.contentMap[lang]; + } else { + content = Object.values(note.contentMap)[0]; + } + text = this.apMfmService.htmlToMfm(content, note.tag); } else if (typeof note._misskey_content !== 'undefined') { text = note._misskey_content; } else if (typeof note.content === 'string') { text = this.apMfmService.htmlToMfm(note.content, note.tag); } - let lang: string | null = null; - if (note.contentMap != null) { - const key = Object.keys(note.contentMap)[0]; - lang = Object.keys(langmap).includes(key) ? key : null; - } - // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); diff --git a/packages/backend/src/misc/langmap.ts b/packages/backend/src/misc/langmap.ts index 6b6e8b5b7..f7e237d30 100644 --- a/packages/backend/src/misc/langmap.ts +++ b/packages/backend/src/misc/langmap.ts @@ -385,3 +385,4 @@ export const iso639Regional = { }; export const langmap = Object.assign({}, langmapNoRegion, iso639Regional); +export const langs = Object.keys(langmap); diff --git a/packages/backend/src/models/json-schema/note.ts b/packages/backend/src/models/json-schema/note.ts index 766ff411f..5a7a1deda 100644 --- a/packages/backend/src/models/json-schema/note.ts +++ b/packages/backend/src/models/json-schema/note.ts @@ -2,7 +2,7 @@ * SPDX-FileCopyrightText: syuilo and misskey-project * SPDX-License-Identifier: AGPL-3.0-only */ -import { langmap } from '@/misc/langmap.js'; +import { langs } from '@/misc/langmap.js'; export const packedNoteSchema = { type: 'object', @@ -29,7 +29,7 @@ export const packedNoteSchema = { }, lang: { type: 'string', - enum: [...Object.keys(langmap)], + enum: langs, nullable: true, }, cw: { diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 2c04b6573..a48c74420 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -17,7 +17,7 @@ import { birthdaySchema, listenbrainzSchema, descriptionSchema, locationSchema, import type { MiUserProfile } from '@/models/UserProfile.js'; import { notificationTypes } from '@/types.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; -import { langmap } from '@/misc/langmap.js'; +import { langs } from '@/misc/langmap.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; @@ -149,7 +149,7 @@ export const paramDef = { location: { ...locationSchema, nullable: true }, birthday: { ...birthdaySchema, nullable: true }, listenbrainz: { ...listenbrainzSchema, nullable: true }, - lang: { type: 'string', enum: [null, ...Object.keys(langmap)] as string[], nullable: true }, + lang: { type: 'string', enum: [null, ...langs] as string[], nullable: true }, avatarId: { type: 'string', format: 'misskey:id', nullable: true }, avatarDecorations: { type: 'array', maxItems: 16, items: { type: 'object', @@ -382,7 +382,7 @@ export default class extends Endpoint { // eslint- updates.backgroundUrl = null; updates.backgroundBlurhash = null; } - + if (ps.avatarDecorations) { const decorations = await this.avatarDecorationService.getAll(true); const [myRoles, myPolicies] = await Promise.all([this.roleService.getUserRoles(user.id), this.roleService.getUserPolicies(user.id)]); diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 3c6714235..96ff1de26 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -20,7 +20,7 @@ import { isPureRenote } from '@/misc/is-pure-renote.js'; import { MetaService } from '@/core/MetaService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { langmap } from '@/misc/langmap.js'; +import { langs } from '@/misc/langmap.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -137,7 +137,7 @@ export const paramDef = { visibleUserIds: { type: 'array', uniqueItems: true, items: { type: 'string', format: 'misskey:id', } }, - lang: { type: 'string', enum: Object.keys(langmap), nullable: true, maxLength: 10 }, + lang: { type: 'string', enum: langs, nullable: true, maxLength: 10 }, cw: { type: 'string', nullable: true, minLength: 1, maxLength: 500 }, localOnly: { type: 'boolean', default: false }, reactionAcceptance: { type: 'string', nullable: true, enum: [null, 'likeOnly', 'likeOnlyForRemote', 'nonSensitiveOnly', 'nonSensitiveOnlyForLocalLikeOnlyForRemote'], default: null }, diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index fa2225667..622965239 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -12,7 +12,7 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { NoteEditService } from '@/core/NoteEditService.js'; import { DI } from '@/di-symbols.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { langmap } from '@/misc/langmap.js'; +import { langs } from '@/misc/langmap.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -165,7 +165,7 @@ export const paramDef = { format: 'misskey:id', }, }, - lang: { type: 'string', enum: Object.keys(langmap), nullable: true, maxLength: 10 }, + lang: { type: 'string', enum: langs, nullable: true, maxLength: 10 }, cw: { type: 'string', nullable: true, minLength: 1, maxLength: 500 }, localOnly: { type: 'boolean', default: false }, reactionAcceptance: { type: 'string', nullable: true, enum: [null, 'likeOnly', 'likeOnlyForRemote', 'nonSensitiveOnly', 'nonSensitiveOnlyForLocalLikeOnlyForRemote'], default: null }, diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index 251493480..18d18ed19 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -134,7 +134,7 @@ import { miLocalStorage } from '@/local-storage.js'; import { claimAchievement } from '@/scripts/achievements.js'; import { emojiPicker } from '@/scripts/emoji-picker.js'; import { mfmFunctionPicker } from '@/scripts/mfm-function-picker.js'; -import { langmap } from '@/scripts/langmap.js'; +import { langmap, langs } from '@/scripts/langmap.js'; import { MenuItem } from '@/types/menu.js'; const $i = signinRequired(); @@ -547,7 +547,6 @@ async function toggleReactionAcceptance() { function attemptNormalizeLang(lang: string | null) { if (lang == null) return null; - const langs = Object.keys(langmap); if (!langs[lang]) lang = lang.split('-')[0]; return lang; } @@ -562,8 +561,6 @@ function setLanguage(ev: MouseEvent) { action: () => {}, }); - const langs = Object.keys(langmap); - // Show recently used language first let recentlyUsedLanguagesExist = false; for (const lang of defaultStore.state.recentlyUsedPostLanguages) { @@ -959,11 +956,10 @@ async function post(ev?: MouseEvent) { // update recentlyUsedLanguages if (language.value != null) { - const languages = Object.keys(langmap); const maxLength = 6; defaultStore.set('recentlyUsedPostLanguages', [language.value].concat(defaultStore.state.recentlyUsedPostLanguages.filter((lang) => { - return (lang !== language.value && languages.includes(lang)); + return (lang !== language.value && langs.includes(lang)); })).slice(0, maxLength)); } } diff --git a/packages/frontend/src/pages/settings/profile.vue b/packages/frontend/src/pages/settings/profile.vue index 408cf4ed6..2a55c7093 100644 --- a/packages/frontend/src/pages/settings/profile.vue +++ b/packages/frontend/src/pages/settings/profile.vue @@ -45,7 +45,7 @@ SPDX-License-Identifier: AGPL-3.0-only - + @@ -128,7 +128,7 @@ import { selectFile } from '@/scripts/select-file.js'; import * as os from '@/os.js'; import { i18n } from '@/i18n.js'; import { signinRequired } from '@/account.js'; -import { langmap } from '@/scripts/langmap.js'; +import { langmap, langs } from '@/scripts/langmap.js'; import { definePageMetadata } from '@/scripts/page-metadata.js'; import { claimAchievement } from '@/scripts/achievements.js'; import { defaultStore } from '@/store.js'; diff --git a/packages/frontend/src/scripts/langmap.ts b/packages/frontend/src/scripts/langmap.ts index eb23fe2d7..9415d0293 100644 --- a/packages/frontend/src/scripts/langmap.ts +++ b/packages/frontend/src/scripts/langmap.ts @@ -385,3 +385,4 @@ export const iso639Regional = { }; export const langmap = Object.assign({}, langmapNoRegion, iso639Regional); +export const langs = Object.keys(langmap);