From 4a72941edaf7273dacd83f7df72ba22a77ef98e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Acid=20Chicken=20=28=E7=A1=AB=E9=85=B8=E9=B6=8F=29?= Date: Fri, 5 May 2023 19:49:34 +0900 Subject: [PATCH] perf: use slacc on check-word-mute (#10721) * perf: use slacc on check-word-mute when all of specified words are single word * perf: use slacc as possible * build: avoid tarball * chore: update slacc * build: update package name --- packages/backend/package.json | 14 +- packages/backend/src/misc/check-word-mute.ts | 20 ++- .../backend/test/unit/misc/check-word-mute.ts | 49 ++++++ pnpm-lock.yaml | 159 +++++++++++++++++- 4 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 packages/backend/test/unit/misc/check-word-mute.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 08557d415..e0ece2bfe 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,7 +34,18 @@ "@swc/core-win32-ia32-msvc": "1.3.56", "@swc/core-win32-x64-msvc": "1.3.56", "@tensorflow/tfjs": "4.4.0", - "@tensorflow/tfjs-node": "4.4.0" + "@tensorflow/tfjs-node": "4.4.0", + "slacc-android-arm-eabi": "0.0.7", + "slacc-android-arm64": "0.0.7", + "slacc-darwin-arm64": "0.0.7", + "slacc-darwin-universal": "0.0.7", + "slacc-darwin-x64": "0.0.7", + "slacc-linux-arm-gnueabihf": "0.0.7", + "slacc-linux-arm64-gnu": "0.0.7", + "slacc-linux-arm64-musl": "0.0.7", + "slacc-linux-x64-gnu": "0.0.7", + "slacc-win32-arm64-msvc": "0.0.7", + "slacc-win32-x64-msvc": "0.0.7" }, "dependencies": { "@aws-sdk/client-s3": "3.321.1", @@ -128,6 +139,7 @@ "semver": "7.5.0", "sharp": "0.32.1", "sharp-read-bmp": "github:misskey-dev/sharp-read-bmp", + "slacc": "0.0.7", "strict-event-emitter-types": "2.0.0", "stringz": "2.1.0", "summaly": "github:misskey-dev/summaly", diff --git a/packages/backend/src/misc/check-word-mute.ts b/packages/backend/src/misc/check-word-mute.ts index e8c66683c..910bebfcf 100644 --- a/packages/backend/src/misc/check-word-mute.ts +++ b/packages/backend/src/misc/check-word-mute.ts @@ -1,3 +1,4 @@ +import { AhoCorasick } from 'slacc'; import RE2 from 're2'; import type { Note } from '@/models/entities/Note.js'; import type { User } from '@/models/entities/User.js'; @@ -12,6 +13,8 @@ type UserLike = { id: User['id']; }; +const acCache = new Map(); + export async function checkWordMute(note: NoteLike, me: UserLike | null | undefined, mutedWords: Array): Promise { // 自分自身 if (me && (note.userId === me.id)) return false; @@ -21,7 +24,22 @@ export async function checkWordMute(note: NoteLike, me: UserLike | null | undefi if (text === '') return false; - const matched = mutedWords.some(filter => { + const acable = mutedWords.filter(filter => Array.isArray(filter) && filter.length === 1).map(filter => filter[0]).sort(); + const unacable = mutedWords.filter(filter => !Array.isArray(filter) || filter.length !== 1); + const acCacheKey = acable.join('\n'); + const ac = acCache.get(acCacheKey) ?? AhoCorasick.withPatterns(acable); + acCache.delete(acCacheKey); + for (const obsoleteKeys of acCache.keys()) { + if (acCache.size > 1000) { + acCache.delete(obsoleteKeys); + } + } + acCache.set(acCacheKey, ac); + if (ac.isMatch(text)) { + return true; + } + + const matched = unacable.some(filter => { if (Array.isArray(filter)) { return filter.every(keyword => text.includes(keyword)); } else { diff --git a/packages/backend/test/unit/misc/check-word-mute.ts b/packages/backend/test/unit/misc/check-word-mute.ts new file mode 100644 index 000000000..7ab838bde --- /dev/null +++ b/packages/backend/test/unit/misc/check-word-mute.ts @@ -0,0 +1,49 @@ +import { checkWordMute } from '@/misc/check-word-mute.js'; + +describe(checkWordMute, () => { + describe('Slacc boost mode', () => { + it('should return false if mutedWords is empty', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [])).toBe(false); + }); + it('should return true if mutedWords is not empty and text contains muted word', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [['foo']])).toBe(true); + }); + it('should return false if mutedWords is not empty and text does not contain muted word', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [['bar']])).toBe(false); + }); + it('should return false when the note is written by me even if mutedWords is not empty and text contains muted word', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo' }, { id: '1' }, [['foo']])).toBe(false); + }); + it('should return true if mutedWords is not empty and text contains muted word in CW', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo', cw: 'bar' }, null, [['bar']])).toBe(true); + }); + it('should return true if mutedWords is not empty and text contains muted word in both CW and text', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo', cw: 'bar' }, null, [['foo'], ['bar']])).toBe(true); + }); + it('should return true if mutedWords is not empty and text does not contain muted word in both CW and text', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo', cw: 'bar' }, null, [['foo'], ['baz']])).toBe(true); + }); + }); + describe('normal mode', () => { + it('should return false if text does not contain muted words', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [['foo', 'bar']])).toBe(false); + }); + it('should return true if text contains muted words', async () => { + expect(await checkWordMute({ userId: '1', text: 'foobar' }, null, [['foo', 'bar']])).toBe(true); + }); + it('should return false when the note is written by me even if text contains muted words', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo bar' }, { id: '1' }, [['foo', 'bar']])).toBe(false); + }); + }); + describe('RegExp mode', () => { + it('should return false if text does not contain muted words', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo' }, null, ['/bar/'])).toBe(false); + }); + it('should return true if text contains muted words', async () => { + expect(await checkWordMute({ userId: '1', text: 'foobar' }, null, ['/bar/'])).toBe(true); + }); + it('should return false when the note is written by me even if text contains muted words', async () => { + expect(await checkWordMute({ userId: '1', text: 'foo bar' }, { id: '1' }, ['/bar/'])).toBe(false); + }); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 31f6b919d..bec090922 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -337,6 +337,9 @@ importers: sharp-read-bmp: specifier: github:misskey-dev/sharp-read-bmp version: github.com/misskey-dev/sharp-read-bmp/02d9dc189fa7df0c4bea09330be26741772dac01 + slacc: + specifier: 0.0.7 + version: 0.0.7 strict-event-emitter-types: specifier: 2.0.0 version: 2.0.0 @@ -345,7 +348,7 @@ importers: version: 2.1.0 summaly: specifier: github:misskey-dev/summaly - version: github.com/misskey-dev/summaly/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e + version: github.com/misskey-dev/summaly/2d63e2a0066f89871e777cc81d43c1ade8c97517 systeminformation: specifier: 5.17.12 version: 5.17.12 @@ -434,6 +437,39 @@ importers: '@tensorflow/tfjs-node': specifier: 4.4.0 version: 4.4.0(seedrandom@3.0.5) + slacc-android-arm-eabi: + specifier: 0.0.7 + version: 0.0.7 + slacc-android-arm64: + specifier: 0.0.7 + version: 0.0.7 + slacc-darwin-arm64: + specifier: 0.0.7 + version: 0.0.7 + slacc-darwin-universal: + specifier: 0.0.7 + version: 0.0.7 + slacc-darwin-x64: + specifier: 0.0.7 + version: 0.0.7 + slacc-linux-arm-gnueabihf: + specifier: 0.0.7 + version: 0.0.7 + slacc-linux-arm64-gnu: + specifier: 0.0.7 + version: 0.0.7 + slacc-linux-arm64-musl: + specifier: 0.0.7 + version: 0.0.7 + slacc-linux-x64-gnu: + specifier: 0.0.7 + version: 0.0.7 + slacc-win32-arm64-msvc: + specifier: 0.0.7 + version: 0.0.7 + slacc-win32-x64-msvc: + specifier: 0.0.7 + version: 0.0.7 devDependencies: '@jest/globals': specifier: 29.5.0 @@ -948,7 +984,7 @@ importers: version: github.com/misskey-dev/storybook-addon-misskey-theme/cf583db098365b2ccc81a82f63ca9c93bc32b640(@storybook/blocks@7.0.7)(@storybook/components@7.0.7)(@storybook/core-events@7.0.7)(@storybook/manager-api@7.0.7)(@storybook/preview-api@7.0.7)(@storybook/theming@7.0.7)(@storybook/types@7.0.7)(react-dom@18.2.0)(react@18.2.0) summaly: specifier: github:misskey-dev/summaly - version: github.com/misskey-dev/summaly/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e + version: github.com/misskey-dev/summaly/2d63e2a0066f89871e777cc81d43c1ade8c97517 vite-plugin-turbosnap: specifier: 1.0.2 version: 1.0.2 @@ -17783,6 +17819,121 @@ packages: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} dev: true + /slacc-android-arm-eabi@0.0.7: + resolution: {integrity: sha512-6TikZlR1jsQscxwphhrf0U4xbsRy6zKJ0zmEULopTzbohgo5OLdZ7L3tQazkYlaaFe3YjGnVLW3FfGhhrajVog==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: false + optional: true + + /slacc-android-arm64@0.0.7: + resolution: {integrity: sha512-aol/9Rg0Hfqu81hpK+HXcx9sGYu4qqYU+djBCgLtb8I6ZMdWUdE0dp8ACBoTOmYn34hYGcUu4FlJUZ8r7Utucg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: false + optional: true + + /slacc-darwin-arm64@0.0.7: + resolution: {integrity: sha512-PkV7rO/c9AImNYDacP+kxtOjVuxjy06IIOAxbWerIWvoeqsCNRtiF/dh+OqIACRFBuHIDe0oAyUCEMGUTnzjyQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /slacc-darwin-universal@0.0.7: + resolution: {integrity: sha512-Y9zXpL40m4Yq3dE5vdnAgfmn0Fxc0Bf0ixC9TSl96gKeIZEd6drkjfpHFdsIDNImzOksIAUo0HHiDdbEfE7zdQ==} + engines: {node: '>= 10'} + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /slacc-darwin-x64@0.0.7: + resolution: {integrity: sha512-yKaGjX2YJl1QHe4NgqQVsY83jees3hjFxEUPoKpuZEQzWbMNn0XSyceFRGXIk1oDqiKU40UcsdcCedjYjSEd0Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /slacc-linux-arm-gnueabihf@0.0.7: + resolution: {integrity: sha512-pdWMdQeX6uA9JfSoWo9EHH0yRiwXKMbaKoS9gflDSyt/hjeR3Qx/KK7Wihd7HeXx7njlNdpr9ycTRmm5NgapQQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /slacc-linux-arm64-gnu@0.0.7: + resolution: {integrity: sha512-hz9TK/w6fxeNZXyFzuLq5cJD/XRyJbo6BaIdW+VrKKnb9nkLnWlqDQtdtJk7Fw7zHjdY3Uqufjwm0iT6qBVpUQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /slacc-linux-arm64-musl@0.0.7: + resolution: {integrity: sha512-wCDAYL7e+lh3XL7g87Ui/Bb2Ap9GcBqeJuj2yHIx6MYC8ontwFSXhqRTmd2zmPLmZA5Nc11aKGN11YNu0Pnwlw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /slacc-linux-x64-gnu@0.0.7: + resolution: {integrity: sha512-E5+2cveizpfHXCk/Hu5VfslWFeDVw47nywODiJ8CsofT2l5ITfYPMFEBXm9ORY25mGBTgsO6lJYiF9Hz4FlS9Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /slacc-win32-arm64-msvc@0.0.7: + resolution: {integrity: sha512-3a+qnkZbP+Pr5RZuzd0Vi1uCal137QiJajRAWT4r7qwu+Zidd50x2oikQ4rAegqZVTm8qTwVmWA+WmH8WHI7iw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /slacc-win32-x64-msvc@0.0.7: + resolution: {integrity: sha512-ydFdZ7wEXQPsw2Tg+yG9uJdCGTehyPtrWBVUMa7fojr3j1gbtThXS2l9Ad/6fYYi2VwdaYPLWbwV3GYElPGL8g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /slacc@0.0.7: + resolution: {integrity: sha512-rwi2F3oJaGPST9JdCoUd5fnSZaoZFgTL00GFKhKufT48uwtUEAHlOL0t8gEVmon71X+53f9nEdsGWhwtOutJTQ==} + engines: {node: '>= 10'} + optionalDependencies: + slacc-android-arm-eabi: 0.0.7 + slacc-android-arm64: 0.0.7 + slacc-darwin-arm64: 0.0.7 + slacc-darwin-universal: 0.0.7 + slacc-darwin-x64: 0.0.7 + slacc-linux-arm-gnueabihf: 0.0.7 + slacc-linux-arm64-gnu: 0.0.7 + slacc-linux-arm64-musl: 0.0.7 + slacc-linux-x64-gnu: 0.0.7 + slacc-win32-arm64-msvc: 0.0.7 + slacc-win32-x64-msvc: 0.0.7 + dev: false + /slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} @@ -20248,8 +20399,8 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true - github.com/misskey-dev/summaly/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e: - resolution: {tarball: https://codeload.github.com/misskey-dev/summaly/tar.gz/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e} + github.com/misskey-dev/summaly/2d63e2a0066f89871e777cc81d43c1ade8c97517: + resolution: {tarball: https://codeload.github.com/misskey-dev/summaly/tar.gz/2d63e2a0066f89871e777cc81d43c1ade8c97517} name: summaly version: 4.0.2 dependencies: