mirror of
https://git.joinsharkey.org/Sharkey/Sharkey.git
synced 2024-11-23 01:43:09 +02:00
perf: use slacc on check-word-mute (#10721)
* perf: use slacc on check-word-mute when all of specified words are single word * perf: use slacc as possible * build: avoid tarball * chore: update slacc * build: update package name
This commit is contained in:
Acid Chicken (硫酸鶏)
GitHub
parent
14e364a74a
commit
4a72941eda
4 changed files with 236 additions and 6 deletions
|
|
@ -34,7 +34,18 @@
|
|||
"@swc/core-win32-ia32-msvc": "1.3.56",
|
||||
"@swc/core-win32-x64-msvc": "1.3.56",
|
||||
"@tensorflow/tfjs": "4.4.0",
|
||||
"@tensorflow/tfjs-node": "4.4.0"
|
||||
"@tensorflow/tfjs-node": "4.4.0",
|
||||
"slacc-android-arm-eabi": "0.0.7",
|
||||
"slacc-android-arm64": "0.0.7",
|
||||
"slacc-darwin-arm64": "0.0.7",
|
||||
"slacc-darwin-universal": "0.0.7",
|
||||
"slacc-darwin-x64": "0.0.7",
|
||||
"slacc-linux-arm-gnueabihf": "0.0.7",
|
||||
"slacc-linux-arm64-gnu": "0.0.7",
|
||||
"slacc-linux-arm64-musl": "0.0.7",
|
||||
"slacc-linux-x64-gnu": "0.0.7",
|
||||
"slacc-win32-arm64-msvc": "0.0.7",
|
||||
"slacc-win32-x64-msvc": "0.0.7"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.321.1",
|
||||
|
|
@ -128,6 +139,7 @@
|
|||
"semver": "7.5.0",
|
||||
"sharp": "0.32.1",
|
||||
"sharp-read-bmp": "github:misskey-dev/sharp-read-bmp",
|
||||
"slacc": "0.0.7",
|
||||
"strict-event-emitter-types": "2.0.0",
|
||||
"stringz": "2.1.0",
|
||||
"summaly": "github:misskey-dev/summaly",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { AhoCorasick } from 'slacc';
|
||||
import RE2 from 're2';
|
||||
import type { Note } from '@/models/entities/Note.js';
|
||||
import type { User } from '@/models/entities/User.js';
|
||||
|
|
@ -12,6 +13,8 @@ type UserLike = {
|
|||
id: User['id'];
|
||||
};
|
||||
|
||||
const acCache = new Map<string, AhoCorasick>();
|
||||
|
||||
export async function checkWordMute(note: NoteLike, me: UserLike | null | undefined, mutedWords: Array<string | string[]>): Promise<boolean> {
|
||||
// 自分自身
|
||||
if (me && (note.userId === me.id)) return false;
|
||||
|
|
@ -21,7 +24,22 @@ export async function checkWordMute(note: NoteLike, me: UserLike | null | undefi
|
|||
|
||||
if (text === '') return false;
|
||||
|
||||
const matched = mutedWords.some(filter => {
|
||||
const acable = mutedWords.filter(filter => Array.isArray(filter) && filter.length === 1).map(filter => filter[0]).sort();
|
||||
const unacable = mutedWords.filter(filter => !Array.isArray(filter) || filter.length !== 1);
|
||||
const acCacheKey = acable.join('\n');
|
||||
const ac = acCache.get(acCacheKey) ?? AhoCorasick.withPatterns(acable);
|
||||
acCache.delete(acCacheKey);
|
||||
for (const obsoleteKeys of acCache.keys()) {
|
||||
if (acCache.size > 1000) {
|
||||
acCache.delete(obsoleteKeys);
|
||||
}
|
||||
}
|
||||
acCache.set(acCacheKey, ac);
|
||||
if (ac.isMatch(text)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const matched = unacable.some(filter => {
|
||||
if (Array.isArray(filter)) {
|
||||
return filter.every(keyword => text.includes(keyword));
|
||||
} else {
|
||||
|
|
|
|||
49
packages/backend/test/unit/misc/check-word-mute.ts
Normal file
49
packages/backend/test/unit/misc/check-word-mute.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import { checkWordMute } from '@/misc/check-word-mute.js';
|
||||
|
||||
describe(checkWordMute, () => {
|
||||
describe('Slacc boost mode', () => {
|
||||
it('should return false if mutedWords is empty', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [])).toBe(false);
|
||||
});
|
||||
it('should return true if mutedWords is not empty and text contains muted word', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [['foo']])).toBe(true);
|
||||
});
|
||||
it('should return false if mutedWords is not empty and text does not contain muted word', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [['bar']])).toBe(false);
|
||||
});
|
||||
it('should return false when the note is written by me even if mutedWords is not empty and text contains muted word', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo' }, { id: '1' }, [['foo']])).toBe(false);
|
||||
});
|
||||
it('should return true if mutedWords is not empty and text contains muted word in CW', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo', cw: 'bar' }, null, [['bar']])).toBe(true);
|
||||
});
|
||||
it('should return true if mutedWords is not empty and text contains muted word in both CW and text', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo', cw: 'bar' }, null, [['foo'], ['bar']])).toBe(true);
|
||||
});
|
||||
it('should return true if mutedWords is not empty and text does not contain muted word in both CW and text', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo', cw: 'bar' }, null, [['foo'], ['baz']])).toBe(true);
|
||||
});
|
||||
});
|
||||
describe('normal mode', () => {
|
||||
it('should return false if text does not contain muted words', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo' }, null, [['foo', 'bar']])).toBe(false);
|
||||
});
|
||||
it('should return true if text contains muted words', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foobar' }, null, [['foo', 'bar']])).toBe(true);
|
||||
});
|
||||
it('should return false when the note is written by me even if text contains muted words', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo bar' }, { id: '1' }, [['foo', 'bar']])).toBe(false);
|
||||
});
|
||||
});
|
||||
describe('RegExp mode', () => {
|
||||
it('should return false if text does not contain muted words', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo' }, null, ['/bar/'])).toBe(false);
|
||||
});
|
||||
it('should return true if text contains muted words', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foobar' }, null, ['/bar/'])).toBe(true);
|
||||
});
|
||||
it('should return false when the note is written by me even if text contains muted words', async () => {
|
||||
expect(await checkWordMute({ userId: '1', text: 'foo bar' }, { id: '1' }, ['/bar/'])).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
159
pnpm-lock.yaml
159
pnpm-lock.yaml
|
|
@ -337,6 +337,9 @@ importers:
|
|||
sharp-read-bmp:
|
||||
specifier: github:misskey-dev/sharp-read-bmp
|
||||
version: github.com/misskey-dev/sharp-read-bmp/02d9dc189fa7df0c4bea09330be26741772dac01
|
||||
slacc:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
strict-event-emitter-types:
|
||||
specifier: 2.0.0
|
||||
version: 2.0.0
|
||||
|
|
@ -345,7 +348,7 @@ importers:
|
|||
version: 2.1.0
|
||||
summaly:
|
||||
specifier: github:misskey-dev/summaly
|
||||
version: github.com/misskey-dev/summaly/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e
|
||||
version: github.com/misskey-dev/summaly/2d63e2a0066f89871e777cc81d43c1ade8c97517
|
||||
systeminformation:
|
||||
specifier: 5.17.12
|
||||
version: 5.17.12
|
||||
|
|
@ -434,6 +437,39 @@ importers:
|
|||
'@tensorflow/tfjs-node':
|
||||
specifier: 4.4.0
|
||||
version: 4.4.0(seedrandom@3.0.5)
|
||||
slacc-android-arm-eabi:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-android-arm64:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-darwin-arm64:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-darwin-universal:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-darwin-x64:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-linux-arm-gnueabihf:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-linux-arm64-gnu:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-linux-arm64-musl:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-linux-x64-gnu:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-win32-arm64-msvc:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
slacc-win32-x64-msvc:
|
||||
specifier: 0.0.7
|
||||
version: 0.0.7
|
||||
devDependencies:
|
||||
'@jest/globals':
|
||||
specifier: 29.5.0
|
||||
|
|
@ -948,7 +984,7 @@ importers:
|
|||
version: github.com/misskey-dev/storybook-addon-misskey-theme/cf583db098365b2ccc81a82f63ca9c93bc32b640(@storybook/blocks@7.0.7)(@storybook/components@7.0.7)(@storybook/core-events@7.0.7)(@storybook/manager-api@7.0.7)(@storybook/preview-api@7.0.7)(@storybook/theming@7.0.7)(@storybook/types@7.0.7)(react-dom@18.2.0)(react@18.2.0)
|
||||
summaly:
|
||||
specifier: github:misskey-dev/summaly
|
||||
version: github.com/misskey-dev/summaly/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e
|
||||
version: github.com/misskey-dev/summaly/2d63e2a0066f89871e777cc81d43c1ade8c97517
|
||||
vite-plugin-turbosnap:
|
||||
specifier: 1.0.2
|
||||
version: 1.0.2
|
||||
|
|
@ -17783,6 +17819,121 @@ packages:
|
|||
resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}
|
||||
dev: true
|
||||
|
||||
/slacc-android-arm-eabi@0.0.7:
|
||||
resolution: {integrity: sha512-6TikZlR1jsQscxwphhrf0U4xbsRy6zKJ0zmEULopTzbohgo5OLdZ7L3tQazkYlaaFe3YjGnVLW3FfGhhrajVog==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-android-arm64@0.0.7:
|
||||
resolution: {integrity: sha512-aol/9Rg0Hfqu81hpK+HXcx9sGYu4qqYU+djBCgLtb8I6ZMdWUdE0dp8ACBoTOmYn34hYGcUu4FlJUZ8r7Utucg==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-darwin-arm64@0.0.7:
|
||||
resolution: {integrity: sha512-PkV7rO/c9AImNYDacP+kxtOjVuxjy06IIOAxbWerIWvoeqsCNRtiF/dh+OqIACRFBuHIDe0oAyUCEMGUTnzjyQ==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-darwin-universal@0.0.7:
|
||||
resolution: {integrity: sha512-Y9zXpL40m4Yq3dE5vdnAgfmn0Fxc0Bf0ixC9TSl96gKeIZEd6drkjfpHFdsIDNImzOksIAUo0HHiDdbEfE7zdQ==}
|
||||
engines: {node: '>= 10'}
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-darwin-x64@0.0.7:
|
||||
resolution: {integrity: sha512-yKaGjX2YJl1QHe4NgqQVsY83jees3hjFxEUPoKpuZEQzWbMNn0XSyceFRGXIk1oDqiKU40UcsdcCedjYjSEd0Q==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-linux-arm-gnueabihf@0.0.7:
|
||||
resolution: {integrity: sha512-pdWMdQeX6uA9JfSoWo9EHH0yRiwXKMbaKoS9gflDSyt/hjeR3Qx/KK7Wihd7HeXx7njlNdpr9ycTRmm5NgapQQ==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-linux-arm64-gnu@0.0.7:
|
||||
resolution: {integrity: sha512-hz9TK/w6fxeNZXyFzuLq5cJD/XRyJbo6BaIdW+VrKKnb9nkLnWlqDQtdtJk7Fw7zHjdY3Uqufjwm0iT6qBVpUQ==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-linux-arm64-musl@0.0.7:
|
||||
resolution: {integrity: sha512-wCDAYL7e+lh3XL7g87Ui/Bb2Ap9GcBqeJuj2yHIx6MYC8ontwFSXhqRTmd2zmPLmZA5Nc11aKGN11YNu0Pnwlw==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-linux-x64-gnu@0.0.7:
|
||||
resolution: {integrity: sha512-E5+2cveizpfHXCk/Hu5VfslWFeDVw47nywODiJ8CsofT2l5ITfYPMFEBXm9ORY25mGBTgsO6lJYiF9Hz4FlS9Q==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-win32-arm64-msvc@0.0.7:
|
||||
resolution: {integrity: sha512-3a+qnkZbP+Pr5RZuzd0Vi1uCal137QiJajRAWT4r7qwu+Zidd50x2oikQ4rAegqZVTm8qTwVmWA+WmH8WHI7iw==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc-win32-x64-msvc@0.0.7:
|
||||
resolution: {integrity: sha512-ydFdZ7wEXQPsw2Tg+yG9uJdCGTehyPtrWBVUMa7fojr3j1gbtThXS2l9Ad/6fYYi2VwdaYPLWbwV3GYElPGL8g==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: false
|
||||
optional: true
|
||||
|
||||
/slacc@0.0.7:
|
||||
resolution: {integrity: sha512-rwi2F3oJaGPST9JdCoUd5fnSZaoZFgTL00GFKhKufT48uwtUEAHlOL0t8gEVmon71X+53f9nEdsGWhwtOutJTQ==}
|
||||
engines: {node: '>= 10'}
|
||||
optionalDependencies:
|
||||
slacc-android-arm-eabi: 0.0.7
|
||||
slacc-android-arm64: 0.0.7
|
||||
slacc-darwin-arm64: 0.0.7
|
||||
slacc-darwin-universal: 0.0.7
|
||||
slacc-darwin-x64: 0.0.7
|
||||
slacc-linux-arm-gnueabihf: 0.0.7
|
||||
slacc-linux-arm64-gnu: 0.0.7
|
||||
slacc-linux-arm64-musl: 0.0.7
|
||||
slacc-linux-x64-gnu: 0.0.7
|
||||
slacc-win32-arm64-msvc: 0.0.7
|
||||
slacc-win32-x64-msvc: 0.0.7
|
||||
dev: false
|
||||
|
||||
/slash@3.0.0:
|
||||
resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
@ -20248,8 +20399,8 @@ packages:
|
|||
react-dom: 18.2.0(react@18.2.0)
|
||||
dev: true
|
||||
|
||||
github.com/misskey-dev/summaly/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e:
|
||||
resolution: {tarball: https://codeload.github.com/misskey-dev/summaly/tar.gz/c7d71a9ec2467268b3911dc2ac805c2b8a898d3e}
|
||||
github.com/misskey-dev/summaly/2d63e2a0066f89871e777cc81d43c1ade8c97517:
|
||||
resolution: {tarball: https://codeload.github.com/misskey-dev/summaly/tar.gz/2d63e2a0066f89871e777cc81d43c1ade8c97517}
|
||||
name: summaly
|
||||
version: 4.0.2
|
||||
dependencies:
|
||||
|
|
|
|||
Loading…
Reference in a new issue