upd: rip out AiService

This commit is contained in:
ShittyKopper 2023-11-05 11:54:52 +03:00
parent 08de1f7baa
commit 2eda00d9b9
14 changed files with 80 additions and 816 deletions

View file

@ -58,8 +58,5 @@
"cypress": "13.4.0",
"eslint": "8.52.0",
"start-server-and-test": "2.0.1"
},
"optionalDependencies": {
"@tensorflow/tfjs-core": "4.4.0"
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -38,8 +38,6 @@
"@swc/core-win32-arm64-msvc": "1.3.56",
"@swc/core-win32-ia32-msvc": "1.3.56",
"@swc/core-win32-x64-msvc": "1.3.56",
"@tensorflow/tfjs": "4.4.0",
"@tensorflow/tfjs-node": "4.4.0",
"bufferutil": "4.0.7",
"slacc-android-arm-eabi": "0.0.10",
"slacc-android-arm64": "0.0.10",
@ -129,7 +127,6 @@
"nested-property": "4.0.0",
"node-fetch": "3.3.2",
"nodemailer": "6.9.7",
"nsfwjs": "2.4.2",
"oauth": "0.10.0",
"oauth2orize": "1.12.0",
"oauth2orize-pkce": "0.1.2",

View file

@ -1,72 +0,0 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import { Injectable } from '@nestjs/common';
import * as nsfw from 'nsfwjs';
import si from 'systeminformation';
import { Mutex } from 'async-mutex';
import { bindThis } from '@/decorators.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const REQUIRED_CPU_FLAGS = ['avx2', 'fma'];
let isSupportedCpu: undefined | boolean = undefined;
@Injectable()
export class AiService {
private model: nsfw.NSFWJS;
private modelLoadMutex: Mutex = new Mutex();
constructor(
) {
}
@bindThis
public async detectSensitive(path: string): Promise<nsfw.predictionType[] | null> {
try {
if (isSupportedCpu === undefined) {
const cpuFlags = await this.getCpuFlags();
isSupportedCpu = REQUIRED_CPU_FLAGS.every(required => cpuFlags.includes(required));
}
if (!isSupportedCpu) {
console.error('This CPU cannot use TensorFlow.');
return null;
}
const tf = await import('@tensorflow/tfjs-node');
if (this.model == null) {
await this.modelLoadMutex.runExclusive(async () => {
if (this.model == null) {
this.model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 });
}
});
}
const buffer = await fs.promises.readFile(path);
const image = await tf.node.decodeImage(buffer, 3) as any;
try {
const predictions = await this.model.classify(image);
return predictions;
} finally {
image.dispose();
}
} catch (err) {
console.error(err);
return null;
}
}
@bindThis
private async getCpuFlags(): Promise<string[]> {
const str = await si.cpuFlags();
return str.split(/\s+/);
}
}

View file

@ -6,7 +6,6 @@
import { Module } from '@nestjs/common';
import { AccountMoveService } from './AccountMoveService.js';
import { AccountUpdateService } from './AccountUpdateService.js';
import { AiService } from './AiService.js';
import { AnnouncementService } from './AnnouncementService.js';
import { AntennaService } from './AntennaService.js';
import { AppLockService } from './AppLockService.js';
@ -138,7 +137,6 @@ import type { Provider } from '@nestjs/common';
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisting: AccountMoveService };
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
const $AiService: Provider = { provide: 'AiService', useExisting: AiService };
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
@ -274,7 +272,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
LoggerService,
AccountMoveService,
AccountUpdateService,
AiService,
AnnouncementService,
AntennaService,
AppLockService,
@ -403,7 +400,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$LoggerService,
$AccountMoveService,
$AccountUpdateService,
$AiService,
$AnnouncementService,
$AntennaService,
$AppLockService,
@ -533,7 +529,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
LoggerService,
AccountMoveService,
AccountUpdateService,
AiService,
AnnouncementService,
AntennaService,
AppLockService,
@ -661,7 +656,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$LoggerService,
$AccountMoveService,
$AccountUpdateService,
$AiService,
$AnnouncementService,
$AntennaService,
$AppLockService,

View file

@ -5,19 +5,13 @@
import * as fs from 'node:fs';
import * as crypto from 'node:crypto';
import { join } from 'node:path';
import * as stream from 'node:stream/promises';
import { Injectable } from '@nestjs/common';
import { FSWatcher } from 'chokidar';
import * as fileType from 'file-type';
import FFmpeg from 'fluent-ffmpeg';
import isSvg from 'is-svg';
import probeImageSize from 'probe-image-size';
import { type predictionType } from 'nsfwjs';
import sharp from 'sharp';
import { encode } from 'blurhash';
import { createTempDir } from '@/misc/create-temp.js';
import { AiService } from '@/core/AiService.js';
import { bindThis } from '@/decorators.js';
export type FileInfo = {
@ -49,7 +43,6 @@ const TYPE_SVG = {
@Injectable()
export class FileInfoService {
constructor(
private aiService: AiService,
) {
}
@ -57,7 +50,8 @@ export class FileInfoService {
* Get file information
*/
@bindThis
public async getFileInfo(path: string, opts: {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async getFileInfo(path: string, _opts: {
skipSensitiveDetection: boolean;
sensitiveThreshold?: number;
sensitiveThresholdForPorn?: number;
@ -128,22 +122,8 @@ export class FileInfoService {
});
}
let sensitive = false;
let porn = false;
if (!opts.skipSensitiveDetection) {
await this.detectSensitivity(
path,
type.mime,
opts.sensitiveThreshold ?? 0.5,
opts.sensitiveThresholdForPorn ?? 0.75,
opts.enableSensitiveMediaDetectionForVideos ?? false,
).then(value => {
[sensitive, porn] = value;
}, error => {
warnings.push(`detectSensitivity failed: ${error}`);
});
}
const sensitive = false;
const porn = false;
return {
size,
@ -159,150 +139,6 @@ export class FileInfoService {
};
}
@bindThis
private async detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> {
let sensitive = false;
let porn = false;
function judgePrediction(result: readonly predictionType[]): [sensitive: boolean, porn: boolean] {
let sensitive = false;
let porn = false;
if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true;
return [sensitive, porn];
}
if ([
'image/jpeg',
'image/png',
'image/webp',
].includes(mime)) {
const result = await this.aiService.detectSensitive(source);
if (result) {
[sensitive, porn] = judgePrediction(result);
}
} else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) {
const [outDir, disposeOutDir] = await createTempDir();
try {
const command = FFmpeg()
.input(source)
.inputOptions([
'-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする(そうなるとは限らない)
'-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする(そうなるとは限らない)
])
.noAudio()
.videoFilters([
{
filter: 'select', // フレームのフィルタリング
options: {
e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタするVP9 とかはデコードしてみないとわからないっぽい)
},
},
{
filter: 'blackframe', // 暗いフレームの検出
options: {
amount: '0', // 暗さに関わらず全てのフレームで測定値を取る
},
},
{
filter: 'metadata',
options: {
mode: 'select', // フレーム選択モード
key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率(前のフィルタからのメタデータを参照する)
value: '50',
function: 'less', // 50% 未満のフレームを選択する50% 以上暗部があるフレームだと誤検知を招くかもしれないので)
},
},
{
filter: 'scale',
options: {
w: 299,
h: 299,
},
},
])
.format('image2')
.output(join(outDir, '%d.png'))
.outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない
const results: ReturnType<typeof judgePrediction>[] = [];
let frameIndex = 0;
let targetIndex = 0;
let nextIndex = 1;
for await (const path of this.asyncIterateFrames(outDir, command)) {
try {
const index = frameIndex++;
if (index !== targetIndex) {
continue;
}
targetIndex = nextIndex;
nextIndex += index; // fibonacci sequence によってフレーム数制限を掛ける
const result = await this.aiService.detectSensitive(path);
if (result) {
results.push(judgePrediction(result));
}
} finally {
fs.promises.unlink(path);
}
}
sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold);
porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn);
} finally {
disposeOutDir();
}
}
return [sensitive, porn];
}
private async *asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator<string, void> {
const watcher = new FSWatcher({
cwd,
disableGlobbing: true,
});
let finished = false;
command.once('end', () => {
finished = true;
watcher.close();
});
command.run();
for (let i = 1; true; i++) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
const current = `${i}.png`;
const next = `${i + 1}.png`;
const framePath = join(cwd, current);
if (await this.exists(join(cwd, next))) {
yield framePath;
} else if (!finished) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
watcher.add(next);
await new Promise<void>((resolve, reject) => {
watcher.on('add', function onAdd(path) {
if (path === next) { // 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている
watcher.unwatch(current);
watcher.off('add', onAdd);
resolve();
}
});
command.once('end', resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている
command.once('error', reject);
});
yield framePath;
} else if (await this.exists(framePath)) {
yield framePath;
} else {
return;
}
}
}
@bindThis
private exists(path: string): Promise<boolean> {
return fs.promises.access(path).then(() => true, () => false);
}
@bindThis
public fixMime(mime: string | fileType.MimeType): string {
// see https://github.com/misskey-dev/misskey/pull/10686

View file

@ -14,7 +14,6 @@ import { describe, beforeAll, afterAll, test } from '@jest/globals';
import { GlobalModule } from '@/GlobalModule.js';
import { FileInfoService } from '@/core/FileInfoService.js';
//import { DI } from '@/di-symbols.js';
import { AiService } from '@/core/AiService.js';
import type { TestingModule } from '@nestjs/testing';
import type { MockFunctionMetadata } from 'jest-mock';
@ -34,14 +33,10 @@ describe('FileInfoService', () => {
GlobalModule,
],
providers: [
AiService,
FileInfoService,
],
})
.useMocker((token) => {
//if (token === AiService) {
// return { };
//}
if (typeof token === 'function') {
const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata<any, any>;
const Mock = moduleMocker.generateFromMetadata(mockMetadata);

File diff suppressed because it is too large Load diff