upd: rip out AiService
This commit is contained in:
ShittyKopper
parent
08de1f7baa
commit
2eda00d9b9
14 changed files with 80 additions and 816 deletions
|
|
@ -1,72 +0,0 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: syuilo and other misskey contributors
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname } from 'node:path';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import * as nsfw from 'nsfwjs';
|
||||
import si from 'systeminformation';
|
||||
import { Mutex } from 'async-mutex';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
|
||||
const _filename = fileURLToPath(import.meta.url);
|
||||
const _dirname = dirname(_filename);
|
||||
|
||||
const REQUIRED_CPU_FLAGS = ['avx2', 'fma'];
|
||||
let isSupportedCpu: undefined | boolean = undefined;
|
||||
|
||||
@Injectable()
|
||||
export class AiService {
|
||||
private model: nsfw.NSFWJS;
|
||||
private modelLoadMutex: Mutex = new Mutex();
|
||||
|
||||
constructor(
|
||||
) {
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async detectSensitive(path: string): Promise<nsfw.predictionType[] | null> {
|
||||
try {
|
||||
if (isSupportedCpu === undefined) {
|
||||
const cpuFlags = await this.getCpuFlags();
|
||||
isSupportedCpu = REQUIRED_CPU_FLAGS.every(required => cpuFlags.includes(required));
|
||||
}
|
||||
|
||||
if (!isSupportedCpu) {
|
||||
console.error('This CPU cannot use TensorFlow.');
|
||||
return null;
|
||||
}
|
||||
|
||||
const tf = await import('@tensorflow/tfjs-node');
|
||||
|
||||
if (this.model == null) {
|
||||
await this.modelLoadMutex.runExclusive(async () => {
|
||||
if (this.model == null) {
|
||||
this.model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const buffer = await fs.promises.readFile(path);
|
||||
const image = await tf.node.decodeImage(buffer, 3) as any;
|
||||
try {
|
||||
const predictions = await this.model.classify(image);
|
||||
return predictions;
|
||||
} finally {
|
||||
image.dispose();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private async getCpuFlags(): Promise<string[]> {
|
||||
const str = await si.cpuFlags();
|
||||
return str.split(/\s+/);
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,6 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { AccountMoveService } from './AccountMoveService.js';
|
||||
import { AccountUpdateService } from './AccountUpdateService.js';
|
||||
import { AiService } from './AiService.js';
|
||||
import { AnnouncementService } from './AnnouncementService.js';
|
||||
import { AntennaService } from './AntennaService.js';
|
||||
import { AppLockService } from './AppLockService.js';
|
||||
|
|
@ -138,7 +137,6 @@ import type { Provider } from '@nestjs/common';
|
|||
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
|
||||
const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisting: AccountMoveService };
|
||||
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
|
||||
const $AiService: Provider = { provide: 'AiService', useExisting: AiService };
|
||||
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
|
||||
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
|
||||
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
|
||||
|
|
@ -274,7 +272,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
|||
LoggerService,
|
||||
AccountMoveService,
|
||||
AccountUpdateService,
|
||||
AiService,
|
||||
AnnouncementService,
|
||||
AntennaService,
|
||||
AppLockService,
|
||||
|
|
@ -403,7 +400,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
|||
$LoggerService,
|
||||
$AccountMoveService,
|
||||
$AccountUpdateService,
|
||||
$AiService,
|
||||
$AnnouncementService,
|
||||
$AntennaService,
|
||||
$AppLockService,
|
||||
|
|
@ -533,7 +529,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
|||
LoggerService,
|
||||
AccountMoveService,
|
||||
AccountUpdateService,
|
||||
AiService,
|
||||
AnnouncementService,
|
||||
AntennaService,
|
||||
AppLockService,
|
||||
|
|
@ -661,7 +656,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
|
|||
$LoggerService,
|
||||
$AccountMoveService,
|
||||
$AccountUpdateService,
|
||||
$AiService,
|
||||
$AnnouncementService,
|
||||
$AntennaService,
|
||||
$AppLockService,
|
||||
|
|
|
|||
|
|
@ -5,19 +5,13 @@
|
|||
|
||||
import * as fs from 'node:fs';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import * as stream from 'node:stream/promises';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { FSWatcher } from 'chokidar';
|
||||
import * as fileType from 'file-type';
|
||||
import FFmpeg from 'fluent-ffmpeg';
|
||||
import isSvg from 'is-svg';
|
||||
import probeImageSize from 'probe-image-size';
|
||||
import { type predictionType } from 'nsfwjs';
|
||||
import sharp from 'sharp';
|
||||
import { encode } from 'blurhash';
|
||||
import { createTempDir } from '@/misc/create-temp.js';
|
||||
import { AiService } from '@/core/AiService.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
|
||||
export type FileInfo = {
|
||||
|
|
@ -49,7 +43,6 @@ const TYPE_SVG = {
|
|||
@Injectable()
|
||||
export class FileInfoService {
|
||||
constructor(
|
||||
private aiService: AiService,
|
||||
) {
|
||||
}
|
||||
|
||||
|
|
@ -57,7 +50,8 @@ export class FileInfoService {
|
|||
* Get file information
|
||||
*/
|
||||
@bindThis
|
||||
public async getFileInfo(path: string, opts: {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
public async getFileInfo(path: string, _opts: {
|
||||
skipSensitiveDetection: boolean;
|
||||
sensitiveThreshold?: number;
|
||||
sensitiveThresholdForPorn?: number;
|
||||
|
|
@ -128,22 +122,8 @@ export class FileInfoService {
|
|||
});
|
||||
}
|
||||
|
||||
let sensitive = false;
|
||||
let porn = false;
|
||||
|
||||
if (!opts.skipSensitiveDetection) {
|
||||
await this.detectSensitivity(
|
||||
path,
|
||||
type.mime,
|
||||
opts.sensitiveThreshold ?? 0.5,
|
||||
opts.sensitiveThresholdForPorn ?? 0.75,
|
||||
opts.enableSensitiveMediaDetectionForVideos ?? false,
|
||||
).then(value => {
|
||||
[sensitive, porn] = value;
|
||||
}, error => {
|
||||
warnings.push(`detectSensitivity failed: ${error}`);
|
||||
});
|
||||
}
|
||||
const sensitive = false;
|
||||
const porn = false;
|
||||
|
||||
return {
|
||||
size,
|
||||
|
|
@ -159,150 +139,6 @@ export class FileInfoService {
|
|||
};
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private async detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> {
|
||||
let sensitive = false;
|
||||
let porn = false;
|
||||
|
||||
function judgePrediction(result: readonly predictionType[]): [sensitive: boolean, porn: boolean] {
|
||||
let sensitive = false;
|
||||
let porn = false;
|
||||
|
||||
if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
|
||||
if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
|
||||
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
|
||||
|
||||
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true;
|
||||
|
||||
return [sensitive, porn];
|
||||
}
|
||||
|
||||
if ([
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/webp',
|
||||
].includes(mime)) {
|
||||
const result = await this.aiService.detectSensitive(source);
|
||||
if (result) {
|
||||
[sensitive, porn] = judgePrediction(result);
|
||||
}
|
||||
} else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) {
|
||||
const [outDir, disposeOutDir] = await createTempDir();
|
||||
try {
|
||||
const command = FFmpeg()
|
||||
.input(source)
|
||||
.inputOptions([
|
||||
'-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする（そうなるとは限らない）
|
||||
'-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする（そうなるとは限らない）
|
||||
])
|
||||
.noAudio()
|
||||
.videoFilters([
|
||||
{
|
||||
filter: 'select', // フレームのフィルタリング
|
||||
options: {
|
||||
e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタする（VP9 とかはデコードしてみないとわからないっぽい）
|
||||
},
|
||||
},
|
||||
{
|
||||
filter: 'blackframe', // 暗いフレームの検出
|
||||
options: {
|
||||
amount: '0', // 暗さに関わらず全てのフレームで測定値を取る
|
||||
},
|
||||
},
|
||||
{
|
||||
filter: 'metadata',
|
||||
options: {
|
||||
mode: 'select', // フレーム選択モード
|
||||
key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率（前のフィルタからのメタデータを参照する）
|
||||
value: '50',
|
||||
function: 'less', // 50% 未満のフレームを選択する（50% 以上暗部があるフレームだと誤検知を招くかもしれないので）
|
||||
},
|
||||
},
|
||||
{
|
||||
filter: 'scale',
|
||||
options: {
|
||||
w: 299,
|
||||
h: 299,
|
||||
},
|
||||
},
|
||||
])
|
||||
.format('image2')
|
||||
.output(join(outDir, '%d.png'))
|
||||
.outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない
|
||||
const results: ReturnType<typeof judgePrediction>[] = [];
|
||||
let frameIndex = 0;
|
||||
let targetIndex = 0;
|
||||
let nextIndex = 1;
|
||||
for await (const path of this.asyncIterateFrames(outDir, command)) {
|
||||
try {
|
||||
const index = frameIndex++;
|
||||
if (index !== targetIndex) {
|
||||
continue;
|
||||
}
|
||||
targetIndex = nextIndex;
|
||||
nextIndex += index; // fibonacci sequence によってフレーム数制限を掛ける
|
||||
const result = await this.aiService.detectSensitive(path);
|
||||
if (result) {
|
||||
results.push(judgePrediction(result));
|
||||
}
|
||||
} finally {
|
||||
fs.promises.unlink(path);
|
||||
}
|
||||
}
|
||||
sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold);
|
||||
porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn);
|
||||
} finally {
|
||||
disposeOutDir();
|
||||
}
|
||||
}
|
||||
|
||||
return [sensitive, porn];
|
||||
}
|
||||
|
||||
private async *asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator<string, void> {
|
||||
const watcher = new FSWatcher({
|
||||
cwd,
|
||||
disableGlobbing: true,
|
||||
});
|
||||
let finished = false;
|
||||
command.once('end', () => {
|
||||
finished = true;
|
||||
watcher.close();
|
||||
});
|
||||
command.run();
|
||||
for (let i = 1; true; i++) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
|
||||
const current = `${i}.png`;
|
||||
const next = `${i + 1}.png`;
|
||||
const framePath = join(cwd, current);
|
||||
if (await this.exists(join(cwd, next))) {
|
||||
yield framePath;
|
||||
} else if (!finished) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
|
||||
watcher.add(next);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
watcher.on('add', function onAdd(path) {
|
||||
if (path === next) { // 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている
|
||||
watcher.unwatch(current);
|
||||
watcher.off('add', onAdd);
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
command.once('end', resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている
|
||||
command.once('error', reject);
|
||||
});
|
||||
yield framePath;
|
||||
} else if (await this.exists(framePath)) {
|
||||
yield framePath;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private exists(path: string): Promise<boolean> {
|
||||
return fs.promises.access(path).then(() => true, () => false);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public fixMime(mime: string | fileType.MimeType): string {
|
||||
// see https://github.com/misskey-dev/misskey/pull/10686
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue