Merge branch 'develop' into pr/ThatOneCalculator/8764

This commit is contained in:
tamaina 2022-07-19 06:21:39 +00:00
commit 6fab21c6a7
233 changed files with 6639 additions and 18111 deletions

View file

@ -28,11 +28,11 @@ export class foreignKeyReports1651224615271 {
queryRunner.query(`CREATE INDEX "IDX_315c779174fe8247ab324f036e" ON "drive_file" ("isLink")`),
queryRunner.query(`CREATE INDEX "IDX_f22169eb10657bded6d875ac8f" ON "note" ("channelId")`),
queryRunner.query(`CREATE INDEX "IDX_a9021cc2e1feb5f72d3db6e9f5" ON "abuse_user_report" ("targetUserId")`),
//queryRunner.query(`CREATE INDEX "IDX_a9021cc2e1feb5f72d3db6e9f5" ON "abuse_user_report" ("targetUserId")`),
queryRunner.query(`DELETE FROM "abuse_user_report" WHERE "targetUserId" NOT IN (SELECT "id" FROM "user")`).then(() => {
queryRunner.query(`ALTER TABLE "abuse_user_report" ADD CONSTRAINT "FK_a9021cc2e1feb5f72d3db6e9f5f" FOREIGN KEY ("targetUserId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
}),
//queryRunner.query(`DELETE FROM "abuse_user_report" WHERE "targetUserId" NOT IN (SELECT "id" FROM "user")`).then(() => {
// queryRunner.query(`ALTER TABLE "abuse_user_report" ADD CONSTRAINT "FK_a9021cc2e1feb5f72d3db6e9f5f" FOREIGN KEY ("targetUserId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
//}),
queryRunner.query(`ALTER TABLE "poll" ADD CONSTRAINT "UQ_da851e06d0dfe2ef397d8b1bf1b" UNIQUE ("noteId")`),
queryRunner.query(`ALTER TABLE "user_keypair" ADD CONSTRAINT "UQ_f4853eb41ab722fe05f81cedeb6" UNIQUE ("userId")`),

View file

@ -0,0 +1,23 @@
export class nsfwDetection1655368940105 {
name = 'nsfwDetection1655368940105'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "drive_file" ADD "forceIsSensitive" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "drive_file" ADD "predictedIsSensitive" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "drive_file"."predictedIsSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetection_enum" AS ENUM('none', 'all', 'local', 'remote')`);
await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetection" "public"."meta_sensitiveimagedetection_enum" NOT NULL DEFAULT 'none'`);
await queryRunner.query(`ALTER TABLE "meta" ADD "forceIsSensitiveWhenPredicted" boolean NOT NULL DEFAULT true`);
await queryRunner.query(`CREATE INDEX "IDX_fc2d74a6d7d8b11292a851d8f8" ON "drive_file" ("predictedIsSensitive") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_fc2d74a6d7d8b11292a851d8f8"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "forceIsSensitiveWhenPredicted"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetection"`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetection_enum"`);
await queryRunner.query(`COMMENT ON COLUMN "drive_file"."predictedIsSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "predictedIsSensitive"`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "forceIsSensitive"`);
}
}

View file

@ -0,0 +1,15 @@
export class nsfwDetection21655371960534 {
name = 'nsfwDetection21655371960534'
async up(queryRunner) {
await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" AS ENUM('medium', 'low', 'high')`);
await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetectionSensitivity" "public"."meta_sensitiveimagedetectionsensitivity_enum" NOT NULL DEFAULT 'medium'`);
await queryRunner.query(`ALTER TABLE "meta" ADD "disallowUploadWhenPredictedAsPorn" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "disallowUploadWhenPredictedAsPorn"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetectionSensitivity"`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum"`);
}
}

View file

@ -0,0 +1,21 @@
export class nsfwDetection31655388169582 {
name = 'nsfwDetection31655388169582'
async up(queryRunner) {
await queryRunner.query(`ALTER TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" RENAME TO "meta_sensitiveimagedetectionsensitivity_enum_old"`);
await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" AS ENUM('medium', 'low', 'high', 'veryLow', 'veryHigh')`);
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" DROP DEFAULT`);
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" USING "sensitiveImageDetectionSensitivity"::"text"::"public"."meta_sensitiveimagedetectionsensitivity_enum"`);
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" SET DEFAULT 'medium'`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old"`);
}
async down(queryRunner) {
await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old" AS ENUM('medium', 'low', 'high')`);
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" DROP DEFAULT`);
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old" USING "sensitiveImageDetectionSensitivity"::"text"::"public"."meta_sensitiveimagedetectionsensitivity_enum_old"`);
await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "sensitiveImageDetectionSensitivity" SET DEFAULT 'medium'`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum"`);
await queryRunner.query(`ALTER TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum_old" RENAME TO "meta_sensitiveimagedetectionsensitivity_enum"`);
}
}

View file

@ -0,0 +1,25 @@
export class nsfwDetection41655393015659 {
name = 'nsfwDetection41655393015659'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetection"`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetection_enum"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveImageDetectionSensitivity"`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum"`);
await queryRunner.query(`CREATE TYPE "public"."meta_sensitivemediadetection_enum" AS ENUM('none', 'all', 'local', 'remote')`);
await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveMediaDetection" "public"."meta_sensitivemediadetection_enum" NOT NULL DEFAULT 'none'`);
await queryRunner.query(`CREATE TYPE "public"."meta_sensitivemediadetectionsensitivity_enum" AS ENUM('medium', 'low', 'high', 'veryLow', 'veryHigh')`);
await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveMediaDetectionSensitivity" "public"."meta_sensitivemediadetectionsensitivity_enum" NOT NULL DEFAULT 'medium'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveMediaDetectionSensitivity"`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitivemediadetectionsensitivity_enum"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveMediaDetection"`);
await queryRunner.query(`DROP TYPE "public"."meta_sensitivemediadetection_enum"`);
await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetectionsensitivity_enum" AS ENUM('medium', 'low', 'high', 'veryLow', 'veryHigh')`);
await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetectionSensitivity" "public"."meta_sensitiveimagedetectionsensitivity_enum" NOT NULL DEFAULT 'medium'`);
await queryRunner.query(`CREATE TYPE "public"."meta_sensitiveimagedetection_enum" AS ENUM('none', 'all', 'local', 'remote')`);
await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveImageDetection" "public"."meta_sensitiveimagedetection_enum" NOT NULL DEFAULT 'none'`);
}
}

View file

@ -0,0 +1,33 @@
export class nsfwDetection51656251734807 {
name = 'nsfwDetection51656251734807'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_fc2d74a6d7d8b11292a851d8f8"`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "forceIsSensitive"`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "predictedIsSensitive"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "forceIsSensitiveWhenPredicted"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "disallowUploadWhenPredictedAsPorn"`);
await queryRunner.query(`ALTER TABLE "drive_file" ADD "maybeSensitive" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "drive_file"."maybeSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
await queryRunner.query(`ALTER TABLE "drive_file" ADD "maybePorn" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "setSensitiveFlagAutomatically" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "user_profile" ADD "autoSensitive" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`CREATE INDEX "IDX_3b33dff77bb64b23c88151d23e" ON "drive_file" ("maybeSensitive") `);
await queryRunner.query(`CREATE INDEX "IDX_8bdcd3dd2bddb78014999a16ce" ON "drive_file" ("maybePorn") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_8bdcd3dd2bddb78014999a16ce"`);
await queryRunner.query(`DROP INDEX "public"."IDX_3b33dff77bb64b23c88151d23e"`);
await queryRunner.query(`ALTER TABLE "user_profile" DROP COLUMN "autoSensitive"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "setSensitiveFlagAutomatically"`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "maybePorn"`);
await queryRunner.query(`COMMENT ON COLUMN "drive_file"."maybeSensitive" IS 'Whether the DriveFile is NSFW. (predict)'`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "maybeSensitive"`);
await queryRunner.query(`ALTER TABLE "meta" ADD "disallowUploadWhenPredictedAsPorn" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "forceIsSensitiveWhenPredicted" boolean NOT NULL DEFAULT true`);
await queryRunner.query(`ALTER TABLE "drive_file" ADD "predictedIsSensitive" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "drive_file" ADD "forceIsSensitive" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`CREATE INDEX "IDX_fc2d74a6d7d8b11292a851d8f8" ON "drive_file" ("predictedIsSensitive") `);
}
}

View file

@ -0,0 +1,11 @@
export class nsfwDetection61656408772602 {
name = 'nsfwDetection61656408772602'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "enableSensitiveMediaDetectionForVideos" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableSensitiveMediaDetectionForVideos"`);
}
}

View file

@ -0,0 +1,11 @@
export class activeEmailValidation1657346559800 {
name = 'activeEmailValidation1657346559800'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "enableActiveEmailValidation" boolean NOT NULL DEFAULT true`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableActiveEmailValidation"`);
}
}

Binary file not shown.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -13,6 +13,9 @@
"mocha": "NODE_ENV=test TS_NODE_FILES=true TS_NODE_TRANSPILE_ONLY=true TS_NODE_PROJECT=\"./test/tsconfig.json\" mocha",
"test": "yarn mocha"
},
"optionalDependencies": {
"@tensorflow/tfjs-node": "3.18.0"
},
"dependencies": {
"@bull-board/api": "4.0.0",
"@bull-board/koa": "4.0.0",
@ -38,6 +41,7 @@
"cbor": "8.1.0",
"chalk": "5.0.1",
"chalk-template": "0.4.0",
"chokidar": "3.5.3",
"cli-highlight": "2.1.11",
"color-convert": "2.0.1",
"content-disposition": "0.5.4",
@ -67,15 +71,16 @@
"koa-send": "5.0.1",
"koa-slow": "2.1.0",
"koa-views": "7.0.2",
"mfm-js": "0.22.1",
"mfm-js": "0.23.0-canary.1",
"mime-types": "2.1.35",
"misskey-js": "0.0.14",
"mocha": "10.0.0",
"ms": "3.0.0-canary.1",
"multer": "1.4.4",
"nested-property": "4.0.0",
"node-fetch": "3.2.6",
"nodemailer": "6.7.6",
"node-fetch": "3.2.8",
"nodemailer": "6.7.7",
"nsfwjs": "2.4.1",
"oauth": "^0.9.15",
"os-utils": "0.0.14",
"parse5": "7.0.0",
@ -104,13 +109,13 @@
"strict-event-emitter-types": "2.0.0",
"stringz": "2.1.0",
"style-loader": "3.3.1",
"summaly": "2.6.0",
"summaly": "2.7.0",
"syslog-pro": "1.0.0",
"systeminformation": "5.11.22",
"systeminformation": "5.12.0",
"tinycolor2": "1.4.2",
"tmp": "0.2.1",
"ts-loader": "9.3.1",
"ts-node": "10.8.1",
"ts-node": "10.8.2",
"tsc-alias": "1.6.11",
"tsconfig-paths": "4.0.0",
"twemoji-parser": "14.0.0",
@ -135,7 +140,7 @@
"@types/jsdom": "16.2.14",
"@types/jsonld": "1.5.6",
"@types/jsrsasign": "10.5.1",
"@types/koa": "2.13.4",
"@types/koa": "2.13.5",
"@types/koa-bodyparser": "4.3.7",
"@types/koa-cors": "0.0.2",
"@types/koa-favicon": "2.0.21",
@ -147,7 +152,7 @@
"@types/koa__multer": "2.0.4",
"@types/koa__router": "8.0.11",
"@types/mocha": "9.1.1",
"@types/node": "18.0.0",
"@types/node": "18.0.3",
"@types/node-fetch": "3.0.3",
"@types/nodemailer": "6.4.4",
"@types/oauth": "0.9.1",
@ -169,10 +174,10 @@
"@types/web-push": "3.3.2",
"@types/websocket": "1.0.5",
"@types/ws": "8.5.3",
"@typescript-eslint/eslint-plugin": "5.30.0",
"@typescript-eslint/parser": "5.30.0",
"@typescript-eslint/eslint-plugin": "5.30.6",
"@typescript-eslint/parser": "5.30.6",
"cross-env": "7.0.3",
"eslint": "8.18.0",
"eslint": "8.19.0",
"eslint-plugin-import": "2.26.0",
"execa": "6.1.0",
"form-data": "^4.0.0",

View file

@ -145,6 +145,12 @@ export function toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMenti
a.textContent = node.props.content;
return a;
},
plain(node) {
const el = doc.createElement('span');
appendChildren(node.children, el);
return el;
},
};
appendChildren(nodes, doc.body);

View file

@ -7,28 +7,31 @@ import { WriteStream } from 'node:fs';
import * as p from 'pureimage';
import gen from 'random-seed';
const size = 256; // px
const size = 128; // px
const n = 5; // resolution
const margin = (size / n);
const margin = (size / 4);
const colors = [
'#e57373',
'#F06292',
'#BA68C8',
'#9575CD',
'#7986CB',
'#64B5F6',
'#4FC3F7',
'#4DD0E1',
'#4DB6AC',
'#81C784',
'#8BC34A',
'#AFB42B',
'#F57F17',
'#FF5722',
'#795548',
'#455A64',
['#FF512F', '#DD2476'],
['#FF61D2', '#FE9090'],
['#72FFB6', '#10D164'],
['#FD8451', '#FFBD6F'],
['#305170', '#6DFC6B'],
['#00C0FF', '#4218B8'],
['#009245', '#FCEE21'],
['#0100EC', '#FB36F4'],
['#FDABDD', '#374A5A'],
['#38A2D7', '#561139'],
['#121C84', '#8278DA'],
['#5761B2', '#1FC5A8'],
['#FFDB01', '#0E197D'],
['#FF3E9D', '#0E1F40'],
['#766eff', '#00d4ff'],
['#9bff6e', '#00d4ff'],
['#ff6e94', '#00d4ff'],
['#ffa96e', '#00d4ff'],
['#ffa96e', '#ff009d'],
['#ffdd6e', '#ff009d'],
];
const bg = '#e9e9e9';
const actualSize = size - (margin * 2);
const cellSize = actualSize / n;
@ -42,11 +45,17 @@ export function genIdenticon(seed: string, stream: WriteStream): Promise<void> {
const canvas = p.make(size, size, undefined);
const ctx = canvas.getContext('2d');
const bgColors = colors[rand(colors.length)];
const bg = ctx.createLinearGradient(0, 0, size, size);
bg.addColorStop(0, bgColors[0]);
bg.addColorStop(1, bgColors[1]);
ctx.fillStyle = bg;
ctx.beginPath();
ctx.fillRect(0, 0, size, size);
ctx.fillStyle = colors[rand(colors.length)];
ctx.fillStyle = '#ffffff';
// side bitmap (filled by false)
const side: boolean[][] = new Array(sideN);

View file

@ -1,12 +1,18 @@
import * as fs from 'node:fs';
import * as crypto from 'node:crypto';
import { join } from 'node:path';
import * as stream from 'node:stream';
import * as util from 'node:util';
import { FSWatcher } from 'chokidar';
import { fileTypeFromFile } from 'file-type';
import FFmpeg from 'fluent-ffmpeg';
import isSvg from 'is-svg';
import probeImageSize from 'probe-image-size';
import { type predictionType } from 'nsfwjs';
import sharp from 'sharp';
import { encode } from 'blurhash';
import { detectSensitive } from '@/services/detect-sensitive.js';
import { createTempDir } from './create-temp.js';
const pipeline = util.promisify(stream.pipeline);
@ -21,6 +27,8 @@ export type FileInfo = {
height?: number;
orientation?: number;
blurhash?: string;
sensitive: boolean;
porn: boolean;
warnings: string[];
};
@ -37,7 +45,12 @@ const TYPE_SVG = {
/**
* Get file information
*/
export async function getFileInfo(path: string): Promise<FileInfo> {
export async function getFileInfo(path: string, opts: {
skipSensitiveDetection: boolean;
sensitiveThreshold?: number;
sensitiveThresholdForPorn?: number;
enableSensitiveMediaDetectionForVideos?: boolean;
}): Promise<FileInfo> {
const warnings = [] as string[];
const size = await getFileSize(path);
@ -58,7 +71,7 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
// うまく判定できない画像は octet-stream にする
if (!imageSize) {
warnings.push(`cannot detect image dimensions`);
warnings.push('cannot detect image dimensions');
type = TYPE_OCTET_STREAM;
} else if (imageSize.wUnits === 'px') {
width = imageSize.width;
@ -67,7 +80,7 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
// 制限を超えている画像は octet-stream にする
if (imageSize.width > 16383 || imageSize.height > 16383) {
warnings.push(`image dimensions exceeds limits`);
warnings.push('image dimensions exceeds limits');
type = TYPE_OCTET_STREAM;
}
} else {
@ -84,6 +97,19 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
});
}
let sensitive = false;
let porn = false;
if (!opts.skipSensitiveDetection) {
[sensitive, porn] = await detectSensitivity(
path,
type.mime,
opts.sensitiveThreshold ?? 0.5,
opts.sensitiveThresholdForPorn ?? 0.75,
opts.enableSensitiveMediaDetectionForVideos ?? false,
);
}
return {
size,
md5,
@ -92,10 +118,150 @@ export async function getFileInfo(path: string): Promise<FileInfo> {
height,
orientation,
blurhash,
sensitive,
porn,
warnings,
};
}
async function detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> {
let sensitive = false;
let porn = false;
function judgePrediction(result: readonly predictionType[]): [sensitive: boolean, porn: boolean] {
let sensitive = false;
let porn = false;
if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true;
return [sensitive, porn];
}
if (['image/jpeg', 'image/png', 'image/webp'].includes(mime)) {
const result = await detectSensitive(source);
if (result) {
[sensitive, porn] = judgePrediction(result);
}
} else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) {
const [outDir, disposeOutDir] = await createTempDir();
try {
const command = FFmpeg()
.input(source)
.inputOptions([
'-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする(そうなるとは限らない)
'-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする(そうなるとは限らない)
])
.noAudio()
.videoFilters([
{
filter: 'select', // フレームのフィルタリング
options: {
e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタするVP9 とかはデコードしてみないとわからないっぽい)
},
},
{
filter: 'blackframe', // 暗いフレームの検出
options: {
amount: '0', // 暗さに関わらず全てのフレームで測定値を取る
},
},
{
filter: 'metadata',
options: {
mode: 'select', // フレーム選択モード
key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率(前のフィルタからのメタデータを参照する)
value: '50',
function: 'less', // 50% 未満のフレームを選択する50% 以上暗部があるフレームだと誤検知を招くかもしれないので)
},
},
{
filter: 'scale',
options: {
w: 299,
h: 299,
},
},
])
.format('image2')
.output(join(outDir, '%d.png'))
.outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない
const results: ReturnType<typeof judgePrediction>[] = [];
let frameIndex = 0;
let targetIndex = 0;
let nextIndex = 1;
for await (const path of asyncIterateFrames(outDir, command)) {
try {
const index = frameIndex++;
if (index !== targetIndex) {
continue;
}
targetIndex = nextIndex;
nextIndex += index; // fibonacci sequence によってフレーム数制限を掛ける
const result = await detectSensitive(path);
if (result) {
results.push(judgePrediction(result));
}
} finally {
fs.promises.unlink(path);
}
}
sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold);
porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn);
} finally {
disposeOutDir();
}
}
return [sensitive, porn];
}
async function* asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator<string, void> {
const watcher = new FSWatcher({
cwd,
disableGlobbing: true,
});
let finished = false;
command.once('end', () => {
finished = true;
watcher.close();
});
command.run();
for (let i = 1; true; i++) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
const current = `${i}.png`;
const next = `${i + 1}.png`;
const framePath = join(cwd, current);
if (await exists(join(cwd, next))) {
yield framePath;
} else if (!finished) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
watcher.add(next);
await new Promise<void>((resolve, reject) => {
watcher.on('add', function onAdd(path) {
if (path === next) { // 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている
watcher.unwatch(current);
watcher.off('add', onAdd);
resolve();
}
});
command.once('end', resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている
command.once('error', reject);
});
yield framePath;
} else if (await exists(framePath)) {
yield framePath;
} else {
return;
}
}
}
function exists(path: string): Promise<boolean> {
return fs.promises.access(path).then(() => true, () => false);
}
/**
* Detect MIME Type and extension
*/

View file

@ -156,6 +156,19 @@ export class DriveFile {
})
public isSensitive: boolean;
@Index()
@Column('boolean', {
default: false,
comment: 'Whether the DriveFile is NSFW. (predict)',
})
public maybeSensitive: boolean;
@Index()
@Column('boolean', {
default: false,
})
public maybePorn: boolean;
/**
* ()URLへの直リンクか否か
*/

View file

@ -188,6 +188,28 @@ export class Meta {
})
public recaptchaSecretKey: string | null;
@Column('enum', {
enum: ['none', 'all', 'local', 'remote'],
default: 'none',
})
public sensitiveMediaDetection: 'none' | 'all' | 'local' | 'remote';
@Column('enum', {
enum: ['medium', 'low', 'high', 'veryLow', 'veryHigh'],
default: 'medium',
})
public sensitiveMediaDetectionSensitivity: 'medium' | 'low' | 'high' | 'veryLow' | 'veryHigh';
@Column('boolean', {
default: false,
})
public setSensitiveFlagAutomatically: boolean;
@Column('boolean', {
default: false,
})
public enableSensitiveMediaDetectionForVideos: boolean;
@Column('integer', {
default: 1024,
comment: 'Drive capacity of a local user (MB)',
@ -432,4 +454,9 @@ export class Meta {
default: false,
})
public enableIpLogging: boolean;
@Column('boolean', {
default: true,
})
public enableActiveEmailValidation: boolean;
}

View file

@ -152,6 +152,11 @@ export class UserProfile {
})
public alwaysMarkNsfw: boolean;
@Column('boolean', {
default: false,
})
public autoSensitive: boolean;
@Column('boolean', {
default: false,
})

View file

@ -360,6 +360,7 @@ export const UserRepository = db.getRepository(User).extend({
injectFeaturedNote: profile!.injectFeaturedNote,
receiveAnnouncementEmail: profile!.receiveAnnouncementEmail,
alwaysMarkNsfw: profile!.alwaysMarkNsfw,
autoSensitive: profile!.autoSensitive,
carefulBot: profile!.carefulBot,
autoAcceptFollowed: profile!.autoAcceptFollowed,
noCrawle: profile!.noCrawle,

View file

@ -161,19 +161,19 @@ export const packedUserDetailedNotMeOnlySchema = {
type: 'array',
nullable: false, optional: false,
items: {
type: 'object',
nullable: false, optional: false,
properties: {
name: {
type: 'string',
nullable: false, optional: false,
},
value: {
type: 'string',
nullable: false, optional: false,
},
type: 'object',
nullable: false, optional: false,
properties: {
name: {
type: 'string',
nullable: false, optional: false,
},
maxLength: 4,
value: {
type: 'string',
nullable: false, optional: false,
},
},
maxLength: 4,
},
},
followersCount: {
@ -292,6 +292,10 @@ export const packedMeDetailedOnlySchema = {
type: 'boolean',
nullable: true, optional: false,
},
autoSensitive: {
type: 'boolean',
nullable: true, optional: false,
},
carefulBot: {
type: 'boolean',
nullable: true, optional: false,

View file

@ -8,7 +8,7 @@ import { queueLogger } from '../../logger.js';
import { addFile } from '@/services/drive/add-file.js';
import { format as dateFormat } from 'date-fns';
import { Users, Emojis } from '@/models/index.js';
import { } from '@/queue/types.js';
import { } from '@/queue/types.js';
import { createTemp, createTempDir } from '@/misc/create-temp.js';
import { downloadUrl } from '@/misc/download-url.js';
import config from '@/config/index.js';

View file

@ -1,6 +1,6 @@
import config from '@/config/index.js';
import Resolver from '../resolver.js';
import { IObject, IQuestion, isQuestion } from '../type.js';
import { IObject, IQuestion, isQuestion } from '../type.js';
import { apLogger } from '../logger.js';
import { Notes, Polls } from '@/models/index.js';
import { IPoll } from '@/models/entities/poll.js';

View file

@ -8,7 +8,7 @@ import renderEmoji from './emoji.js';
export const renderLike = async (noteReaction: NoteReaction, note: Note) => {
const reaction = noteReaction.reaction;
const object = {
const object = {
type: 'Like',
id: `${config.url}/likes/${noteReaction.id}`,
actor: `${config.url}/users/${noteReaction.userId}`,

View file

@ -68,11 +68,11 @@ export default (endpoint: IEndpoint, ctx: Koa.Context) => new Promise<void>((res
}
try {
UserIps.insert({
UserIps.createQueryBuilder().insert().values({
createdAt: new Date(),
userId: user.id,
ip: ip,
});
}).orIgnore(true).execute();
} catch {
}
}

View file

@ -12,13 +12,15 @@ export async function readNotification(
if (notificationIds.length === 0) return;
// Update documents
await Notifications.update({
const result = await Notifications.update({
id: In(notificationIds),
isRead: false,
}, {
isRead: true,
});
if (result.affected === 0) return;
if (!await Users.getHasUnreadNotification(userId)) return postReadAllNotifications(userId);
else return postReadNotifications(userId, notificationIds);
}
@ -27,7 +29,7 @@ export async function readNotificationByQuery(
userId: User['id'],
query: Record<string, any>
) {
const notificationIds = await Notifications.find({
const notificationIds = await Notifications.findBy({
...query,
notifieeId: userId,
isRead: false,

View file

@ -195,6 +195,22 @@ export const meta = {
type: 'string',
optional: true, nullable: true,
},
sensitiveMediaDetection: {
type: 'string',
optional: true, nullable: false,
},
sensitiveMediaDetectionSensitivity: {
type: 'string',
optional: true, nullable: false,
},
setSensitiveFlagAutomatically: {
type: 'boolean',
optional: true, nullable: false,
},
enableSensitiveMediaDetectionForVideos: {
type: 'boolean',
optional: true, nullable: false,
},
proxyAccountId: {
type: 'string',
optional: true, nullable: true,
@ -308,6 +324,10 @@ export const meta = {
type: 'boolean',
optional: true, nullable: false,
},
enableActiveEmailValidation: {
type: 'boolean',
optional: true, nullable: false,
},
},
},
} as const;
@ -370,6 +390,10 @@ export default define(meta, paramDef, async (ps, me) => {
blockedHosts: instance.blockedHosts,
hcaptchaSecretKey: instance.hcaptchaSecretKey,
recaptchaSecretKey: instance.recaptchaSecretKey,
sensitiveMediaDetection: instance.sensitiveMediaDetection,
sensitiveMediaDetectionSensitivity: instance.sensitiveMediaDetectionSensitivity,
setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically,
enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
proxyAccountId: instance.proxyAccountId,
twitterConsumerKey: instance.twitterConsumerKey,
twitterConsumerSecret: instance.twitterConsumerSecret,
@ -401,5 +425,6 @@ export default define(meta, paramDef, async (ps, me) => {
deeplAuthKey: instance.deeplAuthKey,
deeplIsPro: instance.deeplIsPro,
enableIpLogging: instance.enableIpLogging,
enableActiveEmailValidation: instance.enableActiveEmailValidation,
};
});

View file

@ -58,6 +58,7 @@ export default define(meta, paramDef, async (ps, me) => {
autoAcceptFollowed: profile.autoAcceptFollowed,
noCrawle: profile.noCrawle,
alwaysMarkNsfw: profile.alwaysMarkNsfw,
autoSensitive: profile.autoSensitive,
carefulBot: profile.carefulBot,
injectFeaturedNote: profile.injectFeaturedNote,
receiveAnnouncementEmail: profile.receiveAnnouncementEmail,

View file

@ -25,7 +25,7 @@ export const paramDef = {
offset: { type: 'integer', default: 0 },
sort: { type: 'string', enum: ['+follower', '-follower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] },
state: { type: 'string', enum: ['all', 'alive', 'available', 'admin', 'moderator', 'adminOrModerator', 'silenced', 'suspended'], default: 'all' },
origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'local' },
origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'combined' },
username: { type: 'string', nullable: true, default: null },
hostname: {
type: 'string',
@ -61,7 +61,7 @@ export default define(meta, paramDef, async (ps, me) => {
}
if (ps.hostname) {
query.andWhere('user.host like :hostname', { hostname: '%' + ps.hostname.toLowerCase() + '%' });
query.andWhere('user.host = :hostname', { hostname: ps.hostname.toLowerCase() });
}
switch (ps.sort) {

View file

@ -48,6 +48,10 @@ export const paramDef = {
enableRecaptcha: { type: 'boolean' },
recaptchaSiteKey: { type: 'string', nullable: true },
recaptchaSecretKey: { type: 'string', nullable: true },
sensitiveMediaDetection: { type: 'string', enum: ['none', 'all', 'local', 'remote'] },
sensitiveMediaDetectionSensitivity: { type: 'string', enum: ['medium', 'low', 'high', 'veryLow', 'veryHigh'] },
setSensitiveFlagAutomatically: { type: 'boolean' },
enableSensitiveMediaDetectionForVideos: { type: 'boolean' },
proxyAccountId: { type: 'string', format: 'misskey:id', nullable: true },
maintainerName: { type: 'string', nullable: true },
maintainerEmail: { type: 'string', nullable: true },
@ -97,6 +101,7 @@ export const paramDef = {
objectStorageSetPublicRead: { type: 'boolean' },
objectStorageS3ForcePathStyle: { type: 'boolean' },
enableIpLogging: { type: 'boolean' },
enableActiveEmailValidation: { type: 'boolean' },
},
required: [],
} as const;
@ -213,6 +218,22 @@ export default define(meta, paramDef, async (ps, me) => {
set.recaptchaSecretKey = ps.recaptchaSecretKey;
}
if (ps.sensitiveMediaDetection !== undefined) {
set.sensitiveMediaDetection = ps.sensitiveMediaDetection;
}
if (ps.sensitiveMediaDetectionSensitivity !== undefined) {
set.sensitiveMediaDetectionSensitivity = ps.sensitiveMediaDetectionSensitivity;
}
if (ps.setSensitiveFlagAutomatically !== undefined) {
set.setSensitiveFlagAutomatically = ps.setSensitiveFlagAutomatically;
}
if (ps.enableSensitiveMediaDetectionForVideos !== undefined) {
set.enableSensitiveMediaDetectionForVideos = ps.enableSensitiveMediaDetectionForVideos;
}
if (ps.proxyAccountId !== undefined) {
set.proxyAccountId = ps.proxyAccountId;
}
@ -401,6 +422,10 @@ export default define(meta, paramDef, async (ps, me) => {
set.enableIpLogging = ps.enableIpLogging;
}
if (ps.enableActiveEmailValidation !== undefined) {
set.enableActiveEmailValidation = ps.enableActiveEmailValidation;
}
await db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(Meta, {
order: {

View file

@ -2,6 +2,7 @@ import ms from 'ms';
import { addFile } from '@/services/drive/add-file.js';
import { DriveFiles } from '@/models/index.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import define from '../../../define.js';
import { apiLogger } from '../../../logger.js';
@ -35,6 +36,18 @@ export const meta = {
code: 'INVALID_FILE_NAME',
id: 'f449b209-0c60-4e51-84d5-29486263bfd4',
},
inappropriate: {
message: 'Cannot upload the file because it has been determined that it possibly contains inappropriate content.',
code: 'INAPPROPRIATE',
id: 'bec5bd69-fba3-43c9-b4fb-2894b66ad5d2',
},
noFreeSpace: {
message: 'Cannot upload the file because you have no free space of drive.',
code: 'NO_FREE_SPACE',
id: 'd08dbc37-a6a9-463a-8c47-96c32ab5f064',
},
},
} as const;
@ -87,6 +100,10 @@ export default define(meta, paramDef, async (ps, user, _, file, cleanup, ip, hea
if (e instanceof Error || typeof e === 'string') {
apiLogger.error(e);
}
if (e instanceof IdentifiableError) {
if (e.id === '282f77bf-5816-4f72-9264-aa14d8261a21') throw new ApiError(meta.errors.inappropriate);
if (e.id === 'c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6') throw new ApiError(meta.errors.noFreeSpace);
}
throw new ApiError();
} finally {
cleanup!();

View file

@ -1,8 +1,8 @@
import { publishDriveStream } from '@/services/stream.js';
import define from '../../../define.js';
import { ApiError } from '../../../error.js';
import { DriveFiles, DriveFolders, Users } from '@/models/index.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
import define from '../../../define.js';
import { ApiError } from '../../../error.js';
export const meta = {
tags: ['drive'],

View file

@ -3,17 +3,17 @@ import * as mfm from 'mfm-js';
import { publishMainStream, publishUserEvent } from '@/services/stream.js';
import acceptAllFollowRequests from '@/services/following/requests/accept-all.js';
import { publishToFollowers } from '@/services/i/update.js';
import define from '../../define.js';
import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js';
import { extractHashtags } from '@/misc/extract-hashtags.js';
import { updateUsertags } from '@/services/update-hashtag.js';
import { ApiError } from '../../error.js';
import { Users, DriveFiles, UserProfiles, Pages } from '@/models/index.js';
import { User } from '@/models/entities/user.js';
import { UserProfile } from '@/models/entities/user-profile.js';
import { notificationTypes } from '@/types.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import { langmap } from '@/misc/langmap.js';
import { ApiError } from '../../error.js';
import define from '../../define.js';
export const meta = {
tags: ['account'],
@ -57,7 +57,7 @@ export const meta = {
message: 'Invalid Regular Expression.',
code: 'INVALID_REGEXP',
id: '0d786918-10df-41cd-8f33-8dec7d9a89a5',
}
},
},
res: {
@ -77,7 +77,8 @@ export const paramDef = {
lang: { type: 'string', enum: [null, ...Object.keys(langmap)], nullable: true },
avatarId: { type: 'string', format: 'misskey:id', nullable: true },
bannerId: { type: 'string', format: 'misskey:id', nullable: true },
fields: { type: 'array',
fields: {
type: 'array',
minItems: 0,
maxItems: 16,
items: {
@ -102,6 +103,7 @@ export const paramDef = {
injectFeaturedNote: { type: 'boolean' },
receiveAnnouncementEmail: { type: 'boolean' },
alwaysMarkNsfw: { type: 'boolean' },
autoSensitive: { type: 'boolean' },
ffVisibility: { type: 'string', enum: ['public', 'followers', 'private'] },
pinnedPageId: { type: 'array', items: {
type: 'string', format: 'misskey:id',
@ -168,6 +170,7 @@ export default define(meta, paramDef, async (ps, _user, token) => {
if (typeof ps.injectFeaturedNote === 'boolean') profileUpdates.injectFeaturedNote = ps.injectFeaturedNote;
if (typeof ps.receiveAnnouncementEmail === 'boolean') profileUpdates.receiveAnnouncementEmail = ps.receiveAnnouncementEmail;
if (typeof ps.alwaysMarkNsfw === 'boolean') profileUpdates.alwaysMarkNsfw = ps.alwaysMarkNsfw;
if (typeof ps.autoSensitive === 'boolean') profileUpdates.autoSensitive = ps.autoSensitive;
if (ps.emailNotificationTypes !== undefined) profileUpdates.emailNotificationTypes = ps.emailNotificationTypes;
if (ps.avatarId) {
@ -211,7 +214,7 @@ export default define(meta, paramDef, async (ps, _user, token) => {
const newDescription = profileUpdates.description === undefined ? profile.description : profileUpdates.description;
if (newName != null) {
const tokens = mfm.parsePlain(newName);
const tokens = mfm.parseSimple(newName);
emojis = emojis.concat(extractCustomEmojisFromMfm(tokens!));
}

View file

@ -9,6 +9,9 @@ export const meta = {
requireCredential: false,
allowGet: true,
cacheSec: 60,
res: {
type: 'array',
optional: false, nullable: false,

View file

@ -16,6 +16,7 @@ export default class extends Channel {
constructor(id: string, connection: Channel['connection']) {
super(id, connection);
this.onNote = this.onNote.bind(this);
this.emitTypers = this.emitTypers.bind(this);
}
public async init(params: any) {

View file

@ -78,6 +78,7 @@ const nodeinfo2 = async () => {
enableEmail: meta.enableEmail,
enableServiceWorker: meta.enableServiceWorker,
proxyAccountName: proxyAccount ? proxyAccount.username : null,
themeColor: meta.themeColor || '#86b300',
},
};
};

View file

@ -14,9 +14,11 @@
// ブロックの中に入れないと、定義した変数がブラウザのグローバルスコープに登録されてしまい邪魔なので
(async () => {
window.onerror = (e) => {
console.error(e);
renderError('SOMETHING_HAPPENED', e);
};
window.onunhandledrejection = (e) => {
console.error(e);
renderError('SOMETHING_HAPPENED_IN_PROMISE', e);
};
@ -47,18 +49,30 @@
localStorage.setItem('localeVersion', v);
} else {
await checkUpdate();
renderError('LOCALE_FETCH_FAILED');
renderError('LOCALE_FETCH');
return;
}
}
//#endregion
//#region Script
import(`/assets/${CLIENT_ENTRY}`)
.catch(async e => {
await checkUpdate();
renderError('APP_FETCH_FAILED', e);
})
function importAppScript() {
import(`/assets/${CLIENT_ENTRY}`)
.catch(async e => {
await checkUpdate();
console.error(e);
renderError('APP_IMPORT', e);
});
}
// タイミングによっては、この時点でDOMの構築が済んでいる場合とそうでない場合とがある
if (document.readyState !== 'loading') {
importAppScript();
} else {
window.addEventListener('DOMContentLoaded', () => {
importAppScript();
});
}
//#endregion
//#region Theme
@ -102,44 +116,189 @@
document.head.appendChild(style);
}
// eslint-disable-next-line no-inner-declarations
async function addStyle(styleText) {
let css = document.createElement('style');
css.appendChild(document.createTextNode(styleText));
document.head.appendChild(css);
}
function renderError(code, details) {
let errorsElement = document.getElementById('errors');
if (!errorsElement) {
document.documentElement.innerHTML = `
<h1> An error has occurred. </h1>
<p>If the problem persists, please contact the administrator. You may also try the following options:</p>
<ul>
<li>Start <a href="/cli">the simple client</a></li>
<li>Attempt to repair in <a href="/bios">BIOS</a></li>
<li><a href="/flush">Flush preferences and cache</a></li>
</ul>
<hr>
document.body.innerHTML = `
<svg class="icon-warning" xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-alert-triangle" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M12 9v2m0 4v.01"></path>
<path d="M5 19h14a2 2 0 0 0 1.84 -2.75l-7.1 -12.25a2 2 0 0 0 -3.5 0l-7.1 12.25a2 2 0 0 0 1.75 2.75"></path>
</svg>
<h1>An error has occurred!</h1>
<button class="button-big" onclick="location.reload(true);">
<span class="button-label-big">Refresh</span>
</button>
<p class="dont-worry">Don't worry, it's (probably) not your fault.</p>
<p>If the problem persists after refreshing, please contact your instance's administrator.<br>You may also try the following options:</p>
<a href="/flush">
<button class="button-small">
<span class="button-label-small">Clear preferences and cache</span>
</button>
</a>
<br>
<a href="/cli">
<button class="button-small">
<span class="button-label-small">Start the simple client</span>
</button>
</a>
<br>
<a href="/bios">
<button class="button-small">
<span class="button-label-small">Start the repair tool</span>
</button>
</a>
<br>
<div id="errors"></div>
`;
errorsElement = document.getElementById('errors');
}
const detailsElement = document.createElement('details');
detailsElement.innerHTML = `<summary><code>ERROR CODE: ${code}</code></summary>${JSON.stringify(details)}`;
detailsElement.innerHTML = `
<br>
<summary>
<code>ERROR CODE: ${code}</code>
</summary>
<code>${JSON.stringify(details)}</code>`;
errorsElement.appendChild(detailsElement);
addStyle(`
* {
font-family: BIZ UDGothic, Roboto, HelveticaNeue, Arial, sans-serif;
}
#misskey_app,
#splash {
display: none !important;
}
body,
html {
background-color: #222;
color: #dfddcc;
justify-content: center;
margin: auto;
padding: 10px;
text-align: center;
}
button {
border-radius: 999px;
padding: 0px 12px 0px 12px;
border: none;
cursor: pointer;
margin-bottom: 12px;
}
.button-big {
background: linear-gradient(90deg, rgb(134, 179, 0), rgb(74, 179, 0));
line-height: 50px;
}
.button-big:hover {
background: rgb(153, 204, 0);
}
.button-small {
background: #444;
line-height: 40px;
}
.button-small:hover {
background: #555;
}
.button-label-big {
color: #222;
font-weight: bold;
font-size: 20px;
padding: 12px;
}
.button-label-small {
color: rgb(153, 204, 0);
font-size: 16px;
padding: 12px;
}
a {
color: rgb(134, 179, 0);
text-decoration: none;
}
p,
li {
font-size: 16px;
}
.dont-worry,
#msg {
font-size: 18px;
}
.icon-warning {
color: #dec340;
height: 4rem;
padding-top: 2rem;
}
h1 {
font-size: 32px;
}
code {
font-family: Fira, FiraCode, monospace;
}
details {
background: #333;
margin-bottom: 2rem;
padding: 0.5rem 1rem;
width: 40rem;
border-radius: 10px;
justify-content: center;
margin: auto;
}
summary {
cursor: pointer;
}
summary > * {
display: inline;
}
@media screen and (max-width: 500px) {
details {
width: 50%;
}
`)
}
// eslint-disable-next-line no-inner-declarations
async function checkUpdate() {
// TODO: サーバーが落ちている場合などのエラーハンドリング
const res = await fetch('/api/meta', {
method: 'POST',
cache: 'no-cache'
});
try {
const res = await fetch('/api/meta', {
method: 'POST',
cache: 'no-cache'
});
const meta = await res.json();
const meta = await res.json();
if (meta.version != v) {
localStorage.setItem('v', meta.version);
refresh();
if (meta.version != v) {
localStorage.setItem('v', meta.version);
refresh();
}
} catch (e) {
console.error(e);
renderError('UPDATE_CHECK', e);
throw e;
}
}

View file

@ -5,7 +5,7 @@ html
head
meta(charset='utf-8')
meta(name='application-name' content='Misskey')
title Misskey BIOS
title Misskey Repair Tool
style
include ../bios.css
script
@ -13,7 +13,7 @@ html
body
header
h1 Misskey BIOS #{version}
h1 Misskey Repair Tool #{version}
main
div.tabs
button#ls edit local storage

View file

@ -0,0 +1,48 @@
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import * as nsfw from 'nsfwjs';
import si from 'systeminformation';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const REQUIRED_CPU_FLAGS = ['avx2', 'fma'];
let isSupportedCpu: undefined | boolean = undefined;
let model: nsfw.NSFWJS;
export async function detectSensitive(path: string): Promise<nsfw.predictionType[] | null> {
try {
if (isSupportedCpu === undefined) {
const cpuFlags = await getCpuFlags();
isSupportedCpu = REQUIRED_CPU_FLAGS.every(required => cpuFlags.includes(required));
}
if (!isSupportedCpu) {
console.error('This CPU cannot use TensorFlow.');
return null;
}
const tf = await import('@tensorflow/tfjs-node');
if (model == null) model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 });
const buffer = await fs.promises.readFile(path);
const image = await tf.node.decodeImage(buffer, 3) as any;
try {
const predictions = await model.classify(image);
return predictions;
} finally {
image.dispose();
}
} catch (err) {
console.error(err);
return null;
}
}
async function getCpuFlags(): Promise<string[]> {
const str = await si.cpuFlags();
return str.split(/\s+/);
}

View file

@ -16,6 +16,7 @@ import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/i
import { genId } from '@/misc/gen-id.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { getS3 } from './s3.js';
import { InternalStorage } from './internal-storage.js';
import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
@ -349,9 +350,31 @@ export async function addFile({
requestIp = null,
requestHeaders = null,
}: AddFileArgs): Promise<DriveFile> {
const info = await getFileInfo(path);
let skipNsfwCheck = false;
const instance = await fetchMeta();
if (user == null) skipNsfwCheck = true;
if (instance.sensitiveMediaDetection === 'none') skipNsfwCheck = true;
if (user && instance.sensitiveMediaDetection === 'local' && Users.isRemoteUser(user)) skipNsfwCheck = true;
if (user && instance.sensitiveMediaDetection === 'remote' && Users.isLocalUser(user)) skipNsfwCheck = true;
const info = await getFileInfo(path, {
skipSensitiveDetection: skipNsfwCheck,
sensitiveThreshold: // 感度が高いほどしきい値は低くすることになる
instance.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 0.1 :
instance.sensitiveMediaDetectionSensitivity === 'high' ? 0.3 :
instance.sensitiveMediaDetectionSensitivity === 'low' ? 0.7 :
instance.sensitiveMediaDetectionSensitivity === 'veryLow' ? 0.9 :
0.5,
sensitiveThresholdForPorn: 0.75,
enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
});
logger.info(`${JSON.stringify(info)}`);
// 現状 false positive が多すぎて実用に耐えない
//if (info.porn && instance.disallowUploadWhenPredictedAsPorn) {
// throw new IdentifiableError('282f77bf-5816-4f72-9264-aa14d8261a21', 'Detected as porn.');
//}
// detect name
const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : 'untitled');
@ -387,7 +410,7 @@ export async function addFile({
// If usage limit exceeded
if (usage + info.size > driveCapacity) {
if (Users.isLocalUser(user)) {
throw new Error('no-free-space');
throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.');
} else {
// (アバターまたはバナーを含まず)最も古いファイルを削除する
deleteOldFile(await Users.findOneByOrFail({ id: user.id }) as IRemoteUser);
@ -441,6 +464,8 @@ export async function addFile({
file.isLink = isLink;
file.requestIp = requestIp;
file.requestHeaders = requestHeaders;
file.maybeSensitive = info.sensitive;
file.maybePorn = info.porn;
file.isSensitive = user
? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
(sensitive !== null && sensitive !== undefined)
@ -448,6 +473,9 @@ export async function addFile({
: false
: false;
if (info.sensitive && profile!.autoSensitive) file.isSensitive = true;
if (info.sensitive && instance.setSensitiveFlagAutomatically) file.isSensitive = true;
if (url !== null) {
file.src = url;

View file

@ -34,7 +34,7 @@ export async function fetchInstanceMetadata(instance: Instance, force = false):
const [favicon, icon, themeColor, name, description] = await Promise.all([
fetchFaviconUrl(instance, dom).catch(() => null),
fetchIconUrl(instance, dom, manifest).catch(() => null),
getThemeColor(dom, manifest).catch(() => null),
getThemeColor(info, dom, manifest).catch(() => null),
getSiteName(info, dom, manifest).catch(() => null),
getDescription(info, dom, manifest).catch(() => null),
]);
@ -208,8 +208,8 @@ async function fetchIconUrl(instance: Instance, doc: DOMWindow['document'] | nul
return null;
}
async function getThemeColor(doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
const themeColor = doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') || manifest?.theme_color;
async function getThemeColor(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
const themeColor = info?.metadata?.themeColor || doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') || manifest?.theme_color;
if (themeColor) {
const color = new tinycolor(themeColor);

View file

@ -64,6 +64,7 @@ export async function pushNotification<T extends keyof pushNotificationsTypes>(u
type,
body: type === 'notification' ? truncateNotification(body as Packed<'Notification'>) : body,
userId,
dateTime: (new Date()).getTime(),
}), {
proxy: config.proxy,
}).catch((err: any) => {

View file

@ -1,34 +1,37 @@
import { validate as validateEmail } from 'deep-email-validator';
import { UserProfiles } from '@/models/index.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
export async function validateEmailForAccount(emailAddress: string): Promise<{
available: boolean;
reason: null | 'used' | 'format' | 'disposable' | 'mx' | 'smtp';
}> {
const meta = await fetchMeta();
const exist = await UserProfiles.countBy({
emailVerified: true,
email: emailAddress,
});
const validated = await validateEmail({
const validated = meta.enableActiveEmailValidation ? await validateEmail({
email: emailAddress,
validateRegex: true,
validateMx: true,
validateTypo: false, // TLDを見ているみたいだけどclubとか弾かれるので
validateDisposable: true, // 捨てアドかどうかチェック
validateSMTP: false, // 日本だと25ポートが殆どのプロバイダーで塞がれていてタイムアウトになるので
});
}) : { valid: true };
const available = exist === 0 && validated.valid;
return {
available,
reason: available ? null :
exist !== 0 ? 'used' :
validated.reason === 'regex' ? 'format' :
validated.reason === 'disposable' ? 'disposable' :
validated.reason === 'mx' ? 'mx' :
validated.reason === 'smtp' ? 'smtp' :
null,
exist !== 0 ? 'used' :
validated.reason === 'regex' ? 'format' :
validated.reason === 'disposable' ? 'disposable' :
validated.reason === 'mx' ? 'mx' :
validated.reason === 'smtp' ? 'smtp' :
null,
};
}

View file

@ -10,9 +10,11 @@ const _dirname = dirname(_filename);
describe('Get file info', () => {
it('Empty file', async (async () => {
const path = `${_dirname}/resources/emptyfile`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 0,
md5: 'd41d8cd98f00b204e9800998ecf8427e',
@ -28,9 +30,11 @@ describe('Get file info', () => {
it('Generic JPEG', async (async () => {
const path = `${_dirname}/resources/Lenna.jpg`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 25360,
md5: '091b3f259662aa31e2ffef4519951168',
@ -46,9 +50,11 @@ describe('Get file info', () => {
it('Generic APNG', async (async () => {
const path = `${_dirname}/resources/anime.png`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 1868,
md5: '08189c607bea3b952704676bb3c979e0',
@ -64,9 +70,11 @@ describe('Get file info', () => {
it('Generic AGIF', async (async () => {
const path = `${_dirname}/resources/anime.gif`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 2248,
md5: '32c47a11555675d9267aee1a86571e7e',
@ -82,9 +90,11 @@ describe('Get file info', () => {
it('PNG with alpha', async (async () => {
const path = `${_dirname}/resources/with-alpha.png`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 3772,
md5: 'f73535c3e1e27508885b69b10cf6e991',
@ -100,9 +110,11 @@ describe('Get file info', () => {
it('Generic SVG', async (async () => {
const path = `${_dirname}/resources/image.svg`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 505,
md5: 'b6f52b4b021e7b92cdd04509c7267965',
@ -119,9 +131,11 @@ describe('Get file info', () => {
it('SVG with XML definition', async (async () => {
// https://github.com/misskey-dev/misskey/issues/4413
const path = `${_dirname}/resources/with-xml-def.svg`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 544,
md5: '4b7a346cde9ccbeb267e812567e33397',
@ -137,9 +151,11 @@ describe('Get file info', () => {
it('Dimension limit', async (async () => {
const path = `${_dirname}/resources/25000x25000.png`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 75933,
md5: '268c5dde99e17cf8fe09f1ab3f97df56',
@ -155,9 +171,11 @@ describe('Get file info', () => {
it('Rotate JPEG', async (async () => {
const path = `${_dirname}/resources/rotate.jpg`;
const info = await getFileInfo(path) as any;
const info = await getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
assert.deepStrictEqual(info, {
size: 12624,
md5: '68d5b2d8d1d1acbbce99203e3ec3857e',