heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'misskey/develop' into future-2024-04-10

Browse source
This commit is contained in:
dakkar 2024-04-11 13:39:55 +01:00
commit a3b4ca782a
78 changed files with 3068 additions and 2243 deletions
Download patch file Download diff file Expand all files Collapse all files

3
packages/backend/.swcrc
View file

@ -19,5 +19,6 @@
},
"target": "es2022"
},
"minify": false
"minify": false,
"sourceMaps": "inline"
}

10
packages/backend/package.json
View file

@ -11,15 +11,15 @@
"start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js",
"migrate": "pnpm typeorm migration:run -d ormconfig.js",
"revert": "pnpm typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./check_connect.js",
"check:connect": "node ./scripts/check_connect.js",
"build": "swc src -d built -D",
"build:test": "swc test-server -d built-test -D --config-file test-server/.swcrc",
"watch:swc": "swc src -d built -D -w",
"build:tsc": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json",
"watch": "node watch.mjs",
"watch": "node ./scripts/watch.mjs",
"restart": "pnpm build && pnpm start",
"dev": "nodemon -w src -e ts,js,mjs,cjs,json --exec \"cross-env NODE_ENV=development pnpm run restart\"",
"typecheck": "pnpm --filter megalodon build && tsc --noEmit",
"dev": "node ./scripts/dev.mjs",
"typecheck": "pnpm --filter megalodon build && tsc --noEmit && tsc -p test --noEmit",
"eslint": "eslint --quiet \"src/**/*.ts\"",
"lint": "pnpm typecheck && pnpm eslint",
"jest": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --config jest.config.unit.cjs",
@ -31,7 +31,7 @@
"test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e",
"test-and-coverage": "pnpm jest-and-coverage",
"test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e",
"generate-api-json": "pnpm build && node ./generate_api_json.js"
"generate-api-json": "pnpm build && node ./scripts/generate_api_json.js"
},
"optionalDependencies": {
"@swc/core-android-arm64": "1.3.11",

2
packages/backend/check_connect.js → packages/backend/scripts/check_connect.js
View file

@ -4,7 +4,7 @@
*/
import Redis from 'ioredis';
import { loadConfig } from './built/config.js';
import { loadConfig } from '../built/config.js';
const config = loadConfig();
const redis = new Redis(config.redis);

61
packages/backend/scripts/dev.mjs Normal file
View file

@ -0,0 +1,61 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { execa, execaNode } from 'execa';
/** @type {import('execa').ExecaChildProcess | undefined} */
let backendProcess;
async function execBuildAssets() {
await execa('pnpm', ['run', 'build-assets'], {
cwd: '../../',
stdout: process.stdout,
stderr: process.stderr,
})
}
function execStart() {
// pnpm run start を呼び出したいが、windowsだとプロセスグループ単位でのkillが出来ずゾンビプロセス化するので
// 上記と同等の動きをするコマンドで子・孫プロセスを作らないようにしたい
backendProcess = execaNode('./built/boot/entry.js', [], {
stdout: process.stdout,
stderr: process.stderr,
env: {
'NODE_ENV': 'development',
},
});
}
async function killProc() {
if (backendProcess) {
backendProcess.kill();
await new Promise(resolve => backendProcess.on('exit', resolve));
backendProcess = undefined;
}
}
(async () => {
execaNode(
'./node_modules/nodemon/bin/nodemon.js',
[
'-w', 'src',
'-e', 'ts,js,mjs,cjs,json',
'--exec', 'pnpm', 'run', 'build',
],
{
stdio: [process.stdin, process.stdout, process.stderr, 'ipc'],
})
.on('message', async (message) => {
if (message.type === 'exit') {
// かならずbuild->build-assetsの順番で呼び出したいので、
// 少々トリッキーだがnodemonからのexitイベントを利用してbuild-assets->startを行う。
// pnpm restartをbuildが終わる前にbuild-assetsが動いてしまうので、バラバラに呼び出す必要がある
await killProc();
await execBuildAssets();
execStart();
}
})
})();

4
packages/backend/generate_api_json.js → packages/backend/scripts/generate_api_json.js
View file

@ -3,8 +3,8 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { loadConfig } from './built/config.js'
import { genOpenapiSpec } from './built/server/api/openapi/gen-spec.js'
import { loadConfig } from '../built/config.js'
import { genOpenapiSpec } from '../built/server/api/openapi/gen-spec.js'
import { writeFileSync } from "node:fs";
const config = loadConfig();

0
packages/backend/watch.mjs → packages/backend/scripts/watch.mjs
View file

4
packages/backend/src/core/AccountMoveService.ts
View file

@ -305,7 +305,7 @@ export class AccountMoveService {
let resultUser: MiLocalUser | MiRemoteUser | null = null;
if (this.userEntityService.isRemoteUser(dst)) {
if ((new Date()).getTime() - (dst.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
if (Date.now() - (dst.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
await this.apPersonService.updatePerson(dst.uri);
}
dst = await this.apPersonService.fetchPerson(dst.uri) ?? dst;
@ -321,7 +321,7 @@ export class AccountMoveService {
if (!src) continue; // oldAccountを探してもこのサーバーに存在しない場合はフォロー関係もないということなのでスルー
if (this.userEntityService.isRemoteUser(dst)) {
if ((new Date()).getTime() - (src.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
if (Date.now() - (src.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
await this.apPersonService.updatePerson(srcUri);
}

50
packages/backend/src/core/FileInfoService.ts
View file

@ -8,11 +8,13 @@ import * as crypto from 'node:crypto';
import * as stream from 'node:stream/promises';
import { Injectable } from '@nestjs/common';
import * as fileType from 'file-type';
import FFmpeg from 'fluent-ffmpeg';
import isSvg from 'is-svg';
import probeImageSize from 'probe-image-size';
import sharp from 'sharp';
import { sharpBmp } from '@misskey-dev/sharp-read-bmp';
import { encode } from 'blurhash';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
export type FileInfo = {
@ -43,8 +45,12 @@ const TYPE_SVG = {
@Injectable()
export class FileInfoService {
private logger: Logger;
constructor(
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('file-info');
}
/**
@ -147,6 +153,34 @@ export class FileInfoService {
return mime;
}
/**
* ビデオファイルにビデオトラックがあるかどうかチェック
* （ない場合：m4a, webmなど）
*
* @param path ファイルパス
* @returns ビデオトラックがあるかどうか（エラー発生時は常に`true`を返す）
*/
@bindThis
private hasVideoTrackOnVideoFile(path: string): Promise<boolean> {
const sublogger = this.logger.createSubLogger('ffprobe');
sublogger.info(`Checking the video file. File path: ${path}`);
return new Promise((resolve) => {
try {
FFmpeg.ffprobe(path, (err, metadata) => {
if (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
resolve(true);
return;
}
resolve(metadata.streams.some((stream) => stream.codec_type === 'video'));
});
} catch (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
resolve(true);
}
});
}
/**
* Detect MIME Type and extension
*/
@ -169,6 +203,20 @@ export class FileInfoService {
return TYPE_SVG;
}
if ((type.mime.startsWith('video') || type.mime === 'application/ogg') && !(await this.hasVideoTrackOnVideoFile(path))) {
const newMime = `audio/${type.mime.split('/')[1]}`;
if (newMime === 'audio/mp4') {
return {
mime: 'audio/mp4',
ext: 'm4a',
};
}
return {
mime: newMime,
ext: type.ext,
};
}
return {
mime: this.fixMime(type.mime),
ext: type.ext,

2
packages/backend/src/core/PushNotificationService.ts
View file

@ -101,7 +101,7 @@ export class PushNotificationService implements OnApplicationShutdown {
type,
body: (type === 'notification' || type === 'unreadAntennaNote') ? truncateBody(type, body) : body,
userId,
dateTime: (new Date()).getTime(),
dateTime: Date.now(),
}), {
proxy: this.config.proxy,
}).catch((err: any) => {

2
packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View file

@ -63,7 +63,7 @@ export class CleanRemoteFilesProcessorService {
isLink: false,
});
job.updateProgress(deletedCount / total);
job.updateProgress(100 / total * deletedCount);
}
this.logger.succ('All cached remote files has been deleted.');

2
packages/backend/src/server/api/endpoints/i/import-blocking.ts
View file

@ -75,7 +75,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const checkMoving = await this.accountMoveService.validateAlsoKnownAs(
me,
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > (new Date()).getTime(),
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(),
true,
);
if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile);

2
packages/backend/src/server/api/endpoints/i/import-following.ts
View file

@ -75,7 +75,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const checkMoving = await this.accountMoveService.validateAlsoKnownAs(
me,
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > (new Date()).getTime(),
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(),
true,
);
if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile);

2
packages/backend/src/server/api/endpoints/i/import-muting.ts
View file

@ -75,7 +75,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const checkMoving = await this.accountMoveService.validateAlsoKnownAs(
me,
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > (new Date()).getTime(),
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(),
true,
);
if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile);

2
packages/backend/src/server/api/endpoints/i/import-user-lists.ts
View file

@ -74,7 +74,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const checkMoving = await this.accountMoveService.validateAlsoKnownAs(
me,
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > (new Date()).getTime(),
(old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(),
true,
);
if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile);

16
packages/backend/src/server/api/endpoints/notes/translate.ts
View file

@ -21,7 +21,7 @@ export const meta = {
res: {
type: 'object',
optional: false, nullable: false,
optional: true, nullable: false,
properties: {
sourceLang: { type: 'string' },
text: { type: 'string' },
@ -39,6 +39,11 @@ export const meta = {
code: 'NO_SUCH_NOTE',
id: 'bea9b03f-36e0-49c5-a4db-627a029f8971',
},
cannotTranslateInvisibleNote: {
message: 'Cannot translate invisible note.',
code: 'CANNOT_TRANSLATE_INVISIBLE_NOTE',
id: 'ea29f2ca-c368-43b3-aaf1-5ac3e74bbe5d',
},
},
} as const;
@ -72,21 +77,20 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
});
if (!(await this.noteEntityService.isVisibleForMe(note, me.id))) {
return 204; // TODO: 良い感じのエラー返す
throw new ApiError(meta.errors.cannotTranslateInvisibleNote);
}
if (note.text == null) {
return 204;
return;
}
const instance = await this.metaService.fetch();
if (instance.deeplAuthKey == null && !instance.deeplFreeMode) {
return 204; // TODO: 良い感じのエラー返す
if (instance.deeplAuthKey == null && !instance.deeplFree throw new ApiError(meta.errors.unavailable);
}
if (instance.deeplFreeMode && !instance.deeplFreeInstance) {
return 204;
throw new ApiError(meta.errors.unavailable);
}
let targetLang = ps.targetLang;

7
packages/backend/src/server/api/endpoints/users/following.ts
View file

@ -6,6 +6,7 @@
import { IsNull } from 'typeorm';
import { Inject, Injectable } from '@nestjs/common';
import type { UsersRepository, FollowingsRepository, UserProfilesRepository } from '@/models/_.js';
import { birthdaySchema } from '@/models/User.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { QueryService } from '@/core/QueryService.js';
import { FollowingEntityService } from '@/core/entities/FollowingEntityService.js';
@ -66,7 +67,7 @@ export const paramDef = {
description: 'The local host is represented with `null`.',
},
birthday: { type: 'string', nullable: true },
birthday: { ...birthdaySchema, nullable: true },
},
anyOf: [
{ required: ['userId'] },
@ -127,9 +128,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (ps.birthday) {
try {
const d = new Date(ps.birthday);
d.setHours(0, 0, 0, 0);
const birthday = `${(d.getMonth() + 1).toString().padStart(2, '0')}-${d.getDate().toString().padStart(2, '0')}`;
const birthday = ps.birthday.substring(5, 10);
const birthdayUserQuery = this.userProfilesRepository.createQueryBuilder('user_profile');
birthdayUserQuery.select('user_profile.userId')
.where(`SUBSTR(user_profile.birthday, 6, 5) = '${birthday}'`);

107
packages/backend/test/e2e/note.ts
View file

@ -8,12 +8,13 @@ process.env.NODE_ENV = 'test';
import * as assert from 'assert';
import { MiNote } from '@/models/Note.js';
import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { api, initTestDb, post, signup, uploadFile, uploadUrl } from '../utils.js';
import { api, initTestDb, post, role, signup, uploadFile, uploadUrl } from '../utils.js';
import type * as misskey from 'misskey-js';
describe('Note', () => {
let Notes: any;
let root: misskey.entities.SignupResponse;
let alice: misskey.entities.SignupResponse;
let bob: misskey.entities.SignupResponse;
let tom: misskey.entities.SignupResponse;
@ -21,6 +22,7 @@ describe('Note', () => {
beforeAll(async () => {
const connection = await initTestDb(true);
Notes = connection.getRepository(MiNote);
root = await signup({ username: 'root' });
alice = await signup({ username: 'alice' });
bob = await signup({ username: 'bob' });
tom = await signup({ username: 'tom', host: 'example.com' });
@ -473,14 +475,14 @@ describe('Note', () => {
value: true,
},
} as any,
}, alice);
}, root);
assert.strictEqual(res.status, 200);
const assign = await api('admin/roles/assign', {
userId: alice.id,
roleId: res.body.id,
}, alice);
}, root);
assert.strictEqual(assign.status, 204);
assert.strictEqual(file.body!.isSensitive, false);
@ -508,11 +510,11 @@ describe('Note', () => {
await api('admin/roles/unassign', {
userId: alice.id,
roleId: res.body.id,
});
}, root);
await api('admin/roles/delete', {
roleId: res.body.id,
}, alice);
}, root);
});
});
@ -644,7 +646,7 @@ describe('Note', () => {
sensitiveWords: [
'test',
],
}, alice);
}, root);
assert.strictEqual(sensitive.status, 204);
@ -663,7 +665,7 @@ describe('Note', () => {
sensitiveWords: [
'/Test/i',
],
}, alice);
}, root);
assert.strictEqual(sensitive.status, 204);
@ -680,7 +682,7 @@ describe('Note', () => {
sensitiveWords: [
'Test hoge',
],
}, alice);
}, root);
assert.strictEqual(sensitive.status, 204);
@ -697,7 +699,7 @@ describe('Note', () => {
prohibitedWords: [
'test',
],
}, alice);
}, root);
assert.strictEqual(prohibited.status, 204);
@ -716,7 +718,7 @@ describe('Note', () => {
prohibitedWords: [
'/Test/i',
],
}, alice);
}, root);
assert.strictEqual(prohibited.status, 204);
@ -733,7 +735,7 @@ describe('Note', () => {
prohibitedWords: [
'Test hoge',
],
}, alice);
}, root);
assert.strictEqual(prohibited.status, 204);
@ -750,7 +752,7 @@ describe('Note', () => {
prohibitedWords: [
'test',
],
}, alice);
}, root);
assert.strictEqual(prohibited.status, 204);
@ -785,7 +787,7 @@ describe('Note', () => {
value: 0,
},
} as any,
}, alice);
}, root);
assert.strictEqual(res.status, 200);
@ -794,7 +796,7 @@ describe('Note', () => {
const assign = await api('admin/roles/assign', {
userId: alice.id,
roleId: res.body.id,
}, alice);
}, root);
assert.strictEqual(assign.status, 204);
@ -810,11 +812,11 @@ describe('Note', () => {
await api('admin/roles/unassign', {
userId: alice.id,
roleId: res.body.id,
});
}, root);
await api('admin/roles/delete', {
roleId: res.body.id,
}, alice);
}, root);
});
test('ダイレクト投稿もエラーになる', async () => {
@ -839,7 +841,7 @@ describe('Note', () => {
value: 0,
},
} as any,
}, alice);
}, root);
assert.strictEqual(res.status, 200);
@ -848,7 +850,7 @@ describe('Note', () => {
const assign = await api('admin/roles/assign', {
userId: alice.id,
roleId: res.body.id,
}, alice);
}, root);
assert.strictEqual(assign.status, 204);
@ -866,11 +868,11 @@ describe('Note', () => {
await api('admin/roles/unassign', {
userId: alice.id,
roleId: res.body.id,
});
}, root);
await api('admin/roles/delete', {
roleId: res.body.id,
}, alice);
}, root);
});
test('ダイレクトの宛先とメンションが同じ場合は重複してカウントしない', async () => {
@ -895,7 +897,7 @@ describe('Note', () => {
value: 1,
},
} as any,
}, alice);
}, root);
assert.strictEqual(res.status, 200);
@ -904,7 +906,7 @@ describe('Note', () => {
const assign = await api('admin/roles/assign', {
userId: alice.id,
roleId: res.body.id,
}, alice);
}, root);
assert.strictEqual(assign.status, 204);
@ -921,11 +923,11 @@ describe('Note', () => {
await api('admin/roles/unassign', {
userId: alice.id,
roleId: res.body.id,
});
}, root);
await api('admin/roles/delete', {
roleId: res.body.id,
}, alice);
}, root);
});
});
@ -960,4 +962,61 @@ describe('Note', () => {
assert.strictEqual(mainNote.repliesCount, 0);
});
});
describe('notes/translate', () => {
describe('翻訳機能の利用が許可されていない場合', () => {
let cannotTranslateRole: misskey.entities.Role;
beforeAll(async () => {
cannotTranslateRole = await role(root, {}, { canUseTranslator: false });
await api('admin/roles/assign', { roleId: cannotTranslateRole.id, userId: alice.id }, root);
});
test('翻訳機能の利用が許可されていない場合翻訳できない', async () => {
const aliceNote = await post(alice, { text: 'Hello' });
const res = await api('notes/translate', {
noteId: aliceNote.id,
targetLang: 'ja',
}, alice);
assert.strictEqual(res.status, 400);
assert.strictEqual(res.body.error.code, 'UNAVAILABLE');
});
afterAll(async () => {
await api('admin/roles/unassign', { roleId: cannotTranslateRole.id, userId: alice.id }, root);
});
});
test('存在しないノートは翻訳できない', async () => {
const res = await api('notes/translate', { noteId: 'foo', targetLang: 'ja' }, alice);
assert.strictEqual(res.status, 400);
assert.strictEqual(res.body.error.code, 'NO_SUCH_NOTE');
});
test('不可視なノートは翻訳できない', async () => {
const aliceNote = await post(alice, { visibility: 'followers', text: 'Hello' });
const bobTranslateAttempt = await api('notes/translate', { noteId: aliceNote.id, targetLang: 'ja' }, bob);
assert.strictEqual(bobTranslateAttempt.status, 400);
assert.strictEqual(bobTranslateAttempt.body.error.code, 'CANNOT_TRANSLATE_INVISIBLE_NOTE');
});
test('text: null なノートを翻訳すると空のレスポンスが返ってくる', async () => {
const aliceNote = await post(alice, { text: null, poll: { choices: ['kinoko', 'takenoko'] } });
const res = await api('notes/translate', { noteId: aliceNote.id, targetLang: 'ja' }, alice);
assert.strictEqual(res.status, 204);
});
test('サーバーに DeepL 認証キーが登録されていない場合翻訳できない', async () => {
const aliceNote = await post(alice, { text: 'Hello' });
const res = await api('notes/translate', { noteId: aliceNote.id, targetLang: 'ja' }, alice);
// NOTE: デフォルトでは登録されていないので落ちる
assert.strictEqual(res.status, 400);
assert.strictEqual(res.body.error.code, 'UNAVAILABLE');
});
});
});

BIN
packages/backend/test/resources/kick_gaba7.m4a Normal file
View file

Binary file not shown.

27
packages/backend/test/unit/FileInfoService.ts
View file

@ -14,6 +14,7 @@ import { afterAll, beforeAll, describe, test } from '@jest/globals';
import { GlobalModule } from '@/GlobalModule.js';
import { FileInfoService } from '@/core/FileInfoService.js';
//import { DI } from '@/di-symbols.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { TestingModule } from '@nestjs/testing';
import type { MockFunctionMetadata } from 'jest-mock';
@ -33,6 +34,7 @@ describe('FileInfoService', () => {
GlobalModule,
],
providers: [
LoggerService,
FileInfoService,
],
})
@ -318,8 +320,26 @@ describe('FileInfoService', () => {
});
});
/*
* video/webmとして検出されてしまう
test('MPEG-4 AUDIO (M4A)', async () => {
const path = `${resources}/kick_gaba7.m4a`;
const info = await fileInfoService.getFileInfo(path, { skipSensitiveDetection: true }) as any;
delete info.warnings;
delete info.blurhash;
delete info.sensitive;
delete info.porn;
delete info.width;
delete info.height;
delete info.orientation;
assert.deepStrictEqual(info, {
size: 9817,
md5: '74c9279a4abe98789565f1dc1a541a42',
type: {
mime: 'audio/mp4',
ext: 'm4a',
},
});
});
test('WEBM AUDIO', async () => {
const path = `${resources}/kick_gaba7.webm`;
const info = await fileInfoService.getFileInfo(path) as any;
@ -332,13 +352,12 @@ describe('FileInfoService', () => {
delete info.orientation;
assert.deepStrictEqual(info, {
size: 8879,
md5: '3350083dec312419cfdc06c16413aca7',
md5: '53bc1adcb6acbbda67ff9bd484896438',
type: {
mime: 'audio/webm',
ext: 'webm',
},
});
});
*/
});
});