feat(backend): allow disabling cache for sensitive files (#11245)

* feat(backend): allow disabling cache for sensitive files

* Update CHANGELOG.md

* fix storybook

* Update locales/ja-JP.yml

---------

Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>
This commit is contained in:
Kagami Sascha Rosylight 2023-07-15 13:12:20 +02:00 committed by GitHub
parent f96ed9a3f3
commit d5f30ecb86
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 190 additions and 16 deletions

View file

@ -52,6 +52,7 @@
- nsfwjs のモデルロードを排他することで、重複ロードによってメモリ使用量が増加しないように
- 連合の配送ジョブのパフォーマンスを向上ロック機構の見直し、Redisキャッシュの活用
- featuredートのsignedGet回数を減らしました
- リモートサーバーからのNSFW映像のキャッシュだけを無効化できるオプションを追加
- MeilisearchにIndexするートの範囲を設定できるように
- Fix: Remove Meilisearch index when notes are deleted
- Fix: 非英語環境でのPostgreSQLのエラーハンドリングを修正

2
locales/index.d.ts vendored
View file

@ -159,6 +159,8 @@ export interface Locale {
"settingGuide": string;
"cacheRemoteFiles": string;
"cacheRemoteFilesDescription": string;
"cacheRemoteSensitiveFiles": string;
"cacheRemoteSensitiveFilesDescription": string;
"flagAsBot": string;
"flagAsBotDescription": string;
"flagAsCat": string;

View file

@ -156,6 +156,8 @@ addEmoji: "絵文字を追加"
settingGuide: "おすすめ設定"
cacheRemoteFiles: "リモートのファイルをキャッシュする"
cacheRemoteFilesDescription: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。サーバーのストレージを節約できますが、サムネイルが生成されないので通信量が増加します。"
cacheRemoteSensitiveFiles: "リモートのセンシティブなファイルをキャッシュする"
cacheRemoteSensitiveFilesDescription: "この設定を無効にすると、リモートのセンシティブなファイルはキャッシュせず直リンクするようになります。"
flagAsBot: "Botとして設定"
flagAsBotDescription: "このアカウントがプログラムによって運用される場合は、このフラグをオンにします。オンにすると、反応の連鎖を防ぐためのフラグとして他の開発者に役立ったり、Misskeyのシステム上での扱いがBotに合ったものになります。"
flagAsCat: "にゃああああああああああああああ!!!!!!!!!!!!"

View file

@ -0,0 +1,11 @@
export class NsfwCache1689102832143 {
name = 'NsfwCache1689102832143'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "cacheRemoteSensitiveFiles" boolean NOT NULL DEFAULT true`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "cacheRemoteSensitiveFiles"`);
}
}

View file

@ -1,7 +1,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type { RemoteUser } from '@/models/entities/User.js';
import type { DriveFile } from '@/models/entities/DriveFile.js';
import { MetaService } from '@/core/MetaService.js';
@ -20,9 +19,6 @@ export class ApImageService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
@ -47,7 +43,7 @@ export class ApImageService {
const image = await this.apResolverService.createResolver().resolve(value);
if (image.url == null) {
throw new Error('invalid image: url not privided');
throw new Error('invalid image: url not provided');
}
if (typeof image.url !== 'string') {
@ -62,12 +58,17 @@ export class ApImageService {
const instance = await this.metaService.fetch();
// Cache if remote file cache is on AND either
// 1. remote sensitive file is also on
// 2. or the image is not sensitive
const shouldBeCached = instance.cacheRemoteFiles && (instance.cacheRemoteSensitiveFiles || !image.sensitive);
const file = await this.driveService.uploadFromUrl({
url: image.url,
user: actor,
uri: image.url,
sensitive: image.sensitive,
isLink: !instance.cacheRemoteFiles,
isLink: !shouldBeCached,
comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH),
});
if (!file.isLink || file.url === image.url) return file;

View file

@ -1,7 +1,6 @@
import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
import { id } from '../id.js';
import { User } from './User.js';
import type { Clip } from './Clip.js';
@Entity()
export class Meta {
@ -126,6 +125,11 @@ export class Meta {
})
public cacheRemoteFiles: boolean;
@Column('boolean', {
default: true,
})
public cacheRemoteSensitiveFiles: boolean;
@Column({
...id(),
nullable: true,

View file

@ -1,5 +1,4 @@
import { Inject, Injectable } from '@nestjs/common';
import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { MetaService } from '@/core/MetaService.js';
import type { Config } from '@/config.js';
@ -20,6 +19,10 @@ export const meta = {
type: 'boolean',
optional: false, nullable: false,
},
cacheRemoteSensitiveFiles: {
type: 'boolean',
optional: false, nullable: false,
},
emailRequiredForSignup: {
type: 'boolean',
optional: false, nullable: false,
@ -332,6 +335,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
enableServiceWorker: instance.enableServiceWorker,
translatorAvailable: instance.deeplAuthKey != null,
cacheRemoteFiles: instance.cacheRemoteFiles,
cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles,
pinnedUsers: instance.pinnedUsers,
hiddenTags: instance.hiddenTags,
blockedHosts: instance.blockedHosts,

View file

@ -43,6 +43,7 @@ export const paramDef = {
defaultLightTheme: { type: 'string', nullable: true },
defaultDarkTheme: { type: 'string', nullable: true },
cacheRemoteFiles: { type: 'boolean' },
cacheRemoteSensitiveFiles: { type: 'boolean' },
emailRequiredForSignup: { type: 'boolean' },
enableHcaptcha: { type: 'boolean' },
hcaptchaSiteKey: { type: 'string', nullable: true },
@ -193,6 +194,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
set.cacheRemoteFiles = ps.cacheRemoteFiles;
}
if (ps.cacheRemoteSensitiveFiles !== undefined) {
set.cacheRemoteSensitiveFiles = ps.cacheRemoteSensitiveFiles;
}
if (ps.emailRequiredForSignup !== undefined) {
set.emailRequiredForSignup = ps.emailRequiredForSignup;
}

View file

@ -83,6 +83,10 @@ export const meta = {
type: 'boolean',
optional: false, nullable: false,
},
cacheRemoteSensitiveFiles: {
type: 'boolean',
optional: false, nullable: false,
},
emailRequiredForSignup: {
type: 'boolean',
optional: false, nullable: false,
@ -272,7 +276,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
.orWhere('ads.dayOfWeek = 0');
}))
.getMany();
const response: any = {
maintainerName: instance.maintainerName,
maintainerEmail: instance.maintainerEmail,
@ -329,6 +333,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
...(ps.detail ? {
cacheRemoteFiles: instance.cacheRemoteFiles,
cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles,
requireSetup: (await this.usersRepository.countBy({
host: IsNull(),
})) === 0,

View file

@ -4,6 +4,7 @@ import * as assert from 'assert';
import { Test } from '@nestjs/testing';
import { jest } from '@jest/globals';
import { ApImageService } from '@/core/activitypub/models/ApImageService.js';
import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
@ -11,9 +12,12 @@ import { GlobalModule } from '@/GlobalModule.js';
import { CoreModule } from '@/core/CoreModule.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IActor, ICollection, IPost } from '@/core/activitypub/type.js';
import { Note } from '@/models/index.js';
import type { IActor, IApDocument, ICollection, IPost } from '@/core/activitypub/type.js';
import { Meta, Note } from '@/models/index.js';
import { secureRndstr } from '@/misc/secure-rndstr.js';
import { DownloadService } from '@/core/DownloadService.js';
import { MetaService } from '@/core/MetaService.js';
import type { RemoteUser } from '@/models/entities/User.js';
import { MockResolver } from '../misc/mock-resolver.js';
const host = 'https://host1.test';
@ -63,16 +67,47 @@ function createRandomFeaturedCollection(actor: NonTransientIActor, length: numbe
};
}
async function createRandomRemoteUser(
resolver: MockResolver,
personService: ApPersonService,
): Promise<RemoteUser> {
const actor = createRandomActor();
resolver.register(actor.id, actor);
return await personService.createPerson(actor.id, resolver);
}
describe('ActivityPub', () => {
let imageService: ApImageService;
let noteService: ApNoteService;
let personService: ApPersonService;
let rendererService: ApRendererService;
let resolver: MockResolver;
const metaInitial = {
cacheRemoteFiles: true,
cacheRemoteSensitiveFiles: true,
blockedHosts: [] as string[],
sensitiveWords: [] as string[],
} as Meta;
let meta = metaInitial;
beforeAll(async () => {
const app = await Test.createTestingModule({
imports: [GlobalModule, CoreModule],
}).compile();
})
.overrideProvider(DownloadService).useValue({
async downloadUrl(): Promise<{ filename: string }> {
return {
filename: 'dummy.tmp',
};
},
})
.overrideProvider(MetaService).useValue({
async fetch(): Promise<Meta> {
return meta;
},
}).compile();
await app.init();
app.enableShutdownHooks();
@ -80,6 +115,7 @@ describe('ActivityPub', () => {
noteService = app.get<ApNoteService>(ApNoteService);
personService = app.get<ApPersonService>(ApPersonService);
rendererService = app.get<ApRendererService>(ApRendererService);
imageService = app.get<ApImageService>(ApImageService);
resolver = new MockResolver(await app.resolve<LoggerService>(LoggerService));
// Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error
@ -219,4 +255,91 @@ describe('ActivityPub', () => {
assert.strictEqual(note.uri, actor2Note.id);
});
});
describe('Images', () => {
test('Create images', async () => {
const imageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/foo.png',
name: '',
};
const driveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
imageObject,
);
assert.ok(!driveFile.isLink);
const sensitiveImageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/bar.png',
name: '',
sensitive: true,
};
const sensitiveDriveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
sensitiveImageObject,
);
assert.ok(!sensitiveDriveFile.isLink);
});
test('cacheRemoteFiles=false disables caching', async () => {
meta = { ...metaInitial, cacheRemoteFiles: false };
const imageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/foo.png',
name: '',
};
const driveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
imageObject,
);
assert.ok(driveFile.isLink);
const sensitiveImageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/bar.png',
name: '',
sensitive: true,
};
const sensitiveDriveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
sensitiveImageObject,
);
assert.ok(sensitiveDriveFile.isLink);
});
test('cacheRemoteSensitiveFiles=false only affects sensitive files', async () => {
meta = { ...metaInitial, cacheRemoteSensitiveFiles: false };
const imageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/foo.png',
name: '',
};
const driveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
imageObject,
);
assert.ok(!driveFile.isLink);
const sensitiveImageObject: IApDocument = {
type: 'Document',
mediaType: 'image/png',
url: 'http://host1.test/bar.png',
name: '',
sensitive: true,
};
const sensitiveDriveFile = await imageService.createImage(
await createRandomRemoteUser(resolver, personService),
sensitiveImageObject,
);
assert.ok(sensitiveDriveFile.isLink);
});
});
});

View file

@ -37,6 +37,13 @@
<template #label>{{ i18n.ts.cacheRemoteFiles }}</template>
<template #caption>{{ i18n.ts.cacheRemoteFilesDescription }}</template>
</MkSwitch>
<template v-if="cacheRemoteFiles">
<MkSwitch v-model="cacheRemoteSensitiveFiles">
<template #label>{{ i18n.ts.cacheRemoteSensitiveFiles }}</template>
<template #caption>{{ i18n.ts.cacheRemoteSensitiveFilesDescription }}</template>
</MkSwitch>
</template>
</div>
</FormSection>
@ -104,7 +111,6 @@ import { fetchInstance } from '@/instance';
import { i18n } from '@/i18n';
import { definePageMetadata } from '@/scripts/page-metadata';
import MkButton from '@/components/MkButton.vue';
import MkColorInput from '@/components/MkColorInput.vue';
let name: string | null = $ref(null);
let description: string | null = $ref(null);
@ -112,13 +118,14 @@ let maintainerName: string | null = $ref(null);
let maintainerEmail: string | null = $ref(null);
let pinnedUsers: string = $ref('');
let cacheRemoteFiles: boolean = $ref(false);
let cacheRemoteSensitiveFiles: boolean = $ref(false);
let enableServiceWorker: boolean = $ref(false);
let swPublicKey: any = $ref(null);
let swPrivateKey: any = $ref(null);
let deeplAuthKey: string = $ref('');
let deeplIsPro: boolean = $ref(false);
async function init() {
async function init(): Promise<void> {
const meta = await os.api('admin/meta');
name = meta.name;
description = meta.description;
@ -126,6 +133,7 @@ async function init() {
maintainerEmail = meta.maintainerEmail;
pinnedUsers = meta.pinnedUsers.join('\n');
cacheRemoteFiles = meta.cacheRemoteFiles;
cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles;
enableServiceWorker = meta.enableServiceWorker;
swPublicKey = meta.swPublickey;
swPrivateKey = meta.swPrivateKey;
@ -133,7 +141,7 @@ async function init() {
deeplIsPro = meta.deeplIsPro;
}
function save() {
function save(): void {
os.apiWithDialog('admin/update-meta', {
name,
description,
@ -141,6 +149,7 @@ function save() {
maintainerEmail,
pinnedUsers: pinnedUsers.split('\n'),
cacheRemoteFiles,
cacheRemoteSensitiveFiles,
enableServiceWorker,
swPublicKey,
swPrivateKey,

View file

@ -271,6 +271,7 @@ type DetailedInstanceMetadata = LiteInstanceMetadata & {
pinnedPages: string[];
pinnedClipId: string | null;
cacheRemoteFiles: boolean;
cacheRemoteSensitiveFiles: boolean;
requireSetup: boolean;
proxyAccountName: string | null;
features: Record<string, any>;
@ -327,6 +328,10 @@ export type Endpoints = {
req: TODO;
res: TODO;
};
'admin/meta': {
req: TODO;
res: TODO;
};
'admin/reset-password': {
req: TODO;
res: TODO;
@ -2805,7 +2810,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u
//
// src/api.types.ts:16:32 - (ae-forgotten-export) The symbol "TODO" needs to be exported by the entry point index.d.ts
// src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts
// src/api.types.ts:628:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts
// src/api.types.ts:629:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts
// src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts
// (No @packageDocumentation comment for this package)

View file

@ -20,6 +20,7 @@ export type Endpoints = {
'admin/get-table-stats': { req: TODO; res: TODO; };
'admin/invite': { req: TODO; res: TODO; };
'admin/logs': { req: TODO; res: TODO; };
'admin/meta': { req: TODO; res: TODO; };
'admin/reset-password': { req: TODO; res: TODO; };
'admin/resolve-abuse-user-report': { req: TODO; res: TODO; };
'admin/resync-chart': { req: TODO; res: TODO; };

View file

@ -338,6 +338,7 @@ export type DetailedInstanceMetadata = LiteInstanceMetadata & {
pinnedPages: string[];
pinnedClipId: string | null;
cacheRemoteFiles: boolean;
cacheRemoteSensitiveFiles: boolean;
requireSetup: boolean;
proxyAccountName: string | null;
features: Record<string, any>;