Merge remote-tracking branch 'misskey/develop' into future-2024-03-23

packages/backend/src/core/AntennaService.ts

@@ -92,7 +92,7 @@ export class AntennaService implements OnApplicationShutdown {
 	}
 
 	@bindThis
-	public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise<void> {
+	public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<void> {
 		const antennas = await this.getAntennas();
 		const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const)));
 		const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna);
@@ -110,10 +110,12 @@ export class AntennaService implements OnApplicationShutdown {
 	// NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている
 
 	@bindThis
-	public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise<boolean> {
+	public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<boolean> {
 		if (note.visibility === 'specified') return false;
 		if (note.visibility === 'followers') return false;
 
+		if (antenna.excludeBots && noteUser.isBot) return false;
+
 		if (antenna.localOnly && noteUser.host != null) return false;
 
 		if (!antenna.withReplies && note.replyId != null) return false;

packages/backend/src/core/chart/core.ts

@@ -459,13 +459,15 @@ export default abstract class Chart<T extends Schema> {
 				}
 			}
 
-			// bake unique count
+			// bake cardinality
 			for (const [k, v] of Object.entries(finalDiffs)) {
 				if (this.schema[k].uniqueIncrement) {
 					const name = COLUMN_PREFIX + k.replaceAll('.', COLUMN_DELIMITER) as keyof Columns<T>;
 					const tempColumnName = UNIQUE_TEMP_COLUMN_PREFIX + k.replaceAll('.', COLUMN_DELIMITER) as keyof TempColumnsForUnique<T>;
-					queryForHour[name] = new Set([...(v as string[]), ...(logHour[tempColumnName] as unknown as string[])]).size;
-					queryForDay[name] = new Set([...(v as string[]), ...(logDay[tempColumnName] as unknown as string[])]).size;
+					const cardinalityOfHour = new Set([...(v as string[]), ...(logHour[tempColumnName] as unknown as string[])]).size;
+					const cardinalityOfDay = new Set([...(v as string[]), ...(logDay[tempColumnName] as unknown as string[])]).size;
+					queryForHour[name] = cardinalityOfHour;
+					queryForDay[name] = cardinalityOfDay;
 				}
 			}
 
@@ -637,7 +639,7 @@ export default abstract class Chart<T extends Schema> {
 		// 要求された範囲にログがひとつもなかったら
 		if (logs.length === 0) {
 			// もっとも新しいログを持ってくる
-			// (すくなくともひとつログが無いと隙間埋めできないため)
+			// (すくなくともひとつログが無いと補間できないため)
 			const recentLog = await repository.findOne({
 				where: group ? {
 					group: group,
@@ -654,7 +656,7 @@ export default abstract class Chart<T extends Schema> {
 		// 要求された範囲の最も古い箇所に位置するログが存在しなかったら
 		} else if (!isTimeSame(new Date(logs.at(-1)!.date * 1000), gt)) {
 			// 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する
-			// (隙間埋めできないため)
+			// (補間できないため)
 			const outdatedLog = await repository.findOne({
 				where: {
 					date: LessThan(Chart.dateToTimestamp(gt)),
@@ -683,7 +685,7 @@ export default abstract class Chart<T extends Schema> {
 			if (log) {
 				chart.unshift(this.convertRawRecord(log));
 			} else {
-				// 隙間埋め
+				// 補間
 				const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
 				const data = latest ? this.convertRawRecord(latest) : null;
 				chart.unshift(this.getNewLog(data));

packages/backend/src/core/entities/AntennaEntityService.ts

@@ -39,6 +39,7 @@ export class AntennaEntityService {
 			caseSensitive: antenna.caseSensitive,
 			localOnly: antenna.localOnly,
 			notify: antenna.notify,
+			excludeBots: antenna.excludeBots,
 			withReplies: antenna.withReplies,
 			withFile: antenna.withFile,
 			isActive: antenna.isActive,

packages/backend/src/core/entities/MetaEntityService.ts

@@ -114,6 +114,7 @@ export class MetaEntityService {
 			policies: { ...DEFAULT_POLICIES, ...instance.policies },
 
 			mediaProxy: this.config.mediaProxy,
+			enableUrlPreview: instance.urlPreviewEnabled,
 		};
 
 		return packed;

packages/backend/src/models/Antenna.ts

@@ -72,6 +72,11 @@ export class MiAntenna {
 	})
 	public caseSensitive: boolean;
 
+	@Column('boolean', {
+		default: false,
+	})
+	public excludeBots: boolean;
+
 	@Column('boolean', {
 		default: false,
 	})

packages/backend/src/models/Meta.ts

@@ -287,12 +287,6 @@ export class MiMeta {
 	})
 	public enableBotTrending: boolean;
 
-	@Column('varchar', {
-		length: 1024,
-		nullable: true,
-	})
-	public summalyProxy: string | null;
-
 	@Column('boolean', {
 		default: false,
 	})
@@ -631,4 +625,36 @@ export class MiMeta {
 		length: 256, array: true, default: '{}',
 	})
 	public bubbleInstances: string[];
+
+	@Column('boolean', {
+		default: true,
+	})
+	public urlPreviewEnabled: boolean;
+
+	@Column('integer', {
+		default: 10000,
+	})
+	public urlPreviewTimeout: number;
+
+	@Column('bigint', {
+		default: 1024 * 1024 * 10,
+	})
+	public urlPreviewMaximumContentLength: number;
+
+	@Column('boolean', {
+		default: true,
+	})
+	public urlPreviewRequireContentLength: boolean;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public urlPreviewSummaryProxyUrl: string | null;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public urlPreviewUserAgent: string | null;
 }

packages/backend/src/models/json-schema/antenna.ts

@@ -76,6 +76,11 @@ export const packedAntennaSchema = {
 			type: 'boolean',
 			optional: false, nullable: false,
 		},
+		excludeBots: {
+			type: 'boolean',
+			optional: false, nullable: false,
+			default: false,
+		},
 		withReplies: {
 			type: 'boolean',
 			optional: false, nullable: false,

packages/backend/src/models/json-schema/meta.ts

@@ -223,6 +223,10 @@ export const packedMetaLiteSchema = {
 			type: 'string',
 			optional: false, nullable: false,
 		},
+		enableUrlPreview: {
+			type: 'boolean',
+			optional: false, nullable: false,
+		},
 		backgroundImageUrl: {
 			type: 'string',
 			optional: false, nullable: true,

packages/backend/src/queue/processors/ExportAntennasProcessorService.ts

@@ -81,6 +81,7 @@ export class ExportAntennasProcessorService {
 					}) : null,
 					caseSensitive: antenna.caseSensitive,
 					localOnly: antenna.localOnly,
+					excludeBots: antenna.excludeBots,
 					withReplies: antenna.withReplies,
 					withFile: antenna.withFile,
 					notify: antenna.notify,

packages/backend/src/queue/processors/ImportAntennasProcessorService.ts

@@ -44,6 +44,7 @@ const validate = new Ajv().compile({
 		} },
 		caseSensitive: { type: 'boolean' },
 		localOnly: { type: 'boolean' },
+		excludeBots: { type: 'boolean' },
 		withReplies: { type: 'boolean' },
 		withFile: { type: 'boolean' },
 		notify: { type: 'boolean' },
@@ -88,6 +89,7 @@ export class ImportAntennasProcessorService {
 					users: (antenna.src === 'list' && antenna.userListAccts !== null ? antenna.userListAccts : antenna.users).filter(Boolean),
 					caseSensitive: antenna.caseSensitive,
 					localOnly: antenna.localOnly,
+					excludeBots: antenna.excludeBots,
 					withReplies: antenna.withReplies,
 					withFile: antenna.withFile,
 					notify: antenna.notify,

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -465,6 +465,8 @@ export const meta = {
 			summalyProxy: {
 				type: 'string',
 				optional: false, nullable: true,
+				deprecated: true,
+				description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
 			},
 			themeColor: {
 				type: 'string',
@@ -482,6 +484,30 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: false,
 			},
+			urlPreviewEnabled: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
+			urlPreviewTimeout: {
+				type: 'number',
+				optional: false, nullable: false,
+			},
+			urlPreviewMaximumContentLength: {
+				type: 'number',
+				optional: false, nullable: false,
+			},
+			urlPreviewRequireContentLength: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
+			urlPreviewUserAgent: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
+			urlPreviewSummaryProxyUrl: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 		},
 	},
 } as const;
@@ -569,7 +595,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
 				enableBotTrending: instance.enableBotTrending,
 				proxyAccountId: instance.proxyAccountId,
-				summalyProxy: instance.summalyProxy,
 				email: instance.email,
 				smtpSecure: instance.smtpSecure,
 				smtpHost: instance.smtpHost,
@@ -616,6 +641,13 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax,
 				perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax,
 				notesPerOneAd: instance.notesPerOneAd,
+				summalyProxy: instance.urlPreviewSummaryProxyUrl,
+				urlPreviewEnabled: instance.urlPreviewEnabled,
+				urlPreviewTimeout: instance.urlPreviewTimeout,
+				urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength,
+				urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength,
+				urlPreviewUserAgent: instance.urlPreviewUserAgent,
+				urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl,
 			};
 		});
 	}

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -93,7 +93,6 @@ export const paramDef = {
 				type: 'string',
 			},
 		},
-		summalyProxy: { type: 'string', nullable: true },
 		deeplAuthKey: { type: 'string', nullable: true },
 		deeplIsPro: { type: 'boolean' },
 		deeplFreeMode: { type: 'boolean' },
@@ -158,6 +157,16 @@ export const paramDef = {
 				type: 'string',
 			},
 		},
+		summalyProxy: {
+			type: 'string', nullable: true,
+			description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
+		},
+		urlPreviewEnabled: { type: 'boolean' },
+		urlPreviewTimeout: { type: 'integer' },
+		urlPreviewMaximumContentLength: { type: 'integer' },
+		urlPreviewRequireContentLength: { type: 'boolean' },
+		urlPreviewUserAgent: { type: 'string', nullable: true },
+		urlPreviewSummaryProxyUrl: { type: 'string', nullable: true },
 	},
 	required: [],
 } as const;
@@ -357,10 +366,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.langs = ps.langs.filter(Boolean);
 			}
 
-			if (ps.summalyProxy !== undefined) {
-				set.summalyProxy = ps.summalyProxy;
-			}
-
 			if (ps.enableEmail !== undefined) {
 				set.enableEmail = ps.enableEmail;
 			}
@@ -609,6 +614,32 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.bannedEmailDomains = ps.bannedEmailDomains;
 			}
 
+			if (ps.urlPreviewEnabled !== undefined) {
+				set.urlPreviewEnabled = ps.urlPreviewEnabled;
+			}
+
+			if (ps.urlPreviewTimeout !== undefined) {
+				set.urlPreviewTimeout = ps.urlPreviewTimeout;
+			}
+
+			if (ps.urlPreviewMaximumContentLength !== undefined) {
+				set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength;
+			}
+
+			if (ps.urlPreviewRequireContentLength !== undefined) {
+				set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength;
+			}
+
+			if (ps.urlPreviewUserAgent !== undefined) {
+				const value = (ps.urlPreviewUserAgent ?? '').trim();
+				set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent;
+			}
+
+			if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) {
+				const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim();
+				set.urlPreviewSummaryProxyUrl = value === '' ? null : value;
+			}
+
 			const before = await this.metaService.fetch(true);
 
 			await this.metaService.update(set);

packages/backend/src/server/api/endpoints/antennas/create.ts

@@ -64,6 +64,7 @@ export const paramDef = {
 		} },
 		caseSensitive: { type: 'boolean' },
 		localOnly: { type: 'boolean' },
+		excludeBots: { type: 'boolean' },
 		withReplies: { type: 'boolean' },
 		withFile: { type: 'boolean' },
 		notify: { type: 'boolean' },
@@ -124,6 +125,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				users: ps.users,
 				caseSensitive: ps.caseSensitive,
 				localOnly: ps.localOnly,
+				excludeBots: ps.excludeBots,
 				withReplies: ps.withReplies,
 				withFile: ps.withFile,
 				notify: ps.notify,

packages/backend/src/server/api/endpoints/antennas/update.ts

@@ -63,6 +63,7 @@ export const paramDef = {
 		} },
 		caseSensitive: { type: 'boolean' },
 		localOnly: { type: 'boolean' },
+		excludeBots: { type: 'boolean' },
 		withReplies: { type: 'boolean' },
 		withFile: { type: 'boolean' },
 		notify: { type: 'boolean' },
@@ -120,6 +121,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				users: ps.users,
 				caseSensitive: ps.caseSensitive,
 				localOnly: ps.localOnly,
+				excludeBots: ps.excludeBots,
 				withReplies: ps.withReplies,
 				withFile: ps.withFile,
 				notify: ps.notify,

packages/backend/src/server/api/endpoints/flash/create.ts

@@ -44,6 +44,7 @@ export const paramDef = {
 		permissions: { type: 'array', items: {
 			type: 'string',
 		} },
+		visibility: { type: 'string', enum: ['public', 'private'], default: 'public' },
 	},
 	required: ['title', 'summary', 'script', 'permissions'],
 } as const;
@@ -66,6 +67,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				summary: ps.summary,
 				script: ps.script,
 				permissions: ps.permissions,
+				visibility: ps.visibility,
 			}).then(x => this.flashsRepository.findOneByOrFail(x.identifiers[0]));
 
 			return await this.flashEntityService.pack(flash);

packages/backend/src/server/api/openapi/gen-spec.ts

@@ -93,7 +93,7 @@ export function genOpenapiSpec(config: Config, includeSelfRef = false) {
 		const hasBody = (schema.type === 'object' && schema.properties && Object.keys(schema.properties).length >= 1);
 
 		const info = {
-			operationId: endpoint.name,
+			operationId: endpoint.name.replaceAll('/', '___'), // NOTE: スラッシュは使えない
 			summary: endpoint.name,
 			description: desc,
 			externalDocs: {

packages/backend/src/server/web/UrlPreviewService.ts

@@ -5,6 +5,7 @@
 
 import { Inject, Injectable } from '@nestjs/common';
 import { summaly } from '@misskey-dev/summaly';
+import { SummalyResult } from '@misskey-dev/summaly/built/summary.js';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { MetaService } from '@/core/MetaService.js';
@@ -14,6 +15,7 @@ import { query } from '@/misc/prelude/url.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import { bindThis } from '@/decorators.js';
 import { ApiError } from '@/server/api/error.js';
+import { MiMeta } from '@/models/Meta.js';
 import type { FastifyRequest, FastifyReply } from 'fastify';
 
 @Injectable()
@@ -62,24 +64,25 @@ export class UrlPreviewService {
 
 		const meta = await this.metaService.fetch();
 
-		this.logger.info(meta.summalyProxy
+		if (!meta.urlPreviewEnabled) {
+			reply.code(403);
+			return {
+				error: new ApiError({
+					message: 'URL preview is disabled',
+					code: 'URL_PREVIEW_DISABLED',
+					id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8',
+				}),
+			};
+		}
+
+		this.logger.info(meta.urlPreviewSummaryProxyUrl
 			? `(Proxy) Getting preview of ${url}@${lang} ...`
 			: `Getting preview of ${url}@${lang} ...`);
+
 		try {
-			const summary = meta.summalyProxy ?
-				await this.httpRequestService.getJson<ReturnType<typeof summaly>>(`${meta.summalyProxy}?${query({
-					url: url,
-					lang: lang ?? 'ja-JP',
-				})}`)
-				:
-				await summaly(url, {
-					followRedirects: false,
-					lang: lang ?? 'ja-JP',
-					agent: this.config.proxy ? {
-						http: this.httpRequestService.httpAgent,
-						https: this.httpRequestService.httpsAgent,
-					} : undefined,
-				});
+			const summary = meta.urlPreviewSummaryProxyUrl
+				? await this.fetchSummaryFromProxy(url, meta, lang)
+				: await this.fetchSummary(url, meta, lang);
 
 			this.logger.succ(`Got preview of ${url}: ${summary.title}`);
 
@@ -100,6 +103,7 @@ export class UrlPreviewService {
 			return summary;
 		} catch (err) {
 			this.logger.warn(`Failed to get preview of ${url}: ${err}`);
+
 			reply.code(422);
 			reply.header('Cache-Control', 'max-age=86400, immutable');
 			return {
@@ -111,4 +115,37 @@ export class UrlPreviewService {
 			};
 		}
 	}
+
+	private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
+		const agent = this.config.proxy
+			? {
+				http: this.httpRequestService.httpAgent,
+				https: this.httpRequestService.httpsAgent,
+			}
+			: undefined;
+
+		return summaly(url, {
+			followRedirects: false,
+			lang: lang ?? 'ja-JP',
+			agent: agent,
+			userAgent: meta.urlPreviewUserAgent ?? undefined,
+			operationTimeout: meta.urlPreviewTimeout,
+			contentLengthLimit: meta.urlPreviewMaximumContentLength,
+			contentLengthRequired: meta.urlPreviewRequireContentLength,
+		});
+	}
+
+	private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
+		const proxy = meta.urlPreviewSummaryProxyUrl!;
+		const queryStr = query({
+			url: url,
+			lang: lang ?? 'ja-JP',
+			userAgent: meta.urlPreviewUserAgent ?? undefined,
+			operationTimeout: meta.urlPreviewTimeout,
+			contentLengthLimit: meta.urlPreviewMaximumContentLength,
+			contentLengthRequired: meta.urlPreviewRequireContentLength,
+		});
+
+		return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`);
+	}
 }

packages/backend/src/server/web/views/note.pug

@@ -2,7 +2,7 @@ extends ./base
 
 block vars
 	- const user = note.user;
-	- const title = user.name ? `${user.name} (@${user.username})` : `@${user.username}`;
+	- const title = user.name ? `${user.name} (@${user.username}${user.host ? `@${user.host}` : ''})` : `@${user.username}${user.host ? `@${user.host}` : ''}`;
 	- const url = `${config.url}/notes/${note.id}`;
 	- const isRenote = note.renote && note.text == null && note.fileIds.length == 0 && note.poll == null;
 	- const images = (note.files || []).filter(file => file.type.startsWith('image/') && !file.isSensitive)
@@ -28,7 +28,7 @@ block og
 			// FIXME: add embed player for Twitter
 	if images.length
 		meta(property='twitter:card' content='summary_large_image')
-		each image in images	
+		each image in images
 			meta(property='og:image'     content= image.url)
 	else
 		meta(property='twitter:card' content='summary')

packages/backend/src/server/web/views/user.pug

@@ -1,7 +1,7 @@
 extends ./base
 
 block vars
-	- const title = user.name ? `${user.name} (@${user.username})` : `@${user.username}`;
+	- const title = user.name ? `${user.name} (@${user.username}${user.host ? `@${user.host}` : ''})` : `@${user.username}${user.host ? `@${user.host}` : ''}`;
 	- const url = `${config.url}/@${(user.host ? `${user.username}@${user.host}` : user.username)}`;
 
 block title