Merge remote-tracking branch 'misskey/develop' into future-2024-03-23

This commit is contained in:
dakkar 2024-03-24 11:53:52 +00:00
commit bc531ac414
70 changed files with 1770 additions and 838 deletions

View file

@ -0,0 +1,42 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class UrlPreviewMeta1710512074000 {
name = 'UrlPreviewMeta1710512074000'
async up(queryRunner) {
await queryRunner.query(`
alter table meta
rename column "summalyProxy" to "urlPreviewSummaryProxyUrl";
alter table meta
add "urlPreviewEnabled" boolean default true not null;
alter table meta
add "urlPreviewTimeout" integer default 10000 not null;
alter table meta
add "urlPreviewMaximumContentLength" bigint default 10485760 not null;
alter table meta
add "urlPreviewRequireContentLength" boolean default false not null;
alter table meta
add "urlPreviewUserAgent" varchar(1024) default null;
`);
}
async down(queryRunner) {
await queryRunner.query(`
alter table meta
rename column "urlPreviewSummaryProxyUrl" to "summalyProxy";
alter table meta
drop column "urlPreviewEnabled";
alter table meta
drop column "urlPreviewTimeout";
alter table meta
drop column "urlPreviewMaximumContentLength";
alter table meta
drop column "urlPreviewRequireContentLength";
alter table meta
drop column "urlPreviewUserAgent";
`);
}
}

View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AntennaExcludeBots1710919614510 {
name = 'AntennaExcludeBots1710919614510'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" ADD "excludeBots" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "excludeBots"`);
}
}

View file

@ -78,7 +78,7 @@
"@fastify/static": "6.12.0",
"@fastify/view": "8.2.0",
"@misskey-dev/sharp-read-bmp": "1.2.0",
"@misskey-dev/summaly": "5.0.3",
"@misskey-dev/summaly": "5.1.0",
"@nestjs/common": "10.3.3",
"@nestjs/core": "10.3.3",
"@nestjs/testing": "10.3.3",

View file

@ -92,7 +92,7 @@ export class AntennaService implements OnApplicationShutdown {
}
@bindThis
public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise<void> {
public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<void> {
const antennas = await this.getAntennas();
const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const)));
const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna);
@ -110,10 +110,12 @@ export class AntennaService implements OnApplicationShutdown {
// NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている
@bindThis
public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise<boolean> {
public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<boolean> {
if (note.visibility === 'specified') return false;
if (note.visibility === 'followers') return false;
if (antenna.excludeBots && noteUser.isBot) return false;
if (antenna.localOnly && noteUser.host != null) return false;
if (!antenna.withReplies && note.replyId != null) return false;

View file

@ -459,13 +459,15 @@ export default abstract class Chart<T extends Schema> {
}
}
// bake unique count
// bake cardinality
for (const [k, v] of Object.entries(finalDiffs)) {
if (this.schema[k].uniqueIncrement) {
const name = COLUMN_PREFIX + k.replaceAll('.', COLUMN_DELIMITER) as keyof Columns<T>;
const tempColumnName = UNIQUE_TEMP_COLUMN_PREFIX + k.replaceAll('.', COLUMN_DELIMITER) as keyof TempColumnsForUnique<T>;
queryForHour[name] = new Set([...(v as string[]), ...(logHour[tempColumnName] as unknown as string[])]).size;
queryForDay[name] = new Set([...(v as string[]), ...(logDay[tempColumnName] as unknown as string[])]).size;
const cardinalityOfHour = new Set([...(v as string[]), ...(logHour[tempColumnName] as unknown as string[])]).size;
const cardinalityOfDay = new Set([...(v as string[]), ...(logDay[tempColumnName] as unknown as string[])]).size;
queryForHour[name] = cardinalityOfHour;
queryForDay[name] = cardinalityOfDay;
}
}
@ -637,7 +639,7 @@ export default abstract class Chart<T extends Schema> {
// 要求された範囲にログがひとつもなかったら
if (logs.length === 0) {
// もっとも新しいログを持ってくる
// (すくなくともひとつログが無いと隙間埋めできないため)
// (すくなくともひとつログが無いと補間できないため)
const recentLog = await repository.findOne({
where: group ? {
group: group,
@ -654,7 +656,7 @@ export default abstract class Chart<T extends Schema> {
// 要求された範囲の最も古い箇所に位置するログが存在しなかったら
} else if (!isTimeSame(new Date(logs.at(-1)!.date * 1000), gt)) {
// 要求された範囲の最も古い箇所時点での最も新しいログを持ってきて末尾に追加する
// (隙間埋めできないため)
// (補間できないため)
const outdatedLog = await repository.findOne({
where: {
date: LessThan(Chart.dateToTimestamp(gt)),
@ -683,7 +685,7 @@ export default abstract class Chart<T extends Schema> {
if (log) {
chart.unshift(this.convertRawRecord(log));
} else {
// 隙間埋め
// 補間
const latest = logs.find(l => isTimeBefore(new Date(l.date * 1000), current));
const data = latest ? this.convertRawRecord(latest) : null;
chart.unshift(this.getNewLog(data));

View file

@ -39,6 +39,7 @@ export class AntennaEntityService {
caseSensitive: antenna.caseSensitive,
localOnly: antenna.localOnly,
notify: antenna.notify,
excludeBots: antenna.excludeBots,
withReplies: antenna.withReplies,
withFile: antenna.withFile,
isActive: antenna.isActive,

View file

@ -114,6 +114,7 @@ export class MetaEntityService {
policies: { ...DEFAULT_POLICIES, ...instance.policies },
mediaProxy: this.config.mediaProxy,
enableUrlPreview: instance.urlPreviewEnabled,
};
return packed;

View file

@ -72,6 +72,11 @@ export class MiAntenna {
})
public caseSensitive: boolean;
@Column('boolean', {
default: false,
})
public excludeBots: boolean;
@Column('boolean', {
default: false,
})

View file

@ -287,12 +287,6 @@ export class MiMeta {
})
public enableBotTrending: boolean;
@Column('varchar', {
length: 1024,
nullable: true,
})
public summalyProxy: string | null;
@Column('boolean', {
default: false,
})
@ -631,4 +625,36 @@ export class MiMeta {
length: 256, array: true, default: '{}',
})
public bubbleInstances: string[];
@Column('boolean', {
default: true,
})
public urlPreviewEnabled: boolean;
@Column('integer', {
default: 10000,
})
public urlPreviewTimeout: number;
@Column('bigint', {
default: 1024 * 1024 * 10,
})
public urlPreviewMaximumContentLength: number;
@Column('boolean', {
default: true,
})
public urlPreviewRequireContentLength: boolean;
@Column('varchar', {
length: 1024,
nullable: true,
})
public urlPreviewSummaryProxyUrl: string | null;
@Column('varchar', {
length: 1024,
nullable: true,
})
public urlPreviewUserAgent: string | null;
}

View file

@ -76,6 +76,11 @@ export const packedAntennaSchema = {
type: 'boolean',
optional: false, nullable: false,
},
excludeBots: {
type: 'boolean',
optional: false, nullable: false,
default: false,
},
withReplies: {
type: 'boolean',
optional: false, nullable: false,

View file

@ -223,6 +223,10 @@ export const packedMetaLiteSchema = {
type: 'string',
optional: false, nullable: false,
},
enableUrlPreview: {
type: 'boolean',
optional: false, nullable: false,
},
backgroundImageUrl: {
type: 'string',
optional: false, nullable: true,

View file

@ -81,6 +81,7 @@ export class ExportAntennasProcessorService {
}) : null,
caseSensitive: antenna.caseSensitive,
localOnly: antenna.localOnly,
excludeBots: antenna.excludeBots,
withReplies: antenna.withReplies,
withFile: antenna.withFile,
notify: antenna.notify,

View file

@ -44,6 +44,7 @@ const validate = new Ajv().compile({
} },
caseSensitive: { type: 'boolean' },
localOnly: { type: 'boolean' },
excludeBots: { type: 'boolean' },
withReplies: { type: 'boolean' },
withFile: { type: 'boolean' },
notify: { type: 'boolean' },
@ -88,6 +89,7 @@ export class ImportAntennasProcessorService {
users: (antenna.src === 'list' && antenna.userListAccts !== null ? antenna.userListAccts : antenna.users).filter(Boolean),
caseSensitive: antenna.caseSensitive,
localOnly: antenna.localOnly,
excludeBots: antenna.excludeBots,
withReplies: antenna.withReplies,
withFile: antenna.withFile,
notify: antenna.notify,

View file

@ -465,6 +465,8 @@ export const meta = {
summalyProxy: {
type: 'string',
optional: false, nullable: true,
deprecated: true,
description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
},
themeColor: {
type: 'string',
@ -482,6 +484,30 @@ export const meta = {
type: 'string',
optional: false, nullable: false,
},
urlPreviewEnabled: {
type: 'boolean',
optional: false, nullable: false,
},
urlPreviewTimeout: {
type: 'number',
optional: false, nullable: false,
},
urlPreviewMaximumContentLength: {
type: 'number',
optional: false, nullable: false,
},
urlPreviewRequireContentLength: {
type: 'boolean',
optional: false, nullable: false,
},
urlPreviewUserAgent: {
type: 'string',
optional: false, nullable: true,
},
urlPreviewSummaryProxyUrl: {
type: 'string',
optional: false, nullable: true,
},
},
},
} as const;
@ -569,7 +595,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
enableBotTrending: instance.enableBotTrending,
proxyAccountId: instance.proxyAccountId,
summalyProxy: instance.summalyProxy,
email: instance.email,
smtpSecure: instance.smtpSecure,
smtpHost: instance.smtpHost,
@ -616,6 +641,13 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax,
perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax,
notesPerOneAd: instance.notesPerOneAd,
summalyProxy: instance.urlPreviewSummaryProxyUrl,
urlPreviewEnabled: instance.urlPreviewEnabled,
urlPreviewTimeout: instance.urlPreviewTimeout,
urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength,
urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength,
urlPreviewUserAgent: instance.urlPreviewUserAgent,
urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl,
};
});
}

View file

@ -93,7 +93,6 @@ export const paramDef = {
type: 'string',
},
},
summalyProxy: { type: 'string', nullable: true },
deeplAuthKey: { type: 'string', nullable: true },
deeplIsPro: { type: 'boolean' },
deeplFreeMode: { type: 'boolean' },
@ -158,6 +157,16 @@ export const paramDef = {
type: 'string',
},
},
summalyProxy: {
type: 'string', nullable: true,
description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.',
},
urlPreviewEnabled: { type: 'boolean' },
urlPreviewTimeout: { type: 'integer' },
urlPreviewMaximumContentLength: { type: 'integer' },
urlPreviewRequireContentLength: { type: 'boolean' },
urlPreviewUserAgent: { type: 'string', nullable: true },
urlPreviewSummaryProxyUrl: { type: 'string', nullable: true },
},
required: [],
} as const;
@ -357,10 +366,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.langs = ps.langs.filter(Boolean);
}
if (ps.summalyProxy !== undefined) {
set.summalyProxy = ps.summalyProxy;
}
if (ps.enableEmail !== undefined) {
set.enableEmail = ps.enableEmail;
}
@ -609,6 +614,32 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.bannedEmailDomains = ps.bannedEmailDomains;
}
if (ps.urlPreviewEnabled !== undefined) {
set.urlPreviewEnabled = ps.urlPreviewEnabled;
}
if (ps.urlPreviewTimeout !== undefined) {
set.urlPreviewTimeout = ps.urlPreviewTimeout;
}
if (ps.urlPreviewMaximumContentLength !== undefined) {
set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength;
}
if (ps.urlPreviewRequireContentLength !== undefined) {
set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength;
}
if (ps.urlPreviewUserAgent !== undefined) {
const value = (ps.urlPreviewUserAgent ?? '').trim();
set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent;
}
if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) {
const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim();
set.urlPreviewSummaryProxyUrl = value === '' ? null : value;
}
const before = await this.metaService.fetch(true);
await this.metaService.update(set);

View file

@ -64,6 +64,7 @@ export const paramDef = {
} },
caseSensitive: { type: 'boolean' },
localOnly: { type: 'boolean' },
excludeBots: { type: 'boolean' },
withReplies: { type: 'boolean' },
withFile: { type: 'boolean' },
notify: { type: 'boolean' },
@ -124,6 +125,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
users: ps.users,
caseSensitive: ps.caseSensitive,
localOnly: ps.localOnly,
excludeBots: ps.excludeBots,
withReplies: ps.withReplies,
withFile: ps.withFile,
notify: ps.notify,

View file

@ -63,6 +63,7 @@ export const paramDef = {
} },
caseSensitive: { type: 'boolean' },
localOnly: { type: 'boolean' },
excludeBots: { type: 'boolean' },
withReplies: { type: 'boolean' },
withFile: { type: 'boolean' },
notify: { type: 'boolean' },
@ -120,6 +121,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
users: ps.users,
caseSensitive: ps.caseSensitive,
localOnly: ps.localOnly,
excludeBots: ps.excludeBots,
withReplies: ps.withReplies,
withFile: ps.withFile,
notify: ps.notify,

View file

@ -44,6 +44,7 @@ export const paramDef = {
permissions: { type: 'array', items: {
type: 'string',
} },
visibility: { type: 'string', enum: ['public', 'private'], default: 'public' },
},
required: ['title', 'summary', 'script', 'permissions'],
} as const;
@ -66,6 +67,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
summary: ps.summary,
script: ps.script,
permissions: ps.permissions,
visibility: ps.visibility,
}).then(x => this.flashsRepository.findOneByOrFail(x.identifiers[0]));
return await this.flashEntityService.pack(flash);

View file

@ -93,7 +93,7 @@ export function genOpenapiSpec(config: Config, includeSelfRef = false) {
const hasBody = (schema.type === 'object' && schema.properties && Object.keys(schema.properties).length >= 1);
const info = {
operationId: endpoint.name,
operationId: endpoint.name.replaceAll('/', '___'), // NOTE: スラッシュは使えない
summary: endpoint.name,
description: desc,
externalDocs: {

View file

@ -5,6 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { summaly } from '@misskey-dev/summaly';
import { SummalyResult } from '@misskey-dev/summaly/built/summary.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { MetaService } from '@/core/MetaService.js';
@ -14,6 +15,7 @@ import { query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import { ApiError } from '@/server/api/error.js';
import { MiMeta } from '@/models/Meta.js';
import type { FastifyRequest, FastifyReply } from 'fastify';
@Injectable()
@ -62,24 +64,25 @@ export class UrlPreviewService {
const meta = await this.metaService.fetch();
this.logger.info(meta.summalyProxy
if (!meta.urlPreviewEnabled) {
reply.code(403);
return {
error: new ApiError({
message: 'URL preview is disabled',
code: 'URL_PREVIEW_DISABLED',
id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8',
}),
};
}
this.logger.info(meta.urlPreviewSummaryProxyUrl
? `(Proxy) Getting preview of ${url}@${lang} ...`
: `Getting preview of ${url}@${lang} ...`);
try {
const summary = meta.summalyProxy ?
await this.httpRequestService.getJson<ReturnType<typeof summaly>>(`${meta.summalyProxy}?${query({
url: url,
lang: lang ?? 'ja-JP',
})}`)
:
await summaly(url, {
followRedirects: false,
lang: lang ?? 'ja-JP',
agent: this.config.proxy ? {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
} : undefined,
});
const summary = meta.urlPreviewSummaryProxyUrl
? await this.fetchSummaryFromProxy(url, meta, lang)
: await this.fetchSummary(url, meta, lang);
this.logger.succ(`Got preview of ${url}: ${summary.title}`);
@ -100,6 +103,7 @@ export class UrlPreviewService {
return summary;
} catch (err) {
this.logger.warn(`Failed to get preview of ${url}: ${err}`);
reply.code(422);
reply.header('Cache-Control', 'max-age=86400, immutable');
return {
@ -111,4 +115,37 @@ export class UrlPreviewService {
};
}
}
private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
const agent = this.config.proxy
? {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
}
: undefined;
return summaly(url, {
followRedirects: false,
lang: lang ?? 'ja-JP',
agent: agent,
userAgent: meta.urlPreviewUserAgent ?? undefined,
operationTimeout: meta.urlPreviewTimeout,
contentLengthLimit: meta.urlPreviewMaximumContentLength,
contentLengthRequired: meta.urlPreviewRequireContentLength,
});
}
private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise<SummalyResult> {
const proxy = meta.urlPreviewSummaryProxyUrl!;
const queryStr = query({
url: url,
lang: lang ?? 'ja-JP',
userAgent: meta.urlPreviewUserAgent ?? undefined,
operationTimeout: meta.urlPreviewTimeout,
contentLengthLimit: meta.urlPreviewMaximumContentLength,
contentLengthRequired: meta.urlPreviewRequireContentLength,
});
return this.httpRequestService.getJson<SummalyResult>(`${proxy}?${queryStr}`);
}
}

View file

@ -2,7 +2,7 @@ extends ./base
block vars
- const user = note.user;
- const title = user.name ? `${user.name} (@${user.username})` : `@${user.username}`;
- const title = user.name ? `${user.name} (@${user.username}${user.host ? `@${user.host}` : ''})` : `@${user.username}${user.host ? `@${user.host}` : ''}`;
- const url = `${config.url}/notes/${note.id}`;
- const isRenote = note.renote && note.text == null && note.fileIds.length == 0 && note.poll == null;
- const images = (note.files || []).filter(file => file.type.startsWith('image/') && !file.isSensitive)
@ -28,7 +28,7 @@ block og
// FIXME: add embed player for Twitter
if images.length
meta(property='twitter:card' content='summary_large_image')
each image in images
each image in images
meta(property='og:image' content= image.url)
else
meta(property='twitter:card' content='summary')

View file

@ -1,7 +1,7 @@
extends ./base
block vars
- const title = user.name ? `${user.name} (@${user.username})` : `@${user.username}`;
- const title = user.name ? `${user.name} (@${user.username}${user.host ? `@${user.host}` : ''})` : `@${user.username}${user.host ? `@${user.host}` : ''}`;
- const url = `${config.url}/@${(user.host ? `${user.username}@${user.host}` : user.username)}`;
block title

View file

@ -44,6 +44,7 @@ describe('アンテナ', () => {
users: [''],
withFile: false,
withReplies: false,
excludeBots: false,
};
let root: User;
@ -156,6 +157,7 @@ describe('アンテナ', () => {
users: [''],
withFile: false,
withReplies: false,
excludeBots: false,
localOnly: false,
};
assert.deepStrictEqual(response, expected);

View file

@ -158,19 +158,17 @@ describe('Streaming', () => {
assert.strictEqual(fired, true);
});
/*
test('フォローしているユーザーの visibility: followers な投稿への返信が流れる', async () => {
const note = await api('notes/create', { text: 'foo', visibility: 'followers' }, kyoko);
const note = await post(kyoko, { text: 'foo', visibility: 'followers' });
const fired = await waitFire(
ayano, 'homeTimeline', // ayano:home
() => api('notes/create', { text: 'bar', visibility: 'followers', replyId: note.body.id }, kyoko), // kyoko posts
() => api('notes/create', { text: 'bar', visibility: 'followers', replyId: note.id }, kyoko), // kyoko posts
msg => msg.type === 'note' && msg.body.userId === kyoko.id && msg.body.reply.text === 'foo',
);
assert.strictEqual(fired, true);
});
*/
test('フォローしているユーザーのフォローしていないユーザーの visibility: followers な投稿への返信が流れない', async () => {
const chitoseNote = await post(chitose, { text: 'followers-only post', visibility: 'followers' });