heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' into future-2024-05-31

Browse source
This commit is contained in:
dakkar 2024-06-08 16:45:53 +01:00
commit 5dc8c2827c
32 changed files with 415 additions and 222 deletions
Download patch file Download diff file Expand all files Collapse all files

14
packages/backend/src/config.ts
View file

@ -100,6 +100,12 @@ type Source = {
perChannelMaxNoteCacheCount?: number;
perUserNotificationsMaxCount?: number;
deactivateAntennaThreshold?: number;
import?: {
downloadTimeout: number;
maxFileSize: number;
};
pidFile: string;
};
@ -182,6 +188,12 @@ export type Config = {
perChannelMaxNoteCacheCount: number;
perUserNotificationsMaxCount: number;
deactivateAntennaThreshold: number;
import: {
downloadTimeout: number;
maxFileSize: number;
} | undefined;
pidFile: string;
};
@ -291,6 +303,7 @@ export function loadConfig(): Config {
perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
import: config.import,
pidFile: config.pidFile,
};
}
@ -436,4 +449,5 @@ function applyEnvOverrides(config: Source) {
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
}

8
packages/backend/src/core/DownloadService.ts
View file

@ -35,14 +35,14 @@ export class DownloadService {
}
@bindThis
public async downloadUrl(url: string, path: string): Promise<{
public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{
filename: string;
}> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;
const maxSize = this.config.maxFileSize ?? 262144000;
const timeout = options.timeout ?? 30 * 1000;
const operationTimeout = options.operationTimeout ?? 60 * 1000;
const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000;
const urlObj = new URL(url);
let filename = urlObj.pathname.split('/').pop() ?? 'untitled';

288
packages/backend/src/core/MfmService.ts
View file

@ -464,10 +464,10 @@ export class MfmService {
return new XMLSerializer().serializeToString(body);
}
// the toMastoHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version
// the toMastoApiHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version
@bindThis
public async toMastoHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = [], inline = false, quoteUri: string | null = null) {
public async toMastoApiHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = [], inline = false, quoteUri: string | null = null) {
if (nodes == null) {
return null;
}
@ -485,174 +485,174 @@ export class MfmService {
const handlers: {
[K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any;
} = {
async bold(node) {
const el = doc.createElement('span');
el.textContent = '**';
await appendChildren(node.children, el);
el.textContent += '**';
return el;
},
async bold(node) {
const el = doc.createElement('span');
el.textContent = '**';
await appendChildren(node.children, el);
el.textContent += '**';
return el;
},
async small(node) {
const el = doc.createElement('small');
await appendChildren(node.children, el);
return el;
},
async small(node) {
const el = doc.createElement('small');
await appendChildren(node.children, el);
return el;
},
async strike(node) {
const el = doc.createElement('span');
el.textContent = '~~';
await appendChildren(node.children, el);
el.textContent += '~~';
return el;
},
async strike(node) {
const el = doc.createElement('span');
el.textContent = '~~';
await appendChildren(node.children, el);
el.textContent += '~~';
return el;
},
async italic(node) {
const el = doc.createElement('span');
el.textContent = '*';
await appendChildren(node.children, el);
el.textContent += '*';
return el;
},
async italic(node) {
const el = doc.createElement('span');
el.textContent = '*';
await appendChildren(node.children, el);
el.textContent += '*';
return el;
},
async fn(node) {
const el = doc.createElement('span');
el.textContent = '*';
await appendChildren(node.children, el);
el.textContent += '*';
return el;
},
async fn(node) {
const el = doc.createElement('span');
el.textContent = '*';
await appendChildren(node.children, el);
el.textContent += '*';
return el;
},
blockCode(node) {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
blockCode(node) {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
const nodes = node.props.code
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
const nodes = node.props.code
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
inner.appendChild(x === 'br' ? doc.createElement('br') : x);
}
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
inner.appendChild(x === 'br' ? doc.createElement('br') : x);
}
pre.appendChild(inner);
return pre;
},
pre.appendChild(inner);
return pre;
},
async center(node) {
const el = doc.createElement('div');
await appendChildren(node.children, el);
return el;
},
async center(node) {
const el = doc.createElement('div');
await appendChildren(node.children, el);
return el;
},
emojiCode(node) {
return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
},
emojiCode(node) {
return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
},
unicodeEmoji(node) {
return doc.createTextNode(node.props.emoji);
},
unicodeEmoji(node) {
return doc.createTextNode(node.props.emoji);
},
hashtag: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
a.textContent = `#${node.props.hashtag}`;
a.setAttribute('rel', 'tag');
a.setAttribute('class', 'hashtag');
return a;
},
hashtag: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
a.textContent = `#${node.props.hashtag}`;
a.setAttribute('rel', 'tag');
a.setAttribute('class', 'hashtag');
return a;
},
inlineCode(node) {
const el = doc.createElement('code');
el.textContent = node.props.code;
return el;
},
inlineCode(node) {
const el = doc.createElement('code');
el.textContent = node.props.code;
return el;
},
mathInline(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
return el;
},
mathInline(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
return el;
},
mathBlock(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
return el;
},
mathBlock(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
return el;
},
async link(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
await appendChildren(node.children, a);
return a;
},
async link(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
await appendChildren(node.children, a);
return a;
},
async mention(node) {
const { username, host, acct } = node.props;
const resolved = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
async mention(node) {
const { username, host, acct } = node.props;
const resolved = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
const el = doc.createElement('span');
if (!resolved) {
el.textContent = acct;
} else {
el.setAttribute('class', 'h-card');
el.setAttribute('translate', 'no');
const a = doc.createElement('a');
a.setAttribute('href', resolved.url ? resolved.url : resolved.uri);
a.className = 'u-url mention';
const span = doc.createElement('span');
span.textContent = resolved.username || username;
a.textContent = '@';
a.appendChild(span);
el.appendChild(a);
}
const el = doc.createElement('span');
if (!resolved) {
el.textContent = acct;
} else {
el.setAttribute('class', 'h-card');
el.setAttribute('translate', 'no');
const a = doc.createElement('a');
a.setAttribute('href', resolved.url ? resolved.url : resolved.uri);
a.className = 'u-url mention';
const span = doc.createElement('span');
span.textContent = resolved.username || username;
a.textContent = '@';
a.appendChild(span);
el.appendChild(a);
}
return el;
},
return el;
},
async quote(node) {
const el = doc.createElement('blockquote');
await appendChildren(node.children, el);
return el;
},
async quote(node) {
const el = doc.createElement('blockquote');
await appendChildren(node.children, el);
return el;
},
text(node) {
const el = doc.createElement('span');
const nodes = node.props.text
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
text(node) {
const el = doc.createElement('span');
const nodes = node.props.text
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x);
}
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x);
}
return el;
},
return el;
},
url(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
a.textContent = node.props.url.replace(/^https?:\/\//, '');
return a;
},
url(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
a.textContent = node.props.url.replace(/^https?:\/\//, '');
return a;
},
search: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `https"google.com/${node.props.query}`);
a.textContent = node.props.content;
return a;
},
search: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
a.textContent = node.props.content;
return a;
},
async plain(node) {
const el = doc.createElement('span');
await appendChildren(node.children, el);
return el;
},
};
async plain(node) {
const el = doc.createElement('span');
await appendChildren(node.children, el);
return el;
},
};
await appendChildren(nodes, doc.body);

8
packages/backend/src/core/NoteCreateService.ts
View file

@ -627,6 +627,14 @@ export class NoteCreateService implements OnApplicationShutdown {
userHost: user.host,
});
// should really not happen, but better safe than sorry
if (data.reply?.id === insert.id) {
throw new Error("A note can't reply to itself");
}
if (data.renote?.id === insert.id) {
throw new Error("A note can't renote itself");
}
if (data.uri != null) insert.uri = data.uri;
if (data.url != null) insert.url = data.url;

4
packages/backend/src/core/NoteEditService.ts
View file

@ -299,6 +299,10 @@ export class NoteEditService implements OnApplicationShutdown {
}
if (this.isRenote(data)) {
if (data.renote.id === oldnote.id) {
throw new Error("A note can't renote itself");
}
switch (data.renote.visibility) {
case 'public':
// public noteは無条件にrenote可能

4
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -248,7 +248,7 @@ export class ApNoteService {
> => {
if (typeof uri !== 'string' || !/^https?:/.test(uri)) return { status: 'permerror' };
try {
const res = await this.resolveNote(uri);
const res = await this.resolveNote(uri, { resolver });
if (res == null) return { status: 'permerror' };
return { status: 'ok', res };
} catch (e) {
@ -473,7 +473,7 @@ export class ApNoteService {
> => {
if (!/^https?:/.test(uri)) return { status: 'permerror' };
try {
const res = await this.resolveNote(uri);
const res = await this.resolveNote(uri, { resolver });
if (res == null) return { status: 'permerror' };
return { status: 'ok', res };
} catch (e) {

2
packages/backend/src/misc/sql-like-escape.ts
View file

@ -4,5 +4,5 @@
*/
export function sqlLikeEscape(s: string) {
return s.replace(/([%_])/g, '\\$1');
return s.replace(/([%_\\])/g, '\\$1');
}

21
packages/backend/src/queue/processors/ImportNotesProcessorService.ts
View file

@ -19,12 +19,16 @@ import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
import type { Config } from '@/config.js';
@Injectable()
export class ImportNotesProcessorService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@ -73,6 +77,11 @@ export class ImportNotesProcessorService {
}
}
@bindThis
private downloadUrl(url: string, path:string): Promise<{filename: string}> {
return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
}
@bindThis
private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
type NotesMap = {
@ -176,7 +185,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -206,7 +215,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -239,7 +248,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -297,7 +306,7 @@ export class ImportNotesProcessorService {
try {
await fsp.writeFile(path, '', 'utf-8');
await this.downloadService.downloadUrl(file.url, path);
await this.downloadUrl(file.url, path);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
@ -349,7 +358,7 @@ export class ImportNotesProcessorService {
if (!exists) {
try {
await this.downloadService.downloadUrl(file.url, filePath);
await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string));
}
@ -488,7 +497,7 @@ export class ImportNotesProcessorService {
if (!exists) {
try {
await this.downloadService.downloadUrl(file.url, filePath);
await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string));
}

7
packages/backend/src/server/api/endpoints/admin/emoji/list.ts
View file

@ -66,6 +66,7 @@ export const paramDef = {
properties: {
query: { type: 'string', nullable: true, default: null },
limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
offset: { type: 'integer', minimum: 1, nullable: true, default: null },
sinceId: { type: 'string', format: 'misskey:id' },
untilId: { type: 'string', format: 'misskey:id' },
},
@ -91,7 +92,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//q.andWhere('emoji.name ILIKE :q', { q: `%${ sqlLikeEscape(ps.query) }%` });
//const emojis = await q.limit(ps.limit).getMany();
emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
emojis = await q.orderBy('length(emoji.name)', 'ASC').addOrderBy('id', 'DESC').getMany();
const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
if (queryarry) {
@ -105,9 +106,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
emoji.aliases.some(a => a.includes(queryNfc)) ||
emoji.category?.includes(queryNfc));
}
emojis.splice(ps.limit + 1);
emojis = emojis.slice((ps.offset ?? 0), ((ps.offset ?? 0) + ps.limit));
} else {
emojis = await q.limit(ps.limit).getMany();
emojis = await q.take(ps.limit).skip(ps.offset ?? 0).getMany();
}
return this.emojiEntityService.packDetailedMany(emojis);

4
packages/backend/src/server/api/endpoints/notes/create.ts
View file

@ -30,8 +30,8 @@ export const meta = {
prohibitMoved: true,
limit: {
duration: ms('1hour'),
max: 300,
duration: ms('1minute'),
max: 5,
},
kind: 'write:notes',

6
packages/backend/src/server/api/mastodon/converters.ts
View file

@ -110,7 +110,7 @@ export class MastoConverters {
private async encodeField(f: Entity.Field): Promise<Entity.Field> {
return {
name: f.name,
value: await this.mfmService.toMastoHtml(mfm.parse(f.value), [], true) ?? escapeMFM(f.value),
value: await this.mfmService.toMastoApiHtml(mfm.parse(f.value), [], true) ?? escapeMFM(f.value),
verified_at: null,
};
}
@ -179,7 +179,7 @@ export class MastoConverters {
const files = this.driveFileEntityService.packManyByIds(edit.fileIds);
const item = {
account: noteUser,
content: this.mfmService.toMastoHtml(mfm.parse(edit.newText ?? ''), JSON.parse(note.mentionedRemoteUsers)).then(p => p ?? ''),
content: this.mfmService.toMastoApiHtml(mfm.parse(edit.newText ?? ''), JSON.parse(note.mentionedRemoteUsers)).then(p => p ?? ''),
created_at: lastDate.toISOString(),
emojis: [],
sensitive: files.then(files => files.length > 0 ? files.some((f) => f.isSensitive) : false),
@ -240,7 +240,7 @@ export class MastoConverters {
});
const content = note.text !== null
? quoteUri.then(quoteUri => this.mfmService.toMastoHtml(mfm.parse(note.text!), JSON.parse(note.mentionedRemoteUsers), false, quoteUri))
? quoteUri.then(quoteUri => this.mfmService.toMastoApiHtml(mfm.parse(note.text!), JSON.parse(note.mentionedRemoteUsers), false, quoteUri))
.then(p => p ?? escapeMFM(note.text!))
: '';