heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' into future-2024-04-25-post

Browse source
This commit is contained in:
dakkar 2024-05-11 13:11:07 +01:00
commit 30bd7768d6
70 changed files with 305 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/package.json
View file

@ -88,7 +88,7 @@
"@smithy/node-http-handler": "2.1.10",
"@swc/cli": "0.1.63",
"@swc/core": "1.3.107",
"@transfem-org/sfm-js": "0.24.4",
"@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.0.0",
"accepts": "1.3.8",
"ajv": "8.12.0",

7
packages/backend/src/core/NoteEditService.ts
View file

@ -430,11 +430,16 @@ export class NoteEditService implements OnApplicationShutdown {
update.hasPoll = !!data.poll;
}
// technically we should check if the two sets of files are
// different, or if their descriptions have changed. In practice
// this is good enough.
const filesChanged = oldnote.fileIds?.length || data.files?.length;
const poll = await this.pollsRepository.findOneBy({ noteId: oldnote.id });
const oldPoll = poll ? { choices: poll.choices, multiple: poll.multiple, expiresAt: poll.expiresAt } : null;
if (Object.keys(update).length > 0) {
if (Object.keys(update).length > 0 || filesChanged) {
const exists = await this.noteEditRepository.findOneBy({ noteId: oldnote.id });
await this.noteEditRepository.insert({

4
packages/backend/src/core/ReactionService.ts
View file

@ -64,8 +64,8 @@ type DecodedReaction = {
host?: string | null;
};
const isCustomEmojiRegexp = /^:([\w+-]+)(?:@\.)?:$/;
const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/u;
const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/u;
@Injectable()
export class ReactionService {

47
packages/backend/src/core/activitypub/ApRendererService.ts
View file

@ -31,6 +31,7 @@ import { IdService } from '@/core/IdService.js';
import { MetaService } from '../MetaService.js';
import { LdSignatureService } from './LdSignatureService.js';
import { ApMfmService } from './ApMfmService.js';
import { CONTEXT } from './misc/contexts.js';
import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js';
@Injectable()
@ -283,9 +284,10 @@ export class ApRendererService {
if (instance && instance.softwareName === 'mastodon') isMastodon = true;
if (instance && instance.softwareName === 'akkoma') isMastodon = true;
if (instance && instance.softwareName === 'pleroma') isMastodon = true;
if (instance && instance.softwareName === 'iceshrimp.net') isMastodon = true;
}
}
const object: ILike = {
type: 'Like',
id: `${this.config.url}/likes/${noteReaction.id}`,
@ -785,48 +787,7 @@ export class ApRendererService {
x.id = `${this.config.url}/${randomUUID()}`;
}
return Object.assign({
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{
Key: 'sec:Key',
// as non-standards
manuallyApprovesFollowers: 'as:manuallyApprovesFollowers',
sensitive: 'as:sensitive',
Hashtag: 'as:Hashtag',
quoteUrl: 'as:quoteUrl',
fedibird: 'http://fedibird.com/ns#',
quoteUri: 'fedibird:quoteUri',
// Mastodon
toot: 'http://joinmastodon.org/ns#',
Emoji: 'toot:Emoji',
featured: 'toot:featured',
discoverable: 'toot:discoverable',
// schema
schema: 'http://schema.org#',
PropertyValue: 'schema:PropertyValue',
value: 'schema:value',
// Misskey
misskey: 'https://misskey-hub.net/ns#',
'_misskey_content': 'misskey:_misskey_content',
'_misskey_quote': 'misskey:_misskey_quote',
'_misskey_reaction': 'misskey:_misskey_reaction',
'_misskey_votes': 'misskey:_misskey_votes',
'_misskey_summary': 'misskey:_misskey_summary',
'isCat': 'misskey:isCat',
// Firefish
firefish: 'https://joinfirefish.org/ns#',
speakAsCat: 'firefish:speakAsCat',
// Sharkey
sharkey: 'https://joinsharkey.org/ns#',
backgroundUrl: 'sharkey:backgroundUrl',
listenbrainz: 'sharkey:listenbrainz',
// vcard
vcard: 'http://www.w3.org/2006/vcard/ns#',
},
],
}, x as T & { id: string });
return Object.assign({ '@context': CONTEXT }, x as T & { id: string });
}
@bindThis

12
packages/backend/src/core/activitypub/LdSignatureService.ts
View file

@ -7,7 +7,7 @@ import * as crypto from 'node:crypto';
import { Injectable } from '@nestjs/common';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { CONTEXTS } from './misc/contexts.js';
import { CONTEXT, CONTEXTS } from './misc/contexts.js';
import { validateContentTypeSetAsJsonLD } from './misc/validator.js';
import type { JsonLdDocument } from 'jsonld';
import type { JsonLd, RemoteDocument } from 'jsonld/jsonld-spec.js';
@ -88,6 +88,16 @@ class LdSignature {
return verifyData;
}
@bindThis
public async compact(data: any, context: any = CONTEXT): Promise<JsonLdDocument> {
const customLoader = this.getLoader();
// XXX: Importing jsonld dynamically since Jest frequently fails to import it statically
// https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595
return (await import('jsonld')).default.compact(data, context, {
documentLoader: customLoader,
});
}
@bindThis
public async normalize(data: JsonLdDocument): Promise<string> {
const customLoader = this.getLoader();

46
packages/backend/src/core/activitypub/misc/contexts.ts
View file

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import type { JsonLd } from 'jsonld/jsonld-spec.js';
import type { Context, JsonLd } from 'jsonld/jsonld-spec.js';
/* eslint:disable:quotemark indent */
const id_v1 = {
@ -526,6 +526,50 @@ const activitystreams = {
},
} satisfies JsonLd;
const context_iris = [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
];
const extension_context_definition = {
Key: 'sec:Key',
// as non-standards
manuallyApprovesFollowers: 'as:manuallyApprovesFollowers',
sensitive: 'as:sensitive',
Hashtag: 'as:Hashtag',
quoteUrl: 'as:quoteUrl',
fedibird: 'http://fedibird.com/ns#',
quoteUri: 'fedibird:quoteUri',
// Mastodon
toot: 'http://joinmastodon.org/ns#',
Emoji: 'toot:Emoji',
featured: 'toot:featured',
discoverable: 'toot:discoverable',
// schema
schema: 'http://schema.org#',
PropertyValue: 'schema:PropertyValue',
value: 'schema:value',
// Misskey
misskey: 'https://misskey-hub.net/ns#',
'_misskey_content': 'misskey:_misskey_content',
'_misskey_quote': 'misskey:_misskey_quote',
'_misskey_reaction': 'misskey:_misskey_reaction',
'_misskey_votes': 'misskey:_misskey_votes',
'_misskey_summary': 'misskey:_misskey_summary',
'isCat': 'misskey:isCat',
// Firefish
firefish: 'https://joinfirefish.org/ns#',
speakAsCat: 'firefish:speakAsCat',
// Sharkey
sharkey: 'https://joinsharkey.org/ns#',
backgroundUrl: 'sharkey:backgroundUrl',
listenbrainz: 'sharkey:listenbrainz',
// vcard
vcard: 'http://www.w3.org/2006/vcard/ns#',
} satisfies Context;
export const CONTEXT: (string | Context)[] = [...context_iris, extension_context_definition];
export const CONTEXTS: Record<string, JsonLd> = {
'https://w3id.org/identity/v1': id_v1,
'https://w3id.org/security/v1': security_v1,

7
packages/backend/src/core/chart/charts/users.ts
View file

@ -4,7 +4,7 @@
*/
import { Injectable, Inject } from '@nestjs/common';
import { Not, IsNull, DataSource } from 'typeorm';
import { Not, IsNull, Like, DataSource } from 'typeorm';
import type { MiUser } from '@/models/User.js';
import { AppLockService } from '@/core/AppLockService.js';
import { DI } from '@/di-symbols.js';
@ -37,7 +37,10 @@ export default class UsersChart extends Chart<typeof schema> { // eslint-disable
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
this.usersRepository.countBy({ host: IsNull() }),
// that Not(Like()) is ugly, but it matches the logic in
// packages/backend/src/models/User.ts to not count "system"
// accounts
this.usersRepository.countBy({ host: IsNull(), username: Not(Like('%.%')) }),
this.usersRepository.countBy({ host: Not(IsNull()) }),
]);

28
packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View file

@ -33,6 +33,12 @@ export class CleanRemoteFilesProcessorService {
let deletedCount = 0;
let cursor: MiDriveFile['id'] | null = null;
let errorCount = 0;
const total = await this.driveFilesRepository.countBy({
userHost: Not(IsNull()),
isLink: false,
});
while (true) {
const files = await this.driveFilesRepository.find({
@ -41,7 +47,7 @@ export class CleanRemoteFilesProcessorService {
isLink: false,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 8,
take: 256,
order: {
id: 1,
},
@ -54,18 +60,22 @@ export class CleanRemoteFilesProcessorService {
cursor = files.at(-1)?.id ?? null;
await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true)));
// Handle deletion in a batch
const results = await Promise.allSettled(files.map(file => this.driveService.deleteFileSync(file, true)));
deletedCount += 8;
const total = await this.driveFilesRepository.countBy({
userHost: Not(IsNull()),
isLink: false,
results.forEach((result, index) => {
if (result.status === 'fulfilled') {
deletedCount++;
} else {
this.logger.error(`Failed to delete file ID ${files[index].id}: ${result.reason}`);
errorCount++;
}
});
job.updateProgress(100 / total * deletedCount);
await job.updateProgress(100 / total * deletedCount);
}
this.logger.succ('All cached remote files has been deleted.');
this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`);
}
}

2
packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts
View file

@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
});
for (const emoji of customEmojis) {
if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) {
if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
this.logger.error(`invalid emoji name: ${emoji.name}`);
continue;
}

13
packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts
View file

@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
continue;
}
const emojiInfo = record.emoji;
if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) {
this.logger.error(`invalid emojiname: ${emojiInfo.name}`);
const nameNfc = emojiInfo.name.normalize('NFC');
if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
this.logger.error(`invalid emojiname: ${nameNfc}`);
continue;
}
const emojiPath = outputPath + '/' + record.fileName;
await this.emojisRepository.delete({
name: emojiInfo.name,
name: nameNfc,
});
const driveFile = await this.driveService.addFile({
user: null,
@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
force: true,
});
await this.customEmojiService.add({
name: emojiInfo.name,
category: emojiInfo.category,
name: nameNfc,
category: emojiInfo.category?.normalize('NFC'),
host: null,
aliases: emojiInfo.aliases,
aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
driveFile,
license: emojiInfo.license,
isSensitive: emojiInfo.isSensitive,

14
packages/backend/src/queue/processors/InboxProcessorService.ts
View file

@ -15,6 +15,7 @@ import InstanceChart from '@/core/chart/charts/instance.js';
import ApRequestChart from '@/core/chart/charts/ap-request.js';
import FederationChart from '@/core/chart/charts/federation.js';
import { getApId } from '@/core/activitypub/type.js';
import type { IActivity } from '@/core/activitypub/type.js';
import type { MiRemoteUser } from '@/models/User.js';
import type { MiUserPublickey } from '@/models/UserPublickey.js';
import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
@ -52,7 +53,7 @@ export class InboxProcessorService {
@bindThis
public async process(job: Bull.Job<InboxJobData>): Promise<string> {
const signature = job.data.signature; // HTTP-signature
const activity = job.data.activity;
let activity = job.data.activity;
//#region Log
const info = Object.assign({}, activity);
@ -150,6 +151,17 @@ export class InboxProcessorService {
throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
}
// アクティビティを正規化
delete activity.signature;
try {
activity = await ldSignature.compact(activity) as IActivity;
} catch (e) {
throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`);
}
// TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする
// https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29
activity.signature = ldSignature;
// もう一度actorチェック
if (authUser.user.uri !== activity.actor) {
throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);

2
packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

11
packages/backend/src/server/api/endpoints/admin/emoji/add.ts
View file

@ -40,7 +40,7 @@ export const meta = {
export const paramDef = {
type: 'object',
properties: {
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' },
category: {
type: 'string',
@ -73,18 +73,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private emojiEntityService: EmojiEntityService,
) {
super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name.normalize('NFC');
const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
if (driveFile == null) throw new ApiError(meta.errors.noSuchFile);
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null });
const emoji = await this.customEmojiService.add({
driveFile,
name: ps.name,
category: ps.category ?? null,
aliases: ps.aliases ?? [],
name: nameNfc,
category: ps.category?.normalize('NFC') ?? null,
aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [],
host: null,
license: ps.license ?? null,
isSensitive: ps.isSensitive ?? false,

9
packages/backend/src/server/api/endpoints/admin/emoji/copy.ts
View file

@ -82,15 +82,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError();
}
const nameNfc = emoji.name.normalize('NFC');
// Duplication Check
const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name);
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
const addedEmoji = await this.customEmojiService.add({
driveFile,
name: emoji.name,
category: emoji.category,
aliases: emoji.aliases,
name: nameNfc,
category: emoji.category?.normalize('NFC'),
aliases: emoji.aliases?.map(a => a.normalize('NFC')),
host: null,
license: emoji.license,
isSensitive: emoji.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts
View file

@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
if (ps.query) {
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' })
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
.orderBy('length(emoji.name)', 'ASC');
}

11
packages/backend/src/server/api/endpoints/admin/emoji/list.ts
View file

@ -92,17 +92,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//const emojis = await q.limit(ps.limit).getMany();
emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g);
const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
if (queryarry) {
emojis = emojis.filter(emoji =>
queryarry.includes(`:${emoji.name}:`),
queryarry.includes(`:${emoji.name.normalize('NFC')}:`),
);
} else {
const queryNfc = ps.query!.normalize('NFC');
emojis = emojis.filter(emoji =>
emoji.name.includes(ps.query!) ||
emoji.aliases.some(a => a.includes(ps.query!)) ||
emoji.category?.includes(ps.query!));
emoji.name.includes(queryNfc) ||
emoji.aliases.some(a => a.includes(queryNfc)) ||
emoji.category?.includes(queryNfc));
}
emojis.splice(ps.limit + 1);
} else {

2
packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

2
packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

2
packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts
View file

@ -36,7 +36,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null);
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null);
});
}
}

17
packages/backend/src/server/api/endpoints/admin/emoji/update.ts
View file

@ -40,7 +40,7 @@ export const paramDef = {
type: 'object',
properties: {
id: { type: 'string', format: 'misskey:id' },
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' },
category: {
type: 'string',
@ -72,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name?.normalize('NFC');
let driveFile;
if (ps.fileId) {
driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
@ -83,22 +84,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
emojiId = ps.id;
const emoji = await this.customEmojiService.getEmojiById(ps.id);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
if (ps.name && (ps.name !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
if (nameNfc && (nameNfc !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists);
}
} else {
if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(ps.name);
if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(nameNfc);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
emojiId = emoji.id;
}
await this.customEmojiService.update(emojiId, {
driveFile,
name: ps.name,
category: ps.category,
aliases: ps.aliases,
name: nameNfc,
category: ps.category?.normalize('NFC'),
aliases: ps.aliases?.map(a => a.normalize('NFC')),
license: ps.license,
isSensitive: ps.isSensitive,
localOnly: ps.localOnly,

2
packages/backend/src/server/api/mastodon/converters.ts
View file

@ -278,7 +278,7 @@ export class MastoConverters {
reactions: status.emoji_reactions,
emoji_reactions: status.emoji_reactions,
bookmarked: false,
quote: isQuote ? await this.convertReblog(status.reblog) : false,
quote: isQuote ? await this.convertReblog(status.reblog) : null,
// optional chaining cannot be used, as it evaluates to undefined, not null
edited_at: note.updatedAt ? note.updatedAt.toISOString() : null,
});