heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

merge: handle non-ASCII emoji names (!464)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/464

Approved-by: Leah <kevinlukej@gmail.com>
Approved-by: Ember <acomputerdog@gmail.com>
Approved-by: Marie <marie@kaifa.ch>
This commit is contained in:
dakkar 2024-05-02 21:06:10 +00:00
commit d0a2708f91
19 changed files with 55 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/package.json
View file

@ -88,7 +88,7 @@
"@smithy/node-http-handler": "2.1.10",
"@swc/cli": "0.1.63",
"@swc/core": "1.3.107",
"@transfem-org/sfm-js": "0.24.4",
"@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.0.0",
"accepts": "1.3.8",
"ajv": "8.12.0",

4
packages/backend/src/core/ReactionService.ts
View file

@ -64,8 +64,8 @@ type DecodedReaction = {
host?: string | null;
};
const isCustomEmojiRegexp = /^:([\w+-]+)(?:@\.)?:$/;
const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/;
const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/;
@Injectable()
export class ReactionService {

2
packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts
View file

@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
});
for (const emoji of customEmojis) {
if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) {
if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
this.logger.error(`invalid emoji name: ${emoji.name}`);
continue;
}

13
packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts
View file

@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
continue;
}
const emojiInfo = record.emoji;
if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) {
this.logger.error(`invalid emojiname: ${emojiInfo.name}`);
const nameNfc = emojiInfo.name.normalize('NFC');
if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
this.logger.error(`invalid emojiname: ${nameNfc}`);
continue;
}
const emojiPath = outputPath + '/' + record.fileName;
await this.emojisRepository.delete({
name: emojiInfo.name,
name: nameNfc,
});
const driveFile = await this.driveService.addFile({
user: null,
@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
force: true,
});
await this.customEmojiService.add({
name: emojiInfo.name,
category: emojiInfo.category,
name: nameNfc,
category: emojiInfo.category?.normalize('NFC'),
host: null,
aliases: emojiInfo.aliases,
aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
driveFile,
license: emojiInfo.license,
isSensitive: emojiInfo.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

11
packages/backend/src/server/api/endpoints/admin/emoji/add.ts
View file

@ -40,7 +40,7 @@ export const meta = {
export const paramDef = {
type: 'object',
properties: {
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' },
category: {
type: 'string',
@ -73,18 +73,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private emojiEntityService: EmojiEntityService,
) {
super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name.normalize('NFC');
const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
if (driveFile == null) throw new ApiError(meta.errors.noSuchFile);
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null });
const emoji = await this.customEmojiService.add({
driveFile,
name: ps.name,
category: ps.category ?? null,
aliases: ps.aliases ?? [],
name: nameNfc,
category: ps.category?.normalize('NFC') ?? null,
aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [],
host: null,
license: ps.license ?? null,
isSensitive: ps.isSensitive ?? false,

9
packages/backend/src/server/api/endpoints/admin/emoji/copy.ts
View file

@ -82,15 +82,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError();
}
const nameNfc = emoji.name.normalize('NFC');
// Duplication Check
const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name);
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
const addedEmoji = await this.customEmojiService.add({
driveFile,
name: emoji.name,
category: emoji.category,
aliases: emoji.aliases,
name: nameNfc,
category: emoji.category?.normalize('NFC'),
aliases: emoji.aliases?.map(a => a.normalize('NFC')),
host: null,
license: emoji.license,
isSensitive: emoji.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts
View file

@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
if (ps.query) {
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' })
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
.orderBy('length(emoji.name)', 'ASC');
}

11
packages/backend/src/server/api/endpoints/admin/emoji/list.ts
View file

@ -92,17 +92,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//const emojis = await q.limit(ps.limit).getMany();
emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g);
const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
if (queryarry) {
emojis = emojis.filter(emoji =>
queryarry.includes(`:${emoji.name}:`),
queryarry.includes(`:${emoji.name.normalize('NFC')}:`),
);
} else {
const queryNfc = ps.query!.normalize('NFC');
emojis = emojis.filter(emoji =>
emoji.name.includes(ps.query!) ||
emoji.aliases.some(a => a.includes(ps.query!)) ||
emoji.category?.includes(ps.query!));
emoji.name.includes(queryNfc) ||
emoji.aliases.some(a => a.includes(queryNfc)) ||
emoji.category?.includes(queryNfc));
}
emojis.splice(ps.limit + 1);
} else {

2
packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

2
packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

2
packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts
View file

@ -36,7 +36,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null);
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null);
});
}
}

17
packages/backend/src/server/api/endpoints/admin/emoji/update.ts
View file

@ -40,7 +40,7 @@ export const paramDef = {
type: 'object',
properties: {
id: { type: 'string', format: 'misskey:id' },
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' },
category: {
type: 'string',
@ -72,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name?.normalize('NFC');
let driveFile;
if (ps.fileId) {
driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
@ -83,22 +84,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
emojiId = ps.id;
const emoji = await this.customEmojiService.getEmojiById(ps.id);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
if (ps.name && (ps.name !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
if (nameNfc && (nameNfc !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists);
}
} else {
if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(ps.name);
if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(nameNfc);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
emojiId = emoji.id;
}
await this.customEmojiService.update(emojiId, {
driveFile,
name: ps.name,
category: ps.category,
aliases: ps.aliases,
name: nameNfc,
category: ps.category?.normalize('NFC'),
aliases: ps.aliases?.map(a => a.normalize('NFC')),
license: ps.license,
isSensitive: ps.isSensitive,
localOnly: ps.localOnly,