heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

merge: handle non-ASCII emoji names (!464)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/464

Approved-by: Leah <kevinlukej@gmail.com>
Approved-by: Ember <acomputerdog@gmail.com>
Approved-by: Marie <marie@kaifa.ch>
This commit is contained in:
dakkar 2024-05-02 21:06:10 +00:00
commit d0a2708f91
19 changed files with 55 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/package.json
View file

@ -88,7 +88,7 @@
"@smithy/node-http-handler": "2.1.10",
"@swc/cli": "0.1.63",
"@swc/core": "1.3.107",
"@transfem-org/sfm-js": "0.24.4",
"@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.0.0",
"accepts": "1.3.8",
"ajv": "8.12.0",

4
packages/backend/src/core/ReactionService.ts
View file

@ -64,8 +64,8 @@ type DecodedReaction = {
host?: string | null;
};
const isCustomEmojiRegexp = /^:([\w+-]+)(?:@\.)?:$/;
const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/;
const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/;
@Injectable()
export class ReactionService {

2
packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts
View file

@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService {
});
for (const emoji of customEmojis) {
if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) {
if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) {
this.logger.error(`invalid emoji name: ${emoji.name}`);
continue;
}

13
packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts
View file

@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService {
continue;
}
const emojiInfo = record.emoji;
if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) {
this.logger.error(`invalid emojiname: ${emojiInfo.name}`);
const nameNfc = emojiInfo.name.normalize('NFC');
if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) {
this.logger.error(`invalid emojiname: ${nameNfc}`);
continue;
}
const emojiPath = outputPath + '/' + record.fileName;
await this.emojisRepository.delete({
name: emojiInfo.name,
name: nameNfc,
});
const driveFile = await this.driveService.addFile({
user: null,
@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService {
force: true,
});
await this.customEmojiService.add({
name: emojiInfo.name,
category: emojiInfo.category,
name: nameNfc,
category: emojiInfo.category?.normalize('NFC'),
host: null,
aliases: emojiInfo.aliases,
aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')),
driveFile,
license: emojiInfo.license,
isSensitive: emojiInfo.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

11
packages/backend/src/server/api/endpoints/admin/emoji/add.ts
View file

@ -40,7 +40,7 @@ export const meta = {
export const paramDef = {
type: 'object',
properties: {
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' },
category: {
type: 'string',
@ -73,18 +73,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private emojiEntityService: EmojiEntityService,
) {
super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name.normalize('NFC');
const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
if (driveFile == null) throw new ApiError(meta.errors.noSuchFile);
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null });
const emoji = await this.customEmojiService.add({
driveFile,
name: ps.name,
category: ps.category ?? null,
aliases: ps.aliases ?? [],
name: nameNfc,
category: ps.category?.normalize('NFC') ?? null,
aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [],
host: null,
license: ps.license ?? null,
isSensitive: ps.isSensitive ?? false,

9
packages/backend/src/server/api/endpoints/admin/emoji/copy.ts
View file

@ -82,15 +82,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError();
}
const nameNfc = emoji.name.normalize('NFC');
// Duplication Check
const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name);
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.duplicateName);
const addedEmoji = await this.customEmojiService.add({
driveFile,
name: emoji.name,
category: emoji.category,
aliases: emoji.aliases,
name: nameNfc,
category: emoji.category?.normalize('NFC'),
aliases: emoji.aliases?.map(a => a.normalize('NFC')),
host: null,
license: emoji.license,
isSensitive: emoji.isSensitive,

2
packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts
View file

@ -98,7 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
if (ps.query) {
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' })
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
.orderBy('length(emoji.name)', 'ASC');
}

11
packages/backend/src/server/api/endpoints/admin/emoji/list.ts
View file

@ -92,17 +92,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
//const emojis = await q.limit(ps.limit).getMany();
emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany();
const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g);
const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug);
if (queryarry) {
emojis = emojis.filter(emoji =>
queryarry.includes(`:${emoji.name}:`),
queryarry.includes(`:${emoji.name.normalize('NFC')}:`),
);
} else {
const queryNfc = ps.query!.normalize('NFC');
emojis = emojis.filter(emoji =>
emoji.name.includes(ps.query!) ||
emoji.aliases.some(a => a.includes(ps.query!)) ||
emoji.category?.includes(ps.query!));
emoji.name.includes(queryNfc) ||
emoji.aliases.some(a => a.includes(queryNfc)) ||
emoji.category?.includes(queryNfc));
}
emojis.splice(ps.limit + 1);
} else {

2
packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

2
packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts
View file

@ -34,7 +34,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases);
await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC')));
});
}
}

2
packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts
View file

@ -36,7 +36,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null);
await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null);
});
}
}

17
packages/backend/src/server/api/endpoints/admin/emoji/update.ts
View file

@ -40,7 +40,7 @@ export const paramDef = {
type: 'object',
properties: {
id: { type: 'string', format: 'misskey:id' },
name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' },
name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' },
fileId: { type: 'string', format: 'misskey:id' },
category: {
type: 'string',
@ -72,6 +72,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private customEmojiService: CustomEmojiService,
) {
super(meta, paramDef, async (ps, me) => {
const nameNfc = ps.name?.normalize('NFC');
let driveFile;
if (ps.fileId) {
driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
@ -83,22 +84,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
emojiId = ps.id;
const emoji = await this.customEmojiService.getEmojiById(ps.id);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
if (ps.name && (ps.name !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name);
if (nameNfc && (nameNfc !== emoji.name)) {
const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc);
if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists);
}
} else {
if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(ps.name);
if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.');
const emoji = await this.customEmojiService.getEmojiByName(nameNfc);
if (!emoji) throw new ApiError(meta.errors.noSuchEmoji);
emojiId = emoji.id;
}
await this.customEmojiService.update(emojiId, {
driveFile,
name: ps.name,
category: ps.category,
aliases: ps.aliases,
name: nameNfc,
category: ps.category?.normalize('NFC'),
aliases: ps.aliases?.map(a => a.normalize('NFC')),
license: ps.license,
isSensitive: ps.isSensitive,
localOnly: ps.localOnly,

4
packages/frontend/package.json
View file

@ -21,12 +21,12 @@
"@github/webauthn-json": "2.1.1",
"@mcaptcha/vanilla-glue": "0.1.0-alpha-3",
"@misskey-dev/browser-image-resizer": "2024.1.0",
"@phosphor-icons/web": "^2.0.3",
"@rollup/plugin-json": "6.1.0",
"@rollup/plugin-replace": "5.0.5",
"@rollup/pluginutils": "5.1.0",
"@transfem-org/sfm-js": "0.24.4",
"@syuilo/aiscript": "0.17.0",
"@phosphor-icons/web": "^2.0.3",
"@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.0.0",
"@vitejs/plugin-vue": "5.0.4",
"@vue/compiler-sfc": "3.4.21",

2
packages/frontend/src/components/MkAutocomplete.vue
View file

@ -238,7 +238,7 @@ function exec() {
return;
}
emojis.value = searchEmoji(props.q.toLowerCase(), emojiDb.value);
emojis.value = searchEmoji(props.q.normalize('NFC').toLowerCase(), emojiDb.value);
} else if (props.type === 'mfmTag') {
if (!props.q || props.q === '') {
mfmTags.value = MFM_TAGS;

2
packages/frontend/src/components/MkEmojiPicker.vue
View file

@ -205,7 +205,7 @@ watch(q, () => {
return;
}
const newQ = q.value.replace(/:/g, '').toLowerCase();
const newQ = q.value.replace(/:/g, '').normalize('NFC').toLowerCase();
const searchCustom = () => {
const max = 100;

2
packages/frontend/src/pages/emoji-edit-dialog.vue
View file

@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only
</div>
</div>
<MkButton rounded style="margin: 0 auto;" @click="changeImage">{{ i18n.ts.selectFile }}</MkButton>
<MkInput v-model="name" pattern="[a-z0-9_]" autocapitalize="off">
<MkInput v-model="name" autocapitalize="off">
<template #label>{{ i18n.ts.name }}</template>
</MkInput>
<MkInput v-model="category" :datalist="customEmojiCategories">

4
packages/frontend/src/scripts/autocomplete.ts
View file

@ -99,7 +99,7 @@ export class Autocomplete {
const isHashtag = hashtagIndex !== -1;
const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' ');
const isMfmTag = mfmTagIndex !== -1 && !isMfmParam;
const isEmoji = emojiIndex !== -1 && text.split(/:[a-z0-9_+\-]+:/).pop()!.includes(':');
const isEmoji = emojiIndex !== -1 && text.split(/:[\p{Letter}\p{Number}\p{Mark}_+-]+:/u).pop()!.includes(':');
let opened = false;
@ -125,7 +125,7 @@ export class Autocomplete {
if (isEmoji && !opened && this.onlyType.includes('emoji')) {
const emoji = text.substring(emojiIndex + 1);
if (!emoji.includes(' ')) {
this.open('emoji', emoji);
this.open('emoji', emoji.normalize('NFC'));
opened = true;
}
}

12
pnpm-lock.yaml
View file

@ -140,8 +140,8 @@ importers:
specifier: 1.3.107
version: 1.3.107
'@transfem-org/sfm-js':
specifier: 0.24.4
version: 0.24.4
specifier: 0.24.5
version: 0.24.5
'@twemoji/parser':
specifier: 15.0.0
version: 15.0.0
@ -709,8 +709,8 @@ importers:
specifier: 0.17.0
version: 0.17.0
'@transfem-org/sfm-js':
specifier: 0.24.4
version: 0.24.4
specifier: 0.24.5
version: 0.24.5
'@twemoji/parser':
specifier: 15.0.0
version: 15.0.0
@ -7436,8 +7436,8 @@ packages:
resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
dev: false
/@transfem-org/sfm-js@0.24.4:
resolution: {integrity: sha1-0wEXqL5UJseGFO4GGFRrES6NCDk=, tarball: https://activitypub.software/api/v4/projects/2/packages/npm/@transfem-org/sfm-js/-/@transfem-org/sfm-js-0.24.4.tgz}
/@transfem-org/sfm-js@0.24.5:
resolution: {integrity: sha1-c9qJO12lIG+kovDGKjZmK2qPqcw=, tarball: https://activitypub.software/api/v4/projects/2/packages/npm/@transfem-org/sfm-js/-/@transfem-org/sfm-js-0.24.5.tgz}
dependencies:
'@twemoji/parser': 15.0.0
dev: false