enhance: Proxy custom emojis to reduce image size and accelerate the frontend (#9431)

* fix(server): /emoji to accept `@.` host expression

* fix(client): use MkEmoji for custom emoji in MkEmojiPicker

* change convertToWebp

* nanka iroiro

* remove

* fix

* nearLosslessは労多くして益少なしなのでやめる

* do not cleanup tmp for development

* update sharp.js to 0.31.3

* mixed: true

* fix MkAutocomplete of 912791b3ab

* clean up

* https://github.com/misskey-dev/misskey/pull/9431#discussion_r1059215943
This commit is contained in:
tamaina 2022-12-30 12:00:50 +09:00 committed by GitHub
parent f227091826
commit 8b46edeccf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 140 additions and 92 deletions

View File

@ -105,7 +105,7 @@
"sanitize-html": "2.8.1",
"seedrandom": "^3.0.5",
"semver": "7.3.8",
"sharp": "0.29.3",
"sharp": "0.31.3",
"speakeasy": "2.0.0",
"strict-event-emitter-types": "2.0.0",
"stringz": "2.1.0",

View File

@ -2,12 +2,10 @@ import { Inject, Injectable } from '@nestjs/common';
import { DataSource, In, IsNull } from 'typeorm';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { IdService } from '@/core/IdService.js';
import type { DriveFile } from '@/models/entities/DriveFile.js';
import type { Emoji } from '@/models/entities/Emoji.js';
import { Cache } from '@/misc/cache.js';
import { query } from '@/misc/prelude/url.js';
import type { Note } from '@/models/entities/Note.js';
import type { EmojisRepository } from '@/models/index.js';
import { UtilityService } from '@/core/UtilityService.js';
@ -27,9 +25,6 @@ export class CustomEmojiService {
private cache: Cache<Emoji | null>;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.db)
private db: DataSource,
@ -117,7 +112,7 @@ export class CustomEmojiService {
const isLocal = emoji.host == null;
const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため
const url = isLocal ? emojiUrl : `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`;
const url = emojiUrl;
return {
name: emojiName,

View File

@ -33,7 +33,7 @@ export class DownloadService {
@bindThis
public async downloadUrl(url: string, path: string): Promise<void> {
this.logger.info(`Downloading ${chalk.cyan(url)} ...`);
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;

View File

@ -8,6 +8,16 @@ export type IImage = {
ext: string | null;
type: string;
};
export const webpDefault: sharp.WebpOptions = {
quality: 85,
alphaQuality: 95,
lossless: false,
nearLossless: false,
smartSubsample: true,
mixed: true,
};
import { bindThis } from '@/decorators.js';
@Injectable()
@ -53,21 +63,19 @@ export class ImageProcessingService {
* with resize, remove metadata, resolve orientation, stop animation
*/
@bindThis
public async convertToWebp(path: string, width: number, height: number, quality = 85): Promise<IImage> {
return this.convertSharpToWebp(await sharp(path), width, height, quality);
public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
return this.convertSharpToWebp(await sharp(path), width, height, options);
}
@bindThis
public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, quality = 85): Promise<IImage> {
public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
const data = await sharp
.resize(width, height, {
fit: 'inside',
withoutEnlargement: true,
})
.rotate()
.webp({
quality,
})
.webp(options)
.toBuffer();
return {

View File

@ -4,7 +4,7 @@ export function createTemp(): Promise<[string, () => void]> {
return new Promise<[string, () => void]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
});
});
}
@ -17,7 +17,7 @@ export function createTempDir(): Promise<[string, () => void]> {
},
(e, path, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
},
);
});

View File

@ -1,3 +1,8 @@
/* obj
* 1.
* 2. undefinedの時はクエリを付けない
* new URLSearchParams(obj)
*/
export function query(obj: Record<string, unknown>): string {
const params = Object.entries(obj)
.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

View File

@ -9,7 +9,7 @@ import type { Config } from '@/config.js';
import { isMimeImage } from '@/misc/is-mime-image.js';
import { createTemp } from '@/misc/create-temp.js';
import { DownloadService } from '@/core/DownloadService.js';
import { ImageProcessingService } from '@/core/ImageProcessingService.js';
import { ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js';
import type { IImage } from '@/core/ImageProcessingService.js';
import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { StatusError } from '@/misc/status-error.js';
@ -81,8 +81,21 @@ export class MediaProxyServerService {
const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image');
let image: IImage;
if ('emoji' in request.query && isConvertibleImage) {
const data = await sharp(path, { animated: !('static' in request.query) })
.resize({
height: 128,
withoutEnlargement: true,
})
.webp(webpDefault)
.toBuffer();
if ('static' in request.query && isConvertibleImage) {
image = {
data,
ext: 'webp',
type: 'image/webp',
};
} else if ('static' in request.query && isConvertibleImage) {
image = await this.imageProcessingService.convertToWebp(path, 498, 280);
} else if ('preview' in request.query && isConvertibleImage) {
image = await this.imageProcessingService.convertToWebp(path, 200, 200);
@ -122,7 +135,7 @@ export class MediaProxyServerService {
type: 'image/png',
};
} else if (mime === 'image/svg+xml') {
image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, 1);
image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, webpDefault);
} else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) {
throw new StatusError('Rejected type', 403, 'Rejected type');
} else {

View File

@ -220,7 +220,7 @@ export class ClientServerService {
return reply.sendFile('/apple-touch-icon.png', staticAssets);
});
fastify.get<{ Params: { path: string } }>('/emoji/:path(.*)', async (request, reply) => {
fastify.get<{ Params: { path: string }; Querystring: { static?: any; }; }>('/emoji/:path(.*)', async (request, reply) => {
const path = request.params.path;
if (!path.match(/^[a-zA-Z0-9\-_@\.]+?\.webp$/)) {
@ -244,8 +244,15 @@ export class ClientServerService {
reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\'');
// ?? emoji.originalUrl してるのは後方互換性のため
return await reply.redirect(301, emoji.publicUrl ?? emoji.originalUrl);
const url = new URL("/proxy/emoji.webp", this.config.url);
url.searchParams.set('url', emoji.publicUrl ?? emoji.originalUrl); // ?? emoji.originalUrl してるのは後方互換性のため
url.searchParams.set('emoji', '1');
if ('static' in request.query) url.searchParams.set('static', '1');
return await reply.redirect(
301,
url.toString(),
);
});
fastify.get<{ Params: { path: string } }>('/fluent-emoji/:path(.*)', async (request, reply) => {

View File

@ -16,12 +16,12 @@
</li>
</ol>
<ol v-else-if="emojis.length > 0" ref="suggests" class="emojis">
<li v-for="emoji in emojis" tabindex="-1" @click="complete(type, emoji.emoji)" @keydown="onKeydown">
<span v-if="emoji.isCustomEmoji" class="emoji"><img :src="`/emoji/${emoji.name}.webp`" :alt="emoji.emoji"/></span>
<span v-else-if="defaultStore.state.emojiStyle != 'native'" class="emoji"><img :src="emoji.url" :alt="emoji.emoji"/></span>
<span v-else class="emoji">{{ emoji.emoji }}</span>
<li v-for="emoji in emojis" tabindex="-1" :key="emoji.emoji" @click="complete(type, emoji.emoji)" @keydown="onKeydown">
<div class="emoji">
<MkEmoji :emoji="emoji.emoji" />
</div>
<!-- eslint-disable-next-line vue/no-v-html -->
<span class="name" v-html="emoji.name.replace(q, `<b>${q}</b>`)"></span>
<span class="name" v-html="emoji.name.replace(q ?? '', `<b>${q}</b>`)"></span>
<span v-if="emoji.aliasOf" class="alias">({{ emoji.aliasOf }})</span>
</li>
</ol>
@ -37,7 +37,6 @@
import { markRaw, ref, onUpdated, onMounted, onBeforeUnmount, nextTick, watch } from 'vue';
import contains from '@/scripts/contains';
import { char2twemojiFilePath, char2fluentEmojiFilePath } from '@/scripts/emoji-base';
import { getStaticImageUrl } from '@/scripts/get-static-image-url';
import { acct } from '@/filters/user';
import * as os from '@/os';
import { MFM_TAGS } from '@/scripts/mfm-tags';
@ -49,9 +48,13 @@ import { i18n } from '@/i18n';
type EmojiDef = {
emoji: string;
name: string;
url: string;
aliasOf?: string;
url?: string;
isCustomEmoji?: boolean;
} | {
emoji: string;
name: string;
aliasOf?: string;
isCustomEmoji?: true;
};
const lib = emojilist.filter(x => x.category !== 'flags');
@ -87,7 +90,6 @@ for (const x of customEmojis) {
emojiDefinitions.push({
name: x.name,
emoji: `:${x.name}:`,
url: x.url,
isCustomEmoji: true,
});
@ -97,7 +99,6 @@ for (const x of customEmojis) {
name: alias,
aliasOf: x.name,
emoji: `:${x.name}:`,
url: x.url,
isCustomEmoji: true,
});
}
@ -452,14 +453,20 @@ onBeforeUnmount(() => {
> .emojis > li {
.emoji {
display: inline-block;
display: flex;
margin: 0 4px 0 0;
height: 24px;
width: 24px;
justify-content: center;
align-items: center;
font-size: 20px;
> img {
height: 24px;
width: 24px;
vertical-align: bottom;
object-fit: scale-down;
}
}
.alias {

View File

@ -81,7 +81,6 @@ import { ref, computed, watch, onMounted } from 'vue';
import * as Misskey from 'misskey-js';
import XSection from '@/components/MkEmojiPicker.section.vue';
import { emojilist, UnicodeEmojiDef, unicodeEmojiCategories as categories } from '@/scripts/emojilist';
import { getStaticImageUrl } from '@/scripts/get-static-image-url';
import Ripple from '@/components/MkRipple.vue';
import * as os from '@/os';
import { isTouchUsing } from '@/scripts/touch';

View File

@ -23,7 +23,7 @@
<script lang="ts" setup>
import { watch } from 'vue';
import * as misskey from 'misskey-js';
import { getStaticImageUrl } from '@/scripts/get-static-image-url';
import { getStaticImageUrl } from '@/scripts/media-proxy';
import ImgWithBlurhash from '@/components/MkImgWithBlurhash.vue';
import { defaultStore } from '@/store';

View File

@ -12,7 +12,7 @@
<script lang="ts" setup>
import { onMounted, watch } from 'vue';
import * as misskey from 'misskey-js';
import { getStaticImageUrl } from '@/scripts/get-static-image-url';
import { getStaticImageUrl } from '@/scripts/media-proxy';
import { extractAvgColorFromBlurhash } from '@/scripts/extract-avg-color-from-blurhash';
import { acct, userPage } from '@/filters/user';
import MkUserOnlineIndicator from '@/components/MkUserOnlineIndicator.vue';

View File

@ -7,7 +7,7 @@
<script lang="ts" setup>
import { computed } from 'vue';
import { getStaticImageUrl } from '@/scripts/get-static-image-url';
import { getStaticImageUrl } from '@/scripts/media-proxy';
import { char2twemojiFilePath, char2fluentEmojiFilePath } from '@/scripts/emoji-base';
import { defaultStore } from '@/store';
import { getEmojiName } from '@/scripts/emojilist';

View File

@ -21,7 +21,7 @@
<script lang="ts" setup>
import { onMounted } from 'vue';
import * as misskey from 'misskey-js';
import { getStaticImageUrl } from '@/scripts/get-static-image-url';
import { getStaticImageUrl } from '@/scripts/media-proxy';
import { notePage } from '@/filters/note';
import * as os from '@/os';
import MkContainer from '@/components/MkContainer.vue';

View File

@ -1,19 +0,0 @@
import { url as instanceUrl } from '@/config';
import * as url from '@/scripts/url';
export function getStaticImageUrl(baseUrl: string): string {
const u = new URL(baseUrl);
if (u.href.startsWith(`${instanceUrl}/proxy/`)) {
// もう既にproxyっぽそうだったらsearchParams付けるだけ
u.searchParams.set('static', '1');
return u.href;
}
// 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する
const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`;
return `${instanceUrl}/proxy/${dummy}?${url.query({
url: u.href,
static: '1',
})}`;
}

View File

@ -1,7 +1,15 @@
import { query } from '@/scripts/url';
import { query, appendQuery } from '@/scripts/url';
import { url } from '@/config';
export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string {
if (imageUrl.startsWith(`${url}/proxy/`) || imageUrl.startsWith('/proxy/')) {
// もう既にproxyっぽそうだったらsearchParams付けるだけ
return appendQuery(imageUrl, query({
fallback: '1',
...(type ? { [type]: '1' } : {}),
}));
}
return `${url}/proxy/image.webp?${query({
url: imageUrl,
fallback: '1',
@ -13,3 +21,27 @@ export function getProxiedImageUrlNullable(imageUrl: string | null | undefined,
if (imageUrl == null) return null;
return getProxiedImageUrl(imageUrl, type);
}
export function getStaticImageUrl(baseUrl: string): string {
const u = baseUrl.startsWith('http') ? new URL(baseUrl) : new URL(baseUrl, url);
if (u.href.startsWith(`${url}/proxy/`)) {
// もう既にproxyっぽそうだったらsearchParams付けるだけ
u.searchParams.set('static', '1');
return u.href;
}
if (u.href.startsWith(`${url}/emoji/`)) {
// もう既にemojiっぽそうだったらsearchParams付けるだけ
u.searchParams.set('static', '1');
return u.href;
}
// 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する
const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`;
return `${url}/proxy/${dummy}?${query({
url: u.href,
static: '1',
})}`;
}

View File

@ -1,3 +1,8 @@
/* obj
* 1.
* 2. undefinedの時はクエリを付けない
* new URLSearchParams(obj)
*/
export function query(obj: Record<string, any>): string {
const params = Object.entries(obj)
.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

View File

@ -20,7 +20,7 @@ import { onMounted, onUnmounted, reactive, ref } from 'vue';
import { useWidgetPropsManager, Widget, WidgetComponentEmits, WidgetComponentExpose, WidgetComponentProps } from './widget';
import { GetFormResultType } from '@/scripts/form';
import { stream } from '@/stream';
import { getStaticImageUrl } from '@/scripts/get-static-image-url';
import { getStaticImageUrl } from '@/scripts/media-proxy';
import * as os from '@/os';
import MkContainer from '@/components/MkContainer.vue';
import { defaultStore } from '@/store';

View File

@ -1,3 +1,8 @@
/* obj
* 1.
* 2. undefinedの時はクエリを付けない
* new URLSearchParams(obj)
*/
export function query(obj: object): string {
const params = Object.entries(obj)
.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

View File

@ -4257,7 +4257,7 @@ __metadata:
sanitize-html: 2.8.1
seedrandom: ^3.0.5
semver: 7.3.8
sharp: 0.29.3
sharp: 0.31.3
speakeasy: 2.0.0
strict-event-emitter-types: 2.0.0
stringz: 2.1.0
@ -5367,7 +5367,7 @@ __metadata:
languageName: node
linkType: hard
"color@npm:^4.0.1":
"color@npm:^4.2.3":
version: 4.2.3
resolution: "color@npm:4.2.3"
dependencies:
@ -6161,16 +6161,7 @@ __metadata:
languageName: node
linkType: hard
"detect-libc@npm:^1.0.3":
version: 1.0.3
resolution: "detect-libc@npm:1.0.3"
bin:
detect-libc: ./bin/detect-libc.js
checksum: daaaed925ffa7889bd91d56e9624e6c8033911bb60f3a50a74a87500680652969dbaab9526d1e200a4c94acf80fc862a22131841145a0a8482d60a99c24f4a3e
languageName: node
linkType: hard
"detect-libc@npm:^2.0.0":
"detect-libc@npm:^2.0.0, detect-libc@npm:^2.0.1":
version: 2.0.1
resolution: "detect-libc@npm:2.0.1"
checksum: ccb05fcabbb555beb544d48080179c18523a343face9ee4e1a86605a8715b4169f94d663c21a03c310ac824592f2ba9a5270218819bb411ad7be578a527593d7
@ -12167,12 +12158,12 @@ __metadata:
languageName: node
linkType: hard
"node-addon-api@npm:^4.2.0":
version: 4.3.0
resolution: "node-addon-api@npm:4.3.0"
"node-addon-api@npm:^5.0.0":
version: 5.0.0
resolution: "node-addon-api@npm:5.0.0"
dependencies:
node-gyp: latest
checksum: 3de396e23cc209f539c704583e8e99c148850226f6e389a641b92e8967953713228109f919765abc1f4355e801e8f41842f96210b8d61c7dcc10a477002dcf00
checksum: 7c5e2043ac37f6108784d94ed73a44ae6d3e68eb968de60680922fc6bc3d17fa69448c0feb4e0c9d3f4c74a0324822e566a8340a56916d9d6f23cb3e85620334
languageName: node
linkType: hard
@ -13672,7 +13663,7 @@ __metadata:
languageName: node
linkType: hard
"prebuild-install@npm:^7.0.0":
"prebuild-install@npm:^7.1.1":
version: 7.1.1
resolution: "prebuild-install@npm:7.1.1"
dependencies:
@ -15053,7 +15044,7 @@ __metadata:
languageName: node
linkType: hard
"semver@npm:7.3.8, semver@npm:^7.3.2, semver@npm:^7.3.5, semver@npm:^7.3.6, semver@npm:^7.3.7":
"semver@npm:7.3.8, semver@npm:^7.3.2, semver@npm:^7.3.5, semver@npm:^7.3.6, semver@npm:^7.3.7, semver@npm:^7.3.8":
version: 7.3.8
resolution: "semver@npm:7.3.8"
dependencies:
@ -15146,20 +15137,20 @@ __metadata:
languageName: node
linkType: hard
"sharp@npm:0.29.3":
version: 0.29.3
resolution: "sharp@npm:0.29.3"
"sharp@npm:0.31.3":
version: 0.31.3
resolution: "sharp@npm:0.31.3"
dependencies:
color: ^4.0.1
detect-libc: ^1.0.3
node-addon-api: ^4.2.0
color: ^4.2.3
detect-libc: ^2.0.1
node-addon-api: ^5.0.0
node-gyp: latest
prebuild-install: ^7.0.0
semver: ^7.3.5
simple-get: ^4.0.0
prebuild-install: ^7.1.1
semver: ^7.3.8
simple-get: ^4.0.1
tar-fs: ^2.1.1
tunnel-agent: ^0.6.0
checksum: d496cdd546c9abe743aebcee013731295f735687819a18c2bdcbba6f31a6b259f3da95af5c11260a8fedc9d4ab95697f5f8c4f3cd65232792b5cfb876bea7c9a
checksum: 29fd1dfbc616c6389f53f366cec342b4353d9f2a37e98952ca273db38dca57dfa0f336322d6d763f0fae876042ead22fd86ffe26d70c32ade2458d421db60d04
languageName: node
linkType: hard
@ -15204,7 +15195,7 @@ __metadata:
languageName: node
linkType: hard
"simple-get@npm:^4.0.0":
"simple-get@npm:^4.0.0, simple-get@npm:^4.0.1":
version: 4.0.1
resolution: "simple-get@npm:4.0.1"
dependencies: