From 8b46edeccf5a8907afbb871e3eb5b3b8eef967a8 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 30 Dec 2022 12:00:50 +0900 Subject: [PATCH] enhance: Proxy custom emojis to reduce image size and accelerate the frontend (#9431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(server): /emoji to accept `@.` host expression * fix(client): use MkEmoji for custom emoji in MkEmojiPicker * change convertToWebp * nanka iroiro * remove * fix * nearLosslessは労多くして益少なしなのでやめる * do not cleanup tmp for development * update sharp.js to 0.31.3 * mixed: true * fix MkAutocomplete of 912791b3ab * clean up * https://github.com/misskey-dev/misskey/pull/9431#discussion_r1059215943 --- packages/backend/package.json | 2 +- .../backend/src/core/CustomEmojiService.ts | 7 +-- packages/backend/src/core/DownloadService.ts | 2 +- .../src/core/ImageProcessingService.ts | 20 +++++--- packages/backend/src/misc/create-temp.ts | 4 +- packages/backend/src/misc/prelude/url.ts | 5 ++ .../src/server/MediaProxyServerService.ts | 25 +++++++--- .../src/server/web/ClientServerService.ts | 13 +++-- .../src/components/MkAutocomplete.vue | 31 +++++++----- .../frontend/src/components/MkEmojiPicker.vue | 1 - .../frontend/src/components/MkMediaImage.vue | 2 +- .../src/components/global/MkAvatar.vue | 2 +- .../src/components/global/MkEmoji.vue | 2 +- .../frontend/src/pages/user/index.photos.vue | 2 +- .../src/scripts/get-static-image-url.ts | 19 ------- packages/frontend/src/scripts/media-proxy.ts | 34 ++++++++++++- packages/frontend/src/scripts/url.ts | 5 ++ packages/frontend/src/widgets/photos.vue | 2 +- packages/sw/src/scripts/url.ts | 5 ++ yarn.lock | 49 ++++++++----------- 20 files changed, 140 insertions(+), 92 deletions(-) delete mode 100644 packages/frontend/src/scripts/get-static-image-url.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index a742991f3e..b206815641 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -105,7 +105,7 @@ "sanitize-html": "2.8.1", "seedrandom": "^3.0.5", "semver": "7.3.8", - "sharp": "0.29.3", + "sharp": "0.31.3", "speakeasy": "2.0.0", "strict-event-emitter-types": "2.0.0", "stringz": "2.1.0", diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index ff52ad27d6..61cf811192 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -2,12 +2,10 @@ import { Inject, Injectable } from '@nestjs/common'; import { DataSource, In, IsNull } from 'typeorm'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; import { IdService } from '@/core/IdService.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Emoji } from '@/models/entities/Emoji.js'; import { Cache } from '@/misc/cache.js'; -import { query } from '@/misc/prelude/url.js'; import type { Note } from '@/models/entities/Note.js'; import type { EmojisRepository } from '@/models/index.js'; import { UtilityService } from '@/core/UtilityService.js'; @@ -27,9 +25,6 @@ export class CustomEmojiService { private cache: Cache; constructor( - @Inject(DI.config) - private config: Config, - @Inject(DI.db) private db: DataSource, @@ -117,7 +112,7 @@ export class CustomEmojiService { const isLocal = emoji.host == null; const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため - const url = isLocal ? emojiUrl : `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`; + const url = emojiUrl; return { name: emojiName, diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 9097bb08e0..62123246a7 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -33,7 +33,7 @@ export class DownloadService { @bindThis public async downloadUrl(url: string, path: string): Promise { - this.logger.info(`Downloading ${chalk.cyan(url)} ...`); + this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); const timeout = 30 * 1000; const operationTimeout = 60 * 1000; diff --git a/packages/backend/src/core/ImageProcessingService.ts b/packages/backend/src/core/ImageProcessingService.ts index 3a61873044..312189eea4 100644 --- a/packages/backend/src/core/ImageProcessingService.ts +++ b/packages/backend/src/core/ImageProcessingService.ts @@ -8,6 +8,16 @@ export type IImage = { ext: string | null; type: string; }; + +export const webpDefault: sharp.WebpOptions = { + quality: 85, + alphaQuality: 95, + lossless: false, + nearLossless: false, + smartSubsample: true, + mixed: true, +}; + import { bindThis } from '@/decorators.js'; @Injectable() @@ -53,21 +63,19 @@ export class ImageProcessingService { * with resize, remove metadata, resolve orientation, stop animation */ @bindThis - public async convertToWebp(path: string, width: number, height: number, quality = 85): Promise { - return this.convertSharpToWebp(await sharp(path), width, height, quality); + public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise { + return this.convertSharpToWebp(await sharp(path), width, height, options); } @bindThis - public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, quality = 85): Promise { + public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise { const data = await sharp .resize(width, height, { fit: 'inside', withoutEnlargement: true, }) .rotate() - .webp({ - quality, - }) + .webp(options) .toBuffer(); return { diff --git a/packages/backend/src/misc/create-temp.ts b/packages/backend/src/misc/create-temp.ts index 429977669e..7b8942e308 100644 --- a/packages/backend/src/misc/create-temp.ts +++ b/packages/backend/src/misc/create-temp.ts @@ -4,7 +4,7 @@ export function createTemp(): Promise<[string, () => void]> { return new Promise<[string, () => void]>((res, rej) => { tmp.file((e, path, fd, cleanup) => { if (e) return rej(e); - res([path, cleanup]); + res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]); }); }); } @@ -17,7 +17,7 @@ export function createTempDir(): Promise<[string, () => void]> { }, (e, path, cleanup) => { if (e) return rej(e); - res([path, cleanup]); + res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]); }, ); }); diff --git a/packages/backend/src/misc/prelude/url.ts b/packages/backend/src/misc/prelude/url.ts index a4f2f7f5a8..9b1dabc789 100644 --- a/packages/backend/src/misc/prelude/url.ts +++ b/packages/backend/src/misc/prelude/url.ts @@ -1,3 +1,8 @@ +/* objを検査して + * 1. 配列に何も入っていない時はクエリを付けない + * 2. プロパティがundefinedの時はクエリを付けない + * （new URLSearchParams(obj)ではそこまで丁寧なことをしてくれない） + */ export function query(obj: Record): string { const params = Object.entries(obj) .filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined) diff --git a/packages/backend/src/server/MediaProxyServerService.ts b/packages/backend/src/server/MediaProxyServerService.ts index 7355afcb98..4491a17545 100644 --- a/packages/backend/src/server/MediaProxyServerService.ts +++ b/packages/backend/src/server/MediaProxyServerService.ts @@ -9,7 +9,7 @@ import type { Config } from '@/config.js'; import { isMimeImage } from '@/misc/is-mime-image.js'; import { createTemp } from '@/misc/create-temp.js'; import { DownloadService } from '@/core/DownloadService.js'; -import { ImageProcessingService } from '@/core/ImageProcessingService.js'; +import { ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js'; import type { IImage } from '@/core/ImageProcessingService.js'; import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; import { StatusError } from '@/misc/status-error.js'; @@ -81,8 +81,21 @@ export class MediaProxyServerService { const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image'); let image: IImage; - - if ('static' in request.query && isConvertibleImage) { + if ('emoji' in request.query && isConvertibleImage) { + const data = await sharp(path, { animated: !('static' in request.query) }) + .resize({ + height: 128, + withoutEnlargement: true, + }) + .webp(webpDefault) + .toBuffer(); + + image = { + data, + ext: 'webp', + type: 'image/webp', + }; + } else if ('static' in request.query && isConvertibleImage) { image = await this.imageProcessingService.convertToWebp(path, 498, 280); } else if ('preview' in request.query && isConvertibleImage) { image = await this.imageProcessingService.convertToWebp(path, 200, 200); @@ -91,7 +104,7 @@ export class MediaProxyServerService { // 画像でないなら404でお茶を濁す throw new StatusError('Unexpected mime', 404); } - + const mask = sharp(path) .resize(96, 96, { fit: 'inside', @@ -121,8 +134,8 @@ export class MediaProxyServerService { ext: 'png', type: 'image/png', }; - } else if (mime === 'image/svg+xml') { - image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, 1); + } else if (mime === 'image/svg+xml') { + image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, webpDefault); } else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) { throw new StatusError('Rejected type', 403, 'Rejected type'); } else { diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index 84e8481d55..83a30dbe0b 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -220,7 +220,7 @@ export class ClientServerService { return reply.sendFile('/apple-touch-icon.png', staticAssets); }); - fastify.get<{ Params: { path: string } }>('/emoji/:path(.*)', async (request, reply) => { + fastify.get<{ Params: { path: string }; Querystring: { static?: any; }; }>('/emoji/:path(.*)', async (request, reply) => { const path = request.params.path; if (!path.match(/^[a-zA-Z0-9\-_@\.]+?\.webp$/)) { @@ -244,8 +244,15 @@ export class ClientServerService { reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\''); - // ?? emoji.originalUrl してるのは後方互換性のため - return await reply.redirect(301, emoji.publicUrl ?? emoji.originalUrl); + const url = new URL("/proxy/emoji.webp", this.config.url); + url.searchParams.set('url', emoji.publicUrl ?? emoji.originalUrl); // ?? emoji.originalUrl してるのは後方互換性のため + url.searchParams.set('emoji', '1'); + if ('static' in request.query) url.searchParams.set('static', '1'); + + return await reply.redirect( + 301, + url.toString(), + ); }); fastify.get<{ Params: { path: string } }>('/fluent-emoji/:path(.*)', async (request, reply) => { diff --git a/packages/frontend/src/components/MkAutocomplete.vue b/packages/frontend/src/components/MkAutocomplete.vue index 6b1b48e480..d150436fb2 100644 --- a/packages/frontend/src/components/MkAutocomplete.vue +++ b/packages/frontend/src/components/MkAutocomplete.vue @@ -16,12 +16,12 @@
    -
  1. - - - {{ emoji.emoji }} +
  2. +
    + +
    - + ({{ emoji.aliasOf }})
@@ -37,7 +37,6 @@ import { markRaw, ref, onUpdated, onMounted, onBeforeUnmount, nextTick, watch } from 'vue'; import contains from '@/scripts/contains'; import { char2twemojiFilePath, char2fluentEmojiFilePath } from '@/scripts/emoji-base'; -import { getStaticImageUrl } from '@/scripts/get-static-image-url'; import { acct } from '@/filters/user'; import * as os from '@/os'; import { MFM_TAGS } from '@/scripts/mfm-tags'; @@ -49,9 +48,13 @@ import { i18n } from '@/i18n'; type EmojiDef = { emoji: string; name: string; + url: string; aliasOf?: string; - url?: string; - isCustomEmoji?: boolean; +} | { + emoji: string; + name: string; + aliasOf?: string; + isCustomEmoji?: true; }; const lib = emojilist.filter(x => x.category !== 'flags'); @@ -87,7 +90,6 @@ for (const x of customEmojis) { emojiDefinitions.push({ name: x.name, emoji: `:${x.name}:`, - url: x.url, isCustomEmoji: true, }); @@ -97,7 +99,6 @@ for (const x of customEmojis) { name: alias, aliasOf: x.name, emoji: `:${x.name}:`, - url: x.url, isCustomEmoji: true, }); } @@ -452,14 +453,20 @@ onBeforeUnmount(() => { > .emojis > li { .emoji { - display: inline-block; + display: flex; margin: 0 4px 0 0; + height: 24px; width: 24px; + justify-content: center; + align-items: center; + font-size: 20px; > img { + height: 24px; width: 24px; - vertical-align: bottom; + object-fit: scale-down; } + } .alias { diff --git a/packages/frontend/src/components/MkEmojiPicker.vue b/packages/frontend/src/components/MkEmojiPicker.vue index e9e265a916..c94da97747 100644 --- a/packages/frontend/src/components/MkEmojiPicker.vue +++ b/packages/frontend/src/components/MkEmojiPicker.vue @@ -81,7 +81,6 @@ import { ref, computed, watch, onMounted } from 'vue'; import * as Misskey from 'misskey-js'; import XSection from '@/components/MkEmojiPicker.section.vue'; import { emojilist, UnicodeEmojiDef, unicodeEmojiCategories as categories } from '@/scripts/emojilist'; -import { getStaticImageUrl } from '@/scripts/get-static-image-url'; import Ripple from '@/components/MkRipple.vue'; import * as os from '@/os'; import { isTouchUsing } from '@/scripts/touch'; diff --git a/packages/frontend/src/components/MkMediaImage.vue b/packages/frontend/src/components/MkMediaImage.vue index 56570eaa05..9912faffe8 100644 --- a/packages/frontend/src/components/MkMediaImage.vue +++ b/packages/frontend/src/components/MkMediaImage.vue @@ -23,7 +23,7 @@