From 73203a3d72b355e3c230c46771292ff9520675c0 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 6 Apr 2023 11:14:43 +0900 Subject: [PATCH] perf(backend): cache local custom emojis --- .../backend/src/core/CustomEmojiService.ts | 187 +++++++++++++++--- .../backend/src/core/InstanceActorService.ts | 6 +- .../backend/src/core/NoteCreateService.ts | 4 +- packages/backend/src/core/ReactionService.ts | 42 ++-- packages/backend/src/core/RelayService.ts | 6 +- packages/backend/src/core/RoleService.ts | 6 +- .../src/core/activitypub/ApRendererService.ts | 19 +- .../src/core/entities/NoteEntityService.ts | 26 ++- packages/backend/src/misc/cache.ts | 88 ++++++++- .../processors/DeliverProcessorService.ts | 6 +- .../src/server/NodeinfoServerService.ts | 4 +- .../endpoints/admin/emoji/add-aliases-bulk.ts | 34 +--- .../server/api/endpoints/admin/emoji/copy.ts | 2 - .../api/endpoints/admin/emoji/delete-bulk.ts | 35 +--- .../api/endpoints/admin/emoji/delete.ts | 36 +--- .../admin/emoji/remove-aliases-bulk.ts | 34 +--- .../endpoints/admin/emoji/set-aliases-bulk.ts | 30 +-- .../admin/emoji/set-category-bulk.ts | 30 +-- .../api/endpoints/admin/emoji/update.ts | 46 +---- .../src/server/api/endpoints/emojis.ts | 4 - 20 files changed, 335 insertions(+), 310 deletions(-) diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index 1c3b60e5d7..604a94707f 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -1,24 +1,28 @@ import { Inject, Injectable } from '@nestjs/common'; import { DataSource, In, IsNull } from 'typeorm'; +import Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Emoji } from '@/models/entities/Emoji.js'; -import type { EmojisRepository, Note } from '@/models/index.js'; +import type { EmojisRepository } from '@/models/index.js'; import { bindThis } from '@/decorators.js'; -import { MemoryKVCache } from '@/misc/cache.js'; +import { MemoryKVCache, RedisSingleCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import type { Config } from '@/config.js'; -import { ReactionService } from '@/core/ReactionService.js'; import { query } from '@/misc/prelude/url.js'; @Injectable() export class CustomEmojiService { private cache: MemoryKVCache; + public localEmojisCache: RedisSingleCache>; constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, + @Inject(DI.config) private config: Config, @@ -32,9 +36,16 @@ export class CustomEmojiService { private idService: IdService, private emojiEntityService: EmojiEntityService, private globalEventService: GlobalEventService, - private reactionService: ReactionService, ) { this.cache = new MemoryKVCache(1000 * 60 * 60 * 12); + + this.localEmojisCache = new RedisSingleCache>(this.redisClient, 'localEmojis', { + lifetime: 1000 * 60 * 30, // 30m + memoryCacheLifetime: 1000 * 60 * 3, // 3m + fetcher: () => this.emojisRepository.find({ where: { host: IsNull() } }).then(emojis => new Map(emojis.map(emoji => [emoji.name, emoji]))), + toRedisConverter: (value) => JSON.stringify(value.values()), + fromRedisConverter: (value) => new Map(JSON.parse(value).map((x: Emoji) => [x.name, x])), // TODO: Date型の変換 + }); } @bindThis @@ -60,7 +71,7 @@ export class CustomEmojiService { }).then(x => this.emojisRepository.findOneByOrFail(x.identifiers[0])); if (data.host == null) { - await this.db.queryResultCache?.remove(['meta_emojis']); + this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiAdded', { emoji: await this.emojiEntityService.packDetailed(emoji.id), @@ -70,6 +81,146 @@ export class CustomEmojiService { return emoji; } + @bindThis + public async update(id: Emoji['id'], data: { + name?: string; + category?: string | null; + aliases?: string[]; + license?: string | null; + }): Promise { + const emoji = await this.emojisRepository.findOneByOrFail({ id: id }); + const sameNameEmoji = await this.emojisRepository.findOneBy({ name: data.name, host: IsNull() }); + if (sameNameEmoji != null && sameNameEmoji.id !== id) throw new Error('name already exists'); + + await this.emojisRepository.update(emoji.id, { + updatedAt: new Date(), + name: data.name, + category: data.category, + aliases: data.aliases, + license: data.license, + }); + + this.localEmojisCache.refresh(); + + const updated = await this.emojiEntityService.packDetailed(emoji.id); + + if (emoji.name === data.name) { + this.globalEventService.publishBroadcastStream('emojiUpdated', { + emojis: [updated], + }); + } else { + this.globalEventService.publishBroadcastStream('emojiDeleted', { + emojis: [await this.emojiEntityService.packDetailed(emoji)], + }); + + this.globalEventService.publishBroadcastStream('emojiAdded', { + emoji: updated, + }); + } + } + + @bindThis + public async addAliasesBulk(ids: Emoji['id'][], aliases: string[]) { + const emojis = await this.emojisRepository.findBy({ + id: In(ids), + }); + + for (const emoji of emojis) { + await this.emojisRepository.update(emoji.id, { + updatedAt: new Date(), + aliases: [...new Set(emoji.aliases.concat(aliases))], + }); + } + + this.localEmojisCache.refresh(); + + this.globalEventService.publishBroadcastStream('emojiUpdated', { + emojis: await this.emojiEntityService.packDetailedMany(ids), + }); + } + + @bindThis + public async setAliasesBulk(ids: Emoji['id'][], aliases: string[]) { + await this.emojisRepository.update({ + id: In(ids), + }, { + updatedAt: new Date(), + aliases: aliases, + }); + + this.localEmojisCache.refresh(); + + this.globalEventService.publishBroadcastStream('emojiUpdated', { + emojis: await this.emojiEntityService.packDetailedMany(ids), + }); + } + + @bindThis + public async removeAliasesBulk(ids: Emoji['id'][], aliases: string[]) { + const emojis = await this.emojisRepository.findBy({ + id: In(ids), + }); + + for (const emoji of emojis) { + await this.emojisRepository.update(emoji.id, { + updatedAt: new Date(), + aliases: emoji.aliases.filter(x => !aliases.includes(x)), + }); + } + + this.localEmojisCache.refresh(); + + this.globalEventService.publishBroadcastStream('emojiUpdated', { + emojis: await this.emojiEntityService.packDetailedMany(ids), + }); + } + + @bindThis + public async setCategoryBulk(ids: Emoji['id'][], category: string | null) { + await this.emojisRepository.update({ + id: In(ids), + }, { + updatedAt: new Date(), + category: category, + }); + + this.localEmojisCache.refresh(); + + this.globalEventService.publishBroadcastStream('emojiUpdated', { + emojis: await this.emojiEntityService.packDetailedMany(ids), + }); + } + + @bindThis + public async delete(id: Emoji['id']) { + const emoji = await this.emojisRepository.findOneByOrFail({ id: id }); + + await this.emojisRepository.delete(emoji.id); + + this.localEmojisCache.refresh(); + + this.globalEventService.publishBroadcastStream('emojiDeleted', { + emojis: [await this.emojiEntityService.packDetailed(emoji)], + }); + } + + @bindThis + public async deleteBulk(ids: Emoji['id'][]) { + const emojis = await this.emojisRepository.findBy({ + id: In(ids), + }); + + for (const emoji of emojis) { + await this.emojisRepository.delete(emoji.id); + } + + this.localEmojisCache.refresh(); + + this.globalEventService.publishBroadcastStream('emojiDeleted', { + emojis: await this.emojiEntityService.packDetailedMany(emojis), + }); + } + @bindThis private normalizeHost(src: string | undefined, noteUserHost: string | null): string | null { // クエリに使うホスト @@ -84,7 +235,7 @@ export class CustomEmojiService { } @bindThis - private parseEmojiStr(emojiName: string, noteUserHost: string | null) { + public parseEmojiStr(emojiName: string, noteUserHost: string | null) { const match = emojiName.match(/^(\w+)(?:@([\w.-]+))?$/); if (!match) return { name: null, host: null }; @@ -143,30 +294,6 @@ export class CustomEmojiService { return res; } - @bindThis - public aggregateNoteEmojis(notes: Note[]) { - let emojis: { name: string | null; host: string | null; }[] = []; - for (const note of notes) { - emojis = emojis.concat(note.emojis - .map(e => this.parseEmojiStr(e, note.userHost))); - if (note.renote) { - emojis = emojis.concat(note.renote.emojis - .map(e => this.parseEmojiStr(e, note.renote!.userHost))); - if (note.renote.user) { - emojis = emojis.concat(note.renote.user.emojis - .map(e => this.parseEmojiStr(e, note.renote!.userHost))); - } - } - const customReactions = Object.keys(note.reactions).map(x => this.reactionService.decodeReaction(x)).filter(x => x.name != null) as typeof emojis; - emojis = emojis.concat(customReactions); - if (note.user) { - emojis = emojis.concat(note.user.emojis - .map(e => this.parseEmojiStr(e, note.userHost))); - } - } - return emojis.filter(x => x.name != null && x.host != null) as { name: string; host: string; }[]; - } - /** * 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します */ diff --git a/packages/backend/src/core/InstanceActorService.ts b/packages/backend/src/core/InstanceActorService.ts index 898fb4ce85..4fb3fc5b4f 100644 --- a/packages/backend/src/core/InstanceActorService.ts +++ b/packages/backend/src/core/InstanceActorService.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull } from 'typeorm'; import type { LocalUser } from '@/models/entities/User.js'; import type { UsersRepository } from '@/models/index.js'; -import { MemoryCache } from '@/misc/cache.js'; +import { MemorySingleCache } from '@/misc/cache.js'; import { DI } from '@/di-symbols.js'; import { CreateSystemUserService } from '@/core/CreateSystemUserService.js'; import { bindThis } from '@/decorators.js'; @@ -11,7 +11,7 @@ const ACTOR_USERNAME = 'instance.actor' as const; @Injectable() export class InstanceActorService { - private cache: MemoryCache; + private cache: MemorySingleCache; constructor( @Inject(DI.usersRepository) @@ -19,7 +19,7 @@ export class InstanceActorService { private createSystemUserService: CreateSystemUserService, ) { - this.cache = new MemoryCache(Infinity); + this.cache = new MemorySingleCache(Infinity); } @bindThis diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index fcc17ace1e..5c4d13f178 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -20,7 +20,7 @@ import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js import { checkWordMute } from '@/misc/check-word-mute.js'; import type { Channel } from '@/models/entities/Channel.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; -import { MemoryCache } from '@/misc/cache.js'; +import { MemorySingleCache } from '@/misc/cache.js'; import type { UserProfile } from '@/models/entities/UserProfile.js'; import { RelayService } from '@/core/RelayService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; @@ -47,7 +47,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; -const mutedWordsCache = new MemoryCache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5); +const mutedWordsCache = new MemorySingleCache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5); type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 97a0b5ee66..a274b19e4b 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -1,7 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { IsNull } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { EmojisRepository, BlockingsRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/index.js'; +import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/index.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { RemoteUser, User } from '@/models/entities/User.js'; import type { Note } from '@/models/entities/Note.js'; @@ -20,6 +19,7 @@ import { MetaService } from '@/core/MetaService.js'; import { bindThis } from '@/decorators.js'; import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; const FALLBACK = '❤'; @@ -60,9 +60,6 @@ export class ReactionService { @Inject(DI.usersRepository) private usersRepository: UsersRepository, - @Inject(DI.blockingsRepository) - private blockingsRepository: BlockingsRepository, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -74,6 +71,7 @@ export class ReactionService { private utilityService: UtilityService, private metaService: MetaService, + private customEmojiService: CustomEmojiService, private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private userBlockingService: UserBlockingService, @@ -104,7 +102,6 @@ export class ReactionService { if (note.reactionAcceptance === 'likeOnly' || ((note.reactionAcceptance === 'likeOnlyForRemote') && (user.host != null))) { reaction = '❤️'; } else { - // TODO: cache reaction = await this.toDbReaction(reaction, user.host); } @@ -158,21 +155,22 @@ export class ReactionService { // カスタム絵文字リアクションだったら絵文字情報も送る const decodedReaction = this.decodeReaction(reaction); - // TODO: Cache - const emoji = await this.emojisRepository.findOne({ - where: { - name: decodedReaction.name, - host: decodedReaction.host ?? IsNull(), - }, - select: ['name', 'host', 'originalUrl', 'publicUrl'], - }); + const customEmoji = decodedReaction.name == null ? null : decodedReaction.host == null + ? (await this.customEmojiService.localEmojisCache.fetch()).get(decodedReaction.name) + : await this.emojisRepository.findOne( + { + where: { + name: decodedReaction.name, + host: decodedReaction.host, + }, + }); this.globalEventService.publishNoteStream(note.id, 'reacted', { reaction: decodedReaction.reaction, - emoji: emoji != null ? { - name: emoji.host ? `${emoji.name}@${emoji.host}` : `${emoji.name}@.`, + emoji: customEmoji != null ? { + name: customEmoji.host ? `${customEmoji.name}@${customEmoji.host}` : `${customEmoji.name}@.`, // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） - url: emoji.publicUrl || emoji.originalUrl, + url: customEmoji.publicUrl || customEmoji.originalUrl, } : null, userId: user.id, }); @@ -311,10 +309,12 @@ export class ReactionService { const custom = reaction.match(/^:([\w+-]+)(?:@\.)?:$/); if (custom) { const name = custom[1]; - const emoji = await this.emojisRepository.findOneBy({ - host: reacterHost ?? IsNull(), - name, - }); + const emoji = reacterHost == null + ? (await this.customEmojiService.localEmojisCache.fetch()).get(name) + : await this.emojisRepository.findOneBy({ + host: reacterHost, + name, + }); if (emoji) return reacterHost ? `:${name}@${reacterHost}:` : `:${name}:`; } diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index 4df7fb3bff..9d34d82be2 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -3,7 +3,7 @@ import { IsNull } from 'typeorm'; import type { LocalUser, User } from '@/models/entities/User.js'; import type { RelaysRepository, UsersRepository } from '@/models/index.js'; import { IdService } from '@/core/IdService.js'; -import { MemoryCache } from '@/misc/cache.js'; +import { MemorySingleCache } from '@/misc/cache.js'; import type { Relay } from '@/models/entities/Relay.js'; import { QueueService } from '@/core/QueueService.js'; import { CreateSystemUserService } from '@/core/CreateSystemUserService.js'; @@ -16,7 +16,7 @@ const ACTOR_USERNAME = 'relay.actor' as const; @Injectable() export class RelayService { - private relaysCache: MemoryCache; + private relaysCache: MemorySingleCache; constructor( @Inject(DI.usersRepository) @@ -30,7 +30,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new MemoryCache(1000 * 60 * 10); + this.relaysCache = new MemorySingleCache(1000 * 60 * 10); } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 52e6292a1e..54e098ea52 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import Redis from 'ioredis'; import { In } from 'typeorm'; import type { Role, RoleAssignment, RoleAssignmentsRepository, RolesRepository, UsersRepository } from '@/models/index.js'; -import { MemoryKVCache, MemoryCache } from '@/misc/cache.js'; +import { MemoryKVCache, MemorySingleCache } from '@/misc/cache.js'; import type { User } from '@/models/entities/User.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @@ -57,7 +57,7 @@ export const DEFAULT_POLICIES: RolePolicies = { @Injectable() export class RoleService implements OnApplicationShutdown { - private rolesCache: MemoryCache; + private rolesCache: MemorySingleCache; private roleAssignmentByUserIdCache: MemoryKVCache; public static AlreadyAssignedError = class extends Error {}; @@ -84,7 +84,7 @@ export class RoleService implements OnApplicationShutdown { ) { //this.onMessage = this.onMessage.bind(this); - this.rolesCache = new MemoryCache(Infinity); + this.rolesCache = new MemorySingleCache(Infinity); this.roleAssignmentByUserIdCache = new MemoryKVCache(Infinity); this.redisSubscriber.on('message', this.onMessage); diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 15512c8f47..b250b796d6 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -21,6 +21,8 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j import type { UserKeypair } from '@/models/entities/UserKeypair.js'; import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFilesRepository, EmojisRepository, PollsRepository } from '@/models/index.js'; import { bindThis } from '@/decorators.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { LdSignatureService } from './LdSignatureService.js'; import { ApMfmService } from './ApMfmService.js'; import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @@ -50,6 +52,7 @@ export class ApRendererService { @Inject(DI.pollsRepository) private pollsRepository: PollsRepository, + private customEmojiService: CustomEmojiService, private userEntityService: UserEntityService, private driveFileEntityService: DriveFileEntityService, private ldSignatureService: LdSignatureService, @@ -272,11 +275,7 @@ export class ApRendererService { if (reaction.startsWith(':')) { const name = reaction.replaceAll(':', ''); - // TODO: cache - const emoji = await this.emojisRepository.findOneBy({ - name, - host: IsNull(), - }); + const emoji = (await this.customEmojiService.localEmojisCache.fetch()).get(name); if (emoji) object.tag = [this.renderEmoji(emoji)]; } @@ -701,13 +700,9 @@ export class ApRendererService { private async getEmojis(names: string[]): Promise { if (names == null || names.length === 0) return []; - const emojis = await Promise.all( - names.map(name => this.emojisRepository.findOneBy({ - name, - host: IsNull(), - })), - ); + const allEmojis = await this.customEmojiService.localEmojisCache.fetch(); + const emojis = names.map(name => allEmojis.get(name)).filter(isNotNull); - return emojis.filter(emoji => emoji != null) as Emoji[]; + return emojis; } } diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 5660600692..94b3029c58 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -406,7 +406,7 @@ export class NoteEntityService implements OnModuleInit { } } - await this.customEmojiService.prefetchEmojis(this.customEmojiService.aggregateNoteEmojis(notes)); + await this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes)); // TODO: 本当は renote とか reply がないのに renoteId とか replyId があったらここで解決しておく const fileIds = notes.map(n => [n.fileIds, n.renote?.fileIds, n.reply?.fileIds]).flat(2).filter(isNotNull); const packedFiles = await this.driveFileEntityService.packManyByIdsMap(fileIds); @@ -420,6 +420,30 @@ export class NoteEntityService implements OnModuleInit { }))); } + @bindThis + public aggregateNoteEmojis(notes: Note[]) { + let emojis: { name: string | null; host: string | null; }[] = []; + for (const note of notes) { + emojis = emojis.concat(note.emojis + .map(e => this.customEmojiService.parseEmojiStr(e, note.userHost))); + if (note.renote) { + emojis = emojis.concat(note.renote.emojis + .map(e => this.customEmojiService.parseEmojiStr(e, note.renote!.userHost))); + if (note.renote.user) { + emojis = emojis.concat(note.renote.user.emojis + .map(e => this.customEmojiService.parseEmojiStr(e, note.renote!.userHost))); + } + } + const customReactions = Object.keys(note.reactions).map(x => this.reactionService.decodeReaction(x)).filter(x => x.name != null) as typeof emojis; + emojis = emojis.concat(customReactions); + if (note.user) { + emojis = emojis.concat(note.user.emojis + .map(e => this.customEmojiService.parseEmojiStr(e, note.userHost))); + } + } + return emojis.filter(x => x.name != null && x.host != null) as { name: string; host: string; }[]; + } + @bindThis public async countSameRenotes(userId: string, renoteId: string, excludeNoteId: string | undefined): Promise { // 指定したユーザーの指定したノートのリノートがいくつあるか数える diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index ef6f610125..d35414acf7 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -85,6 +85,90 @@ export class RedisKVCache { } } +export class RedisSingleCache { + private redisClient: Redis.Redis; + private name: string; + private lifetime: number; + private memoryCache: MemorySingleCache; + private fetcher: () => Promise; + private toRedisConverter: (value: T) => string; + private fromRedisConverter: (value: string) => T; + + constructor(redisClient: RedisSingleCache['redisClient'], name: RedisSingleCache['name'], opts: { + lifetime: RedisSingleCache['lifetime']; + memoryCacheLifetime: number; + fetcher: RedisSingleCache['fetcher']; + toRedisConverter: RedisSingleCache['toRedisConverter']; + fromRedisConverter: RedisSingleCache['fromRedisConverter']; + }) { + this.redisClient = redisClient; + this.name = name; + this.lifetime = opts.lifetime; + this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); + this.fetcher = opts.fetcher; + this.toRedisConverter = opts.toRedisConverter; + this.fromRedisConverter = opts.fromRedisConverter; + } + + @bindThis + public async set(value: T): Promise { + this.memoryCache.set(value); + if (this.lifetime === Infinity) { + await this.redisClient.set( + `singlecache:${this.name}`, + this.toRedisConverter(value), + ); + } else { + await this.redisClient.set( + `singlecache:${this.name}`, + this.toRedisConverter(value), + 'ex', Math.round(this.lifetime / 1000), + ); + } + } + + @bindThis + public async get(): Promise { + const memoryCached = this.memoryCache.get(); + if (memoryCached !== undefined) return memoryCached; + + const cached = await this.redisClient.get(`singlecache:${this.name}`); + if (cached == null) return undefined; + return this.fromRedisConverter(cached); + } + + @bindThis + public async delete(): Promise { + this.memoryCache.delete(); + await this.redisClient.del(`singlecache:${this.name}`); + } + + /** + * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します + */ + @bindThis + public async fetch(): Promise { + const cachedValue = await this.get(); + if (cachedValue !== undefined) { + // Cache HIT + return cachedValue; + } + + // Cache MISS + const value = await this.fetcher(); + this.set(value); + return value; + } + + @bindThis + public async refresh() { + const value = await this.fetcher(); + this.set(value); + + // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする + } +} + // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ export class MemoryKVCache { @@ -173,12 +257,12 @@ export class MemoryKVCache { } } -export class MemoryCache { +export class MemorySingleCache { private cachedAt: number | null = null; private value: T | undefined; private lifetime: number; - constructor(lifetime: MemoryCache['lifetime']) { + constructor(lifetime: MemorySingleCache['lifetime']) { this.lifetime = lifetime; } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index a9af22ad09..0e99b7bcd2 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -7,7 +7,7 @@ import { MetaService } from '@/core/MetaService.js'; import { ApRequestService } from '@/core/activitypub/ApRequestService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; -import { MemoryCache } from '@/misc/cache.js'; +import { MemorySingleCache } from '@/misc/cache.js'; import type { Instance } from '@/models/entities/Instance.js'; import InstanceChart from '@/core/chart/charts/instance.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; @@ -22,7 +22,7 @@ import type { DeliverJobData } from '../types.js'; @Injectable() export class DeliverProcessorService { private logger: Logger; - private suspendedHostsCache: MemoryCache; + private suspendedHostsCache: MemorySingleCache; private latest: string | null; constructor( @@ -46,7 +46,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemoryCache(1000 * 60 * 60); + this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); } @bindThis diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 66c1faaac2..666a91fcee 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -4,7 +4,7 @@ import type { NotesRepository, UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; import { MAX_NOTE_TEXT_LENGTH } from '@/const.js'; -import { MemoryCache } from '@/misc/cache.js'; +import { MemorySingleCache } from '@/misc/cache.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import NotesChart from '@/core/chart/charts/notes.js'; @@ -118,7 +118,7 @@ export class NodeinfoServerService { }; }; - const cache = new MemoryCache>>(1000 * 60 * 10); + const cache = new MemorySingleCache>>(1000 * 60 * 10); fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(() => nodeinfo2()); diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts index 4e4f845b0b..6e604ed885 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts @@ -1,10 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource, In } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { EmojisRepository } from '@/models/index.js'; -import { DI } from '@/di-symbols.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; export const meta = { tags: ['admin'], @@ -26,38 +22,14 @@ export const paramDef = { required: ['ids', 'aliases'], } as const; -// TODO: ロジックをサービスに切り出す - // eslint-disable-next-line import/no-default-export @Injectable() export default class extends Endpoint { constructor( - @Inject(DI.db) - private db: DataSource, - - @Inject(DI.emojisRepository) - private emojisRepository: EmojisRepository, - - private emojiEntityService: EmojiEntityService, - private globalEventService: GlobalEventService, + private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - const emojis = await this.emojisRepository.findBy({ - id: In(ps.ids), - }); - - for (const emoji of emojis) { - await this.emojisRepository.update(emoji.id, { - updatedAt: new Date(), - aliases: [...new Set(emoji.aliases.concat(ps.aliases))], - }); - } - - await this.db.queryResultCache?.remove(['meta_emojis']); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ps.ids), - }); + await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index fea11a67d6..82dca9cc70 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -90,8 +90,6 @@ export default class extends Endpoint { license: emoji.license, }).then(x => this.emojisRepository.findOneByOrFail(x.identifiers[0])); - await this.db.queryResultCache?.remove(['meta_emojis']); - this.globalEventService.publishBroadcastStream('emojiAdded', { emoji: await this.emojiEntityService.packDetailed(copied.id), }); diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/delete-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/delete-bulk.ts index 84aad020af..d5acee36a8 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/delete-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/delete-bulk.ts @@ -1,11 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource, In } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { EmojisRepository } from '@/models/index.js'; -import { DI } from '@/di-symbols.js'; -import { ModerationLogService } from '@/core/ModerationLogService.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService'; export const meta = { tags: ['admin'], @@ -24,38 +19,14 @@ export const paramDef = { required: ['ids'], } as const; -// TODO: ロジックをサービスに切り出す - // eslint-disable-next-line import/no-default-export @Injectable() export default class extends Endpoint { constructor( - @Inject(DI.db) - private db: DataSource, - - @Inject(DI.emojisRepository) - private emojisRepository: EmojisRepository, - - private moderationLogService: ModerationLogService, - private emojiEntityService: EmojiEntityService, - private globalEventService: GlobalEventService, + private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - const emojis = await this.emojisRepository.findBy({ - id: In(ps.ids), - }); - - for (const emoji of emojis) { - await this.emojisRepository.delete(emoji.id); - await this.db.queryResultCache?.remove(['meta_emojis']); - this.moderationLogService.insertModerationLog(me, 'deleteEmoji', { - emoji: emoji, - }); - } - - this.globalEventService.publishBroadcastStream('emojiDeleted', { - emojis: await this.emojiEntityService.packDetailedMany(emojis), - }); + await this.customEmojiService.deleteBulk(ps.ids); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/delete.ts b/packages/backend/src/server/api/endpoints/admin/emoji/delete.ts index 90a5856a1b..429c819fe0 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/delete.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/delete.ts @@ -1,12 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { EmojisRepository } from '@/models/index.js'; -import { DI } from '@/di-symbols.js'; -import { ModerationLogService } from '@/core/ModerationLogService.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; -import { ApiError } from '../../../error.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; export const meta = { tags: ['admin'], @@ -31,38 +25,14 @@ export const paramDef = { required: ['id'], } as const; -// TODO: ロジックをサービスに切り出す - // eslint-disable-next-line import/no-default-export @Injectable() export default class extends Endpoint { constructor( - @Inject(DI.db) - private db: DataSource, - - @Inject(DI.emojisRepository) - private emojisRepository: EmojisRepository, - - private moderationLogService: ModerationLogService, - private emojiEntityService: EmojiEntityService, - private globalEventService: GlobalEventService, + private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - const emoji = await this.emojisRepository.findOneBy({ id: ps.id }); - - if (emoji == null) throw new ApiError(meta.errors.noSuchEmoji); - - await this.emojisRepository.delete(emoji.id); - - await this.db.queryResultCache?.remove(['meta_emojis']); - - this.globalEventService.publishBroadcastStream('emojiDeleted', { - emojis: [await this.emojiEntityService.packDetailed(emoji)], - }); - - this.moderationLogService.insertModerationLog(me, 'deleteEmoji', { - emoji: emoji, - }); + await this.customEmojiService.delete(ps.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts index 3935183502..83f882cac5 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts @@ -1,10 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource, In } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { EmojisRepository } from '@/models/index.js'; -import { DI } from '@/di-symbols.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; export const meta = { tags: ['admin'], @@ -26,38 +22,14 @@ export const paramDef = { required: ['ids', 'aliases'], } as const; -// TODO: ロジックをサービスに切り出す - // eslint-disable-next-line import/no-default-export @Injectable() export default class extends Endpoint { constructor( - @Inject(DI.db) - private db: DataSource, - - @Inject(DI.emojisRepository) - private emojisRepository: EmojisRepository, - - private emojiEntityService: EmojiEntityService, - private globalEventService: GlobalEventService, + private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - const emojis = await this.emojisRepository.findBy({ - id: In(ps.ids), - }); - - for (const emoji of emojis) { - await this.emojisRepository.update(emoji.id, { - updatedAt: new Date(), - aliases: emoji.aliases.filter(x => !ps.aliases.includes(x)), - }); - } - - await this.db.queryResultCache?.remove(['meta_emojis']); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ps.ids), - }); + await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts index 6a875f9c83..1d3a432bb7 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts @@ -1,10 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource, In } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { EmojisRepository } from '@/models/index.js'; -import { DI } from '@/di-symbols.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; export const meta = { tags: ['admin'], @@ -26,34 +22,14 @@ export const paramDef = { required: ['ids', 'aliases'], } as const; -// TODO: ロジックをサービスに切り出す - // eslint-disable-next-line import/no-default-export @Injectable() export default class extends Endpoint { constructor( - @Inject(DI.db) - private db: DataSource, - - @Inject(DI.emojisRepository) - private emojisRepository: EmojisRepository, - - private emojiEntityService: EmojiEntityService, - private globalEventService: GlobalEventService, + private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.emojisRepository.update({ - id: In(ps.ids), - }, { - updatedAt: new Date(), - aliases: ps.aliases, - }); - - await this.db.queryResultCache?.remove(['meta_emojis']); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ps.ids), - }); + await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts index d3b999c0ed..453968c7a9 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts @@ -1,10 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource, In } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { EmojisRepository } from '@/models/index.js'; -import { DI } from '@/di-symbols.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; export const meta = { tags: ['admin'], @@ -28,34 +24,14 @@ export const paramDef = { required: ['ids'], } as const; -// TODO: ロジックをサービスに切り出す - // eslint-disable-next-line import/no-default-export @Injectable() export default class extends Endpoint { constructor( - @Inject(DI.db) - private db: DataSource, - - @Inject(DI.emojisRepository) - private emojisRepository: EmojisRepository, - - private emojiEntityService: EmojiEntityService, - private globalEventService: GlobalEventService, + private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.emojisRepository.update({ - id: In(ps.ids), - }, { - updatedAt: new Date(), - category: ps.category, - }); - - await this.db.queryResultCache?.remove(['meta_emojis']); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ps.ids), - }); + await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts index bc0475e05c..f63348b60b 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts @@ -1,10 +1,6 @@ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource, IsNull } from 'typeorm'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { EmojisRepository } from '@/models/index.js'; -import { DI } from '@/di-symbols.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -45,51 +41,19 @@ export const paramDef = { required: ['id', 'name', 'aliases'], } as const; -// TODO: ロジックをサービスに切り出す - // eslint-disable-next-line import/no-default-export @Injectable() export default class extends Endpoint { constructor( - @Inject(DI.db) - private db: DataSource, - - @Inject(DI.emojisRepository) - private emojisRepository: EmojisRepository, - - private emojiEntityService: EmojiEntityService, - private globalEventService: GlobalEventService, + private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - const emoji = await this.emojisRepository.findOneBy({ id: ps.id }); - const sameNameEmoji = await this.emojisRepository.findOneBy({ name: ps.name, host: IsNull() }); - if (emoji == null) throw new ApiError(meta.errors.noSuchEmoji); - if (sameNameEmoji != null && sameNameEmoji.id !== ps.id) throw new ApiError(meta.errors.sameNameEmojiExists); - await this.emojisRepository.update(emoji.id, { - updatedAt: new Date(), + await this.customEmojiService.update(ps.id, { name: ps.name, - category: ps.category, + category: ps.category ?? null, aliases: ps.aliases, - license: ps.license, + license: ps.license ?? null, }); - - await this.db.queryResultCache?.remove(['meta_emojis']); - - const updated = await this.emojiEntityService.packDetailed(emoji.id); - - if (emoji.name === ps.name) { - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: [updated], - }); - } else { - this.globalEventService.publishBroadcastStream('emojiDeleted', { - emojis: [await this.emojiEntityService.packDetailed(emoji)], - }); - - this.globalEventService.publishBroadcastStream('emojiAdded', { - emoji: updated, - }); - } }); } } diff --git a/packages/backend/src/server/api/endpoints/emojis.ts b/packages/backend/src/server/api/endpoints/emojis.ts index 0711fe4a57..13cc709d31 100644 --- a/packages/backend/src/server/api/endpoints/emojis.ts +++ b/packages/backend/src/server/api/endpoints/emojis.ts @@ -58,10 +58,6 @@ export default class extends Endpoint { category: 'ASC', name: 'ASC', }, - cache: { - id: 'meta_emojis', - milliseconds: 3600000, // 1 hour - }, }); return {