From 4f9f625e6574990dfcd736c7a7d059af8fef7234 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 3 Apr 2023 11:49:58 +0900 Subject: [PATCH] perf(backend): cache timeline of a channel to redis --- packages/backend/src/core/IdService.ts | 15 +++++++++- .../backend/src/core/NoteCreateService.ts | 8 +++++ packages/backend/src/misc/id/aid.ts | 5 ++++ .../server/api/endpoints/channels/timeline.ts | 29 +++++++++++++++++-- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/IdService.ts b/packages/backend/src/core/IdService.ts index 31c0819e5..94084ad84 100644 --- a/packages/backend/src/core/IdService.ts +++ b/packages/backend/src/core/IdService.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { ulid } from 'ulid'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import { genAid } from '@/misc/id/aid.js'; +import { genAid, parseAid } from '@/misc/id/aid.js'; import { genMeid } from '@/misc/id/meid.js'; import { genMeidg } from '@/misc/id/meidg.js'; import { genObjectId } from '@/misc/id/object-id.js'; @@ -32,4 +32,17 @@ export class IdService { default: throw new Error('unrecognized id generation method'); } } + + @bindThis + public parse(id: string): { date: Date; } { + switch (this.method) { + case 'aid': return parseAid(id); + // TODO + //case 'meid': + //case 'meidg': + //case 'ulid': + //case 'objectid': + default: throw new Error('unrecognized id generation method'); + } + } } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 7d0805376..93fab9d17 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -1,6 +1,7 @@ import { setImmediate } from 'node:timers/promises'; import * as mfm from 'mfm-js'; import { In, DataSource } from 'typeorm'; +import Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { extractMentions } from '@/misc/extract-mentions.js'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js'; @@ -150,6 +151,9 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, + @Inject(DI.redis) + private redisClient: Redis.Redis, + @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -321,6 +325,10 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); + if (data.channel) { + this.redisClient.xadd(`channelTimeline:${data.channel.id}`, 'MAXLEN', '~', '1000', `${this.idService.parse(note.id).date.getTime()}-*`, 'note', note.id); + } + setImmediate('post created', { signal: this.#shutdownController.signal }).then( () => this.postNoteCreated(note, user, data, silent, tags!, mentionedUsers!), () => { /* aborted, ignore this */ }, diff --git a/packages/backend/src/misc/id/aid.ts b/packages/backend/src/misc/id/aid.ts index 19c8546f9..93a9929aa 100644 --- a/packages/backend/src/misc/id/aid.ts +++ b/packages/backend/src/misc/id/aid.ts @@ -23,3 +23,8 @@ export function genAid(date: Date): string { counter++; return getTime(t) + getNoise(); } + +export function parseAid(id: string): { date: Date; } { + const time = parseInt(id.slice(0, 8), 36) + TIME2000; + return { date: new Date(time) }; +} diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index cdaa40013..eef343d13 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -1,10 +1,12 @@ import { Inject, Injectable } from '@nestjs/common'; +import Redis from 'ioredis'; import { Endpoint } from '@/server/api/endpoint-base.js'; import type { ChannelsRepository, NotesRepository } from '@/models/index.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; +import { IdService } from '@/core/IdService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -48,12 +50,16 @@ export const paramDef = { @Injectable() export default class extends Endpoint { constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, + @Inject(DI.notesRepository) private notesRepository: NotesRepository, @Inject(DI.channelsRepository) private channelsRepository: ChannelsRepository, + private idService: IdService, private noteEntityService: NoteEntityService, private queryService: QueryService, private activeUsersChart: ActiveUsersChart, @@ -67,9 +73,25 @@ export default class extends Endpoint { throw new ApiError(meta.errors.noSuchChannel); } + const noteIdsRes = await this.redisClient.xrevrange( + `channelTimeline:${channel.id}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : '+', + '-', + 'COUNT', ps.limit + 1); // untilIdに指定したものも含まれるため+1 + + if (noteIdsRes.length === 0) { + return []; + } + + const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId); + + if (noteIds.length === 0) { + return []; + } + //#region Construct query - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.channelId = :channelId', { channelId: channel.id }) + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('user.avatar', 'avatar') .leftJoinAndSelect('user.banner', 'banner') @@ -90,7 +112,8 @@ export default class extends Endpoint { } //#endregion - const timeline = await query.take(ps.limit).getMany(); + const timeline = await query.getMany(); + timeline.sort((a, b) => a.id > b.id ? -1 : 1); if (me) this.activeUsersChart.read(me);