From dab205edb87ea9cdaee5b6564aa11dfcea245d7b Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Oct 2023 14:24:25 +0900 Subject: [PATCH] enhance(backend): improve featured system --- CHANGELOG.md | 3 +- .../migration/1696569742153-clean-up.js | 18 +++ packages/backend/src/core/CoreModule.ts | 6 + packages/backend/src/core/FeaturedService.ts | 126 ++++++++++++++++++ .../backend/src/core/NoteCreateService.ts | 12 +- .../backend/src/core/NoteDeleteService.ts | 1 - packages/backend/src/core/ReactionService.ts | 18 ++- packages/backend/src/models/Note.ts | 5 - packages/backend/src/models/NoteReaction.ts | 1 - .../server/api/endpoints/notes/featured.ts | 51 ++++--- 10 files changed, 208 insertions(+), 33 deletions(-) create mode 100644 packages/backend/migration/1696569742153-clean-up.js create mode 100644 packages/backend/src/core/FeaturedService.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2780ebb86a..c95ad4dd65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,8 @@ - Fix: リアクションしたユーザ一覧のUIが稀に左上に残ってしまう不具合を修正 ### Server -- Enhance: タイムライン取得時のパフォーマンスを改善 +- Enhance: タイムライン取得時のパフォーマンスを大幅に向上 +- Enhance: ハイライト取得時のパフォーマンスを大幅に向上 ## 2023.9.3 ### General diff --git a/packages/backend/migration/1696569742153-clean-up.js b/packages/backend/migration/1696569742153-clean-up.js new file mode 100644 index 0000000000..de48fab5aa --- /dev/null +++ b/packages/backend/migration/1696569742153-clean-up.js @@ -0,0 +1,18 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class CleanUp1696569742153 { + name = 'CleanUp1696569742153' + + async up(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_01f4581f114e0ebd2bbb876f0b"`); + await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "score"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" ADD "score" integer NOT NULL DEFAULT '0'`); + await queryRunner.query(`CREATE INDEX "IDX_01f4581f114e0ebd2bbb876f0b" ON "note_reaction" ("createdAt") `); + } +} diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index cd66d1a81c..1984d9e6c2 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -60,6 +60,7 @@ import { UtilityService } from './UtilityService.js'; import { FileInfoService } from './FileInfoService.js'; import { SearchService } from './SearchService.js'; import { ClipService } from './ClipService.js'; +import { FeaturedService } from './FeaturedService.js'; import { ChartLoggerService } from './chart/ChartLoggerService.js'; import FederationChart from './chart/charts/federation.js'; import NotesChart from './chart/charts/notes.js'; @@ -187,6 +188,7 @@ const $UtilityService: Provider = { provide: 'UtilityService', useExisting: Util const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService }; const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService }; const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService }; +const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -318,6 +320,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting FileInfoService, SearchService, ClipService, + FeaturedService, ChartLoggerService, FederationChart, NotesChart, @@ -442,6 +445,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $FileInfoService, $SearchService, $ClipService, + $FeaturedService, $ChartLoggerService, $FederationChart, $NotesChart, @@ -567,6 +571,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting FileInfoService, SearchService, ClipService, + FeaturedService, FederationChart, NotesChart, UsersChart, @@ -690,6 +695,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $FileInfoService, $SearchService, $ClipService, + $FeaturedService, $FederationChart, $NotesChart, $UsersChart, diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts new file mode 100644 index 0000000000..89b86b2e30 --- /dev/null +++ b/packages/backend/src/core/FeaturedService.ts @@ -0,0 +1,126 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import type { MiNote } from '@/models/_.js'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; + +@Injectable() +export class FeaturedService { + constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, + ) { + } + + @bindThis + private getCurrentPerUserFriendRankingWindow(): number { + const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime(); + return Math.floor(passed / (1000 * 60 * 60 * 24 * 7)); // 1週間ごと + } + + @bindThis + private getCurrentGlobalNotesRankingWindow(): number { + const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime(); + return Math.floor(passed / (1000 * 60 * 60 * 24 * 3)); // 3日ごと + } + + @bindThis + public async updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise { + // TODO: フォロワー数の多い人が常にランキング上位になるのを防ぎたい + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const redisTransaction = this.redisClient.multi(); + redisTransaction.zincrby( + `featuredGlobalNotesRanking:${currentWindow}`, + score.toString(), + noteId); + redisTransaction.expire( + `featuredGlobalNotesRanking:${currentWindow}`, + 60 * 60 * 24 * 9, // 9日間保持 + 'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 + await redisTransaction.exec(); + } + + @bindThis + public async updateInChannelNotesRanking(noteId: MiNote['id'], channelId: MiNote['channelId'], score = 1): Promise { + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const redisTransaction = this.redisClient.multi(); + redisTransaction.zincrby( + `featuredInChannelNotesRanking:${channelId}:${currentWindow}`, + score.toString(), + noteId); + redisTransaction.expire( + `featuredInChannelNotesRanking:${channelId}:${currentWindow}`, + 60 * 60 * 24 * 9, // 9日間保持 + 'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 + await redisTransaction.exec(); + } + + @bindThis + public async getGlobalNotesRanking(limit: number): Promise { + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const previousWindow = currentWindow - 1; + + const [currentRankingResult, previousRankingResult] = await Promise.all([ + this.redisClient.zrange( + `featuredGlobalNotesRanking:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'), + this.redisClient.zrange( + `featuredGlobalNotesRanking:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'), + ]); + + const ranking = new Map(); + for (let i = 0; i < currentRankingResult.length; i += 2) { + const noteId = currentRankingResult[i]; + const score = parseInt(currentRankingResult[i + 1], 10); + ranking.set(noteId, score); + } + for (let i = 0; i < previousRankingResult.length; i += 2) { + const noteId = previousRankingResult[i]; + const score = parseInt(previousRankingResult[i + 1], 10); + const exist = ranking.get(noteId); + if (exist != null) { + ranking.set(noteId, (exist + score) / 2); + } else { + ranking.set(noteId, score); + } + } + + return Array.from(ranking.keys()); + } + + @bindThis + public async getInChannelNotesRanking(channelId: MiNote['channelId'], limit: number): Promise { + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const previousWindow = currentWindow - 1; + + const [currentRankingResult, previousRankingResult] = await Promise.all([ + this.redisClient.zrange( + `featuredInChannelNotesRanking:${channelId}:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'), + this.redisClient.zrange( + `featuredInChannelNotesRanking:${channelId}:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'), + ]); + + const ranking = new Map(); + for (let i = 0; i < currentRankingResult.length; i += 2) { + const noteId = currentRankingResult[i]; + const score = parseInt(currentRankingResult[i + 1], 10); + ranking.set(noteId, score); + } + for (let i = 0; i < previousRankingResult.length; i += 2) { + const noteId = previousRankingResult[i]; + const score = parseInt(previousRankingResult[i + 1], 10); + const exist = ranking.get(noteId); + if (exist != null) { + ranking.set(noteId, (exist + score) / 2); + } else { + ranking.set(noteId, score); + } + } + + return Array.from(ranking.keys()); + } +} diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 34d103df77..ca9dbfa642 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -53,6 +53,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -200,6 +201,7 @@ export class NoteCreateService implements OnApplicationShutdown { private hashtagService: HashtagService, private antennaService: AntennaService, private webhookService: WebhookService, + private featuredService: FeaturedService, private remoteUserResolveService: RemoteUserResolveService, private apDeliverManagerService: ApDeliverManagerService, private apRendererService: ApRendererService, @@ -721,10 +723,18 @@ export class NoteCreateService implements OnApplicationShutdown { this.notesRepository.createQueryBuilder().update() .set({ renoteCount: () => '"renoteCount" + 1', - score: () => '"score" + 1', }) .where('id = :id', { id: renote.id }) .execute(); + + // 30%の確率でハイライト用ランキング更新 + if (Math.random() < 0.3) { + if (renote.channelId != null) { + this.featuredService.updateInChannelNotesRanking(renote.id, renote.channelId, 1); + } else if (renote.visibility === 'public' && renote.userHost == null) { + this.featuredService.updateGlobalNotesRanking(renote.id, 1); + } + } } @bindThis diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 87979f22ac..3d443b4a06 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -67,7 +67,6 @@ export class NoteDeleteService { // この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき if (note.renoteId && (await this.noteEntityService.countSameRenotes(user.id, note.renoteId, note.id)) === 0) { this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1); - if (!user.isBot) this.notesRepository.decrement({ id: note.renoteId }, 'score', 1); } if (note.replyId) { diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 25464b19a8..298a62ffd9 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -4,6 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; @@ -26,6 +27,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { RoleService } from '@/core/RoleService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; const FALLBACK = '❤'; @@ -66,6 +68,9 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/; @Injectable() export class ReactionService { constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, + @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -86,6 +91,7 @@ export class ReactionService { private noteEntityService: NoteEntityService, private userBlockingService: UserBlockingService, private idService: IdService, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, private apRendererService: ApRendererService, private apDeliverManagerService: ApDeliverManagerService, @@ -182,11 +188,19 @@ export class ReactionService { await this.notesRepository.createQueryBuilder().update() .set({ reactions: () => sql, - ... (!user.isBot ? { score: () => '"score" + 1' } : {}), }) .where('id = :id', { id: note.id }) .execute(); + // 30%の確率でハイライト用ランキング更新 + if (Math.random() < 0.3) { + if (note.channelId != null) { + this.featuredService.updateInChannelNotesRanking(note.id, note.channelId, 1); + } else if (note.visibility === 'public' && note.userHost == null) { + this.featuredService.updateGlobalNotesRanking(note.id, 1); + } + } + const meta = await this.metaService.fetch(); if (meta.enableChartsForRemoteUser || (user.host == null)) { @@ -275,8 +289,6 @@ export class ReactionService { .where('id = :id', { id: note.id }) .execute(); - if (!user.isBot) this.notesRepository.decrement({ id: note.id }, 'score', 1); - this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, userId: user.id, diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index 0d2422c4f3..3e2adf4d82 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -138,11 +138,6 @@ export class MiNote { }) public url: string | null; - @Column('integer', { - default: 0, select: false, - }) - public score: number; - @Index() @Column({ ...id(), diff --git a/packages/backend/src/models/NoteReaction.ts b/packages/backend/src/models/NoteReaction.ts index 7c08d31c6d..43323f8a43 100644 --- a/packages/backend/src/models/NoteReaction.ts +++ b/packages/backend/src/models/NoteReaction.ts @@ -14,7 +14,6 @@ export class MiNoteReaction { @PrimaryColumn(id()) public id: string; - @Index() @Column('timestamp with time zone', { comment: 'The created date of the NoteReaction.', }) diff --git a/packages/backend/src/server/api/endpoints/notes/featured.ts b/packages/backend/src/server/api/endpoints/notes/featured.ts index 5283b0e0bc..bf4ad1deb6 100644 --- a/packages/backend/src/server/api/endpoints/notes/featured.ts +++ b/packages/backend/src/server/api/endpoints/notes/featured.ts @@ -6,9 +6,9 @@ import { Inject, Injectable } from '@nestjs/common'; import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; export const meta = { tags: ['notes'], @@ -40,41 +40,50 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export + private globalNotesRankingCache: string[] = []; + private globalNotesRankingCacheLastFetchedAt = 0; + constructor( @Inject(DI.notesRepository) private notesRepository: NotesRepository, private noteEntityService: NoteEntityService, - private queryService: QueryService, + private featuredService: FeaturedService, ) { super(meta, paramDef, async (ps, me) => { - const day = 1000 * 60 * 60 * 24 * 3; // 3日前まで + let noteIds: string[]; + if (ps.channelId) { + noteIds = await this.featuredService.getInChannelNotesRanking(ps.channelId, 50); + } else { + if (this.globalNotesRankingCacheLastFetchedAt !== 0 && (Date.now() - this.globalNotesRankingCacheLastFetchedAt < 1000 * 60 * 30)) { + noteIds = this.globalNotesRankingCache; + } else { + noteIds = await this.featuredService.getGlobalNotesRanking(100); + this.globalNotesRankingCache = noteIds; + this.globalNotesRankingCacheLastFetchedAt = Date.now(); + } + } + + if (noteIds.length === 0) { + return []; + } + + noteIds.sort((a, b) => a > b ? -1 : 1); + noteIds.slice(ps.offset, ps.offset + ps.limit); const query = this.notesRepository.createQueryBuilder('note') - .addSelect('note.score') - .where('note.userHost IS NULL') - .andWhere('note.score > 0') - .andWhere('note.createdAt > :date', { date: new Date(Date.now() - day) }) - .andWhere('note.visibility = \'public\'') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); - if (ps.channelId) query.andWhere('note.channelId = :channelId', { channelId: ps.channelId }); + const notes = await query.getMany(); + notes.sort((a, b) => a.id > b.id ? -1 : 1); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); - - let notes = await query - .orderBy('note.score', 'DESC') - .limit(100) - .getMany(); - - notes.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()); - - notes = notes.slice(ps.offset, ps.offset + ps.limit); + // TODO: ミュート等考慮 return await this.noteEntityService.packMany(notes, me); });