heartles
/
egirlskey
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

perf(backend): store notes of an antenna to redis instead of postgresql 

Resolve #10169
This commit is contained in:
syuilo 2023-04-03 12:11:16 +09:00
parent c032dd1214
commit b53d6c7f8c
14 changed files with 64 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -17,6 +17,8 @@
### General
- チャンネルをお気に入りに登録できるように
- チャンネルにノートをピン留めできるように
- アンテナのタイムライン取得時のパフォーマンスを向上
- チャンネルのタイムライン取得時のパフォーマンスを向上
### Client
- 検索ページでURLを入力した際に照会したときと同等の挙動をするように

10
packages/backend/migration/1680491187535-cleanup.js Normal file
View File

@ -0,0 +1,10 @@
export class cleanup1680491187535 {
name = 'cleanup1680491187535'
async up(queryRunner) {
await queryRunner.query(`DROP TABLE "antenna_note" `);
}
async down(queryRunner) {
}
}

61
packages/backend/src/core/AntennaService.ts
View File

@ -12,7 +12,7 @@ import { PushNotificationService } from '@/core/PushNotificationService.js';
import * as Acct from '@/misc/acct.js';
import type { Packed } from '@/misc/json-schema.js';
import { DI } from '@/di-symbols.js';
import type { MutingsRepository, NotesRepository, AntennaNotesRepository, AntennasRepository, UserListJoiningsRepository } from '@/models/index.js';
import type { MutingsRepository, NotesRepository, AntennasRepository, UserListJoiningsRepository } from '@/models/index.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { StreamMessages } from '@/server/api/stream/types.js';
@ -24,6 +24,9 @@ export class AntennaService implements OnApplicationShutdown {
private antennas: Antenna[];
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisSubscriber)
private redisSubscriber: Redis.Redis,
@ -33,9 +36,6 @@ export class AntennaService implements OnApplicationShutdown {
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@Inject(DI.antennaNotesRepository)
private antennaNotesRepository: AntennaNotesRepository,
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
@ -92,54 +92,13 @@ export class AntennaService implements OnApplicationShutdown {
@bindThis
public async addNoteToAntenna(antenna: Antenna, note: Note, noteUser: { id: User['id']; }): Promise<void> {
// 通知しない設定になっているか、自分自身の投稿なら既読にする
const read = !antenna.notify || (antenna.userId === noteUser.id);
this.antennaNotesRepository.insert({
id: this.idService.genId(),
antennaId: antenna.id,
noteId: note.id,
read: read,
});
this.redisClient.xadd(
`antennaTimeline:${antenna.id}`,
'MAXLEN', '~', '200',
`${this.idService.parse(note.id).date.getTime()}-*`,
'note', note.id);
this.globalEventService.publishAntennaStream(antenna.id, 'note', note);
if (!read) {
const mutings = await this.mutingsRepository.find({
where: {
muterId: antenna.userId,
},
select: ['muteeId'],
});
// Copy
const _note: Note = {
...note,
};
if (note.replyId != null) {
_note.reply = await this.notesRepository.findOneByOrFail({ id: note.replyId });
}
if (note.renoteId != null) {
_note.renote = await this.notesRepository.findOneByOrFail({ id: note.renoteId });
}
if (isUserRelated(_note, new Set<string>(mutings.map(x => x.muteeId)))) {
return;
}
// 2秒経っても既読にならなかったら通知
setTimeout(async () => {
const unread = await this.antennaNotesRepository.findOneBy({ antennaId: antenna.id, read: false });
if (unread) {
this.globalEventService.publishMainStream(antenna.userId, 'unreadAntenna', antenna);
this.pushNotificationService.pushNotification(antenna.userId, 'unreadAntennaNote', {
antenna: { id: antenna.id, name: antenna.name },
note: await this.noteEntityService.pack(note),
});
}
}, 2000);
}
}
// NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている

6
packages/backend/src/core/NoteCreateService.ts
View File

@ -326,7 +326,11 @@ export class NoteCreateService implements OnApplicationShutdown {
const note = await this.insertNote(user, data, tags, emojis, mentionedUsers);
if (data.channel) {
this.redisClient.xadd(`channelTimeline:${data.channel.id}`, 'MAXLEN', '~', '1000', `${this.idService.parse(note.id).date.getTime()}-*`, 'note', note.id);
this.redisClient.xadd(
`channelTimeline:${data.channel.id}`,
'MAXLEN', '~', '1000',
`${this.idService.parse(note.id).date.getTime()}-*`,
'note', note.id);
}
setImmediate('post created', { signal: this.#shutdownController.signal }).then(

43
packages/backend/src/core/NoteReadService.ts
View File

@ -8,7 +8,7 @@ import type { Packed } from '@/misc/json-schema.js';
import type { Note } from '@/models/entities/Note.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import type { UsersRepository, NoteUnreadsRepository, MutingsRepository, NoteThreadMutingsRepository, FollowingsRepository, ChannelFollowingsRepository, AntennaNotesRepository } from '@/models/index.js';
import type { UsersRepository, NoteUnreadsRepository, MutingsRepository, NoteThreadMutingsRepository, FollowingsRepository, ChannelFollowingsRepository } from '@/models/index.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import { NotificationService } from './NotificationService.js';
@ -38,9 +38,6 @@ export class NoteReadService implements OnApplicationShutdown {
@Inject(DI.channelFollowingsRepository)
private channelFollowingsRepository: ChannelFollowingsRepository,
@Inject(DI.antennaNotesRepository)
private antennaNotesRepository: AntennaNotesRepository,
private userEntityService: UserEntityService,
private idService: IdService,
private globalEventService: GlobalEventService,
@ -121,7 +118,6 @@ export class NoteReadService implements OnApplicationShutdown {
const readMentions: (Note | Packed<'Note'>)[] = [];
const readSpecifiedNotes: (Note | Packed<'Note'>)[] = [];
const readChannelNotes: (Note | Packed<'Note'>)[] = [];
const readAntennaNotes: (Note | Packed<'Note'>)[] = [];
for (const note of notes) {
if (note.mentions && note.mentions.includes(userId)) {
@ -133,14 +129,6 @@ export class NoteReadService implements OnApplicationShutdown {
if (note.channelId && followingChannels.has(note.channelId)) {
readChannelNotes.push(note);
}
if (note.user != null) { // たぶんnullになることは無いはずだけど一応
for (const antenna of myAntennas) {
if (await this.antennaService.checkHitAntenna(antenna, note, note.user)) {
readAntennaNotes.push(note);
}
}
}
}
if ((readMentions.length > 0) || (readSpecifiedNotes.length > 0) || (readChannelNotes.length > 0)) {
@ -186,35 +174,6 @@ export class NoteReadService implements OnApplicationShutdown {
noteId: In([...readMentions.map(n => n.id), ...readSpecifiedNotes.map(n => n.id)]),
});
}
if (readAntennaNotes.length > 0) {
await this.antennaNotesRepository.update({
antennaId: In(myAntennas.map(a => a.id)),
noteId: In(readAntennaNotes.map(n => n.id)),
}, {
read: true,
});
// TODO: まとめてクエリしたい
for (const antenna of myAntennas) {
const count = await this.antennaNotesRepository.countBy({
antennaId: antenna.id,
read: false,
});
if (count === 0) {
this.globalEventService.publishMainStream(userId, 'readAntenna', antenna);
this.pushNotificationService.pushNotification(userId, 'readAntenna', { antennaId: antenna.id });
}
}
this.userEntityService.getHasUnreadAntenna(userId).then(unread => {
if (!unread) {
this.globalEventService.publishMainStream(userId, 'readAllAntennas');
this.pushNotificationService.pushNotification(userId, 'readAllAntennas', undefined);
}
});
}
}
onApplicationShutdown(signal?: string | undefined): void {

9
packages/backend/src/core/entities/AntennaEntityService.ts
View File

@ -1,6 +1,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { AntennaNotesRepository, AntennasRepository } from '@/models/index.js';
import type { AntennasRepository } from '@/models/index.js';
import type { Packed } from '@/misc/json-schema.js';
import type { Antenna } from '@/models/entities/Antenna.js';
import { bindThis } from '@/decorators.js';
@ -10,9 +10,6 @@ export class AntennaEntityService {
constructor(
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
@Inject(DI.antennaNotesRepository)
private antennaNotesRepository: AntennaNotesRepository,
) {
}
@ -22,8 +19,6 @@ export class AntennaEntityService {
): Promise<Packed<'Antenna'>> {
const antenna = typeof src === 'object' ? src : await this.antennasRepository.findOneByOrFail({ id: src });
const hasUnreadNote = (await this.antennaNotesRepository.findOneBy({ antennaId: antenna.id, read: false })) != null;
return {
id: antenna.id,
createdAt: antenna.createdAt.toISOString(),
@ -38,7 +33,7 @@ export class AntennaEntityService {
withReplies: antenna.withReplies,
withFile: antenna.withFile,
isActive: antenna.isActive,
hasUnreadNote,
hasUnreadNote: false, // TODO
};
}
}

8
packages/backend/src/core/entities/UserEntityService.ts
View File

@ -12,7 +12,7 @@ import { KVCache } from '@/misc/cache.js';
import type { Instance } from '@/models/entities/Instance.js';
import type { LocalUser, RemoteUser, User } from '@/models/entities/User.js';
import { birthdaySchema, descriptionSchema, localUsernameSchema, locationSchema, nameSchema, passwordSchema } from '@/models/entities/User.js';
import type { UsersRepository, UserSecurityKeysRepository, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, DriveFilesRepository, NoteUnreadsRepository, ChannelFollowingsRepository, NotificationsRepository, UserNotePiningsRepository, UserProfilesRepository, InstancesRepository, AnnouncementReadsRepository, AnnouncementsRepository, AntennaNotesRepository, PagesRepository, UserProfile, RenoteMutingsRepository } from '@/models/index.js';
import type { UsersRepository, UserSecurityKeysRepository, FollowingsRepository, FollowRequestsRepository, BlockingsRepository, MutingsRepository, DriveFilesRepository, NoteUnreadsRepository, ChannelFollowingsRepository, NotificationsRepository, UserNotePiningsRepository, UserProfilesRepository, InstancesRepository, AnnouncementReadsRepository, AnnouncementsRepository, PagesRepository, UserProfile, RenoteMutingsRepository } from '@/models/index.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import type { OnModuleInit } from '@nestjs/common';
@ -108,9 +108,6 @@ export class UserEntityService implements OnModuleInit {
@Inject(DI.announcementsRepository)
private announcementsRepository: AnnouncementsRepository,
@Inject(DI.antennaNotesRepository)
private antennaNotesRepository: AntennaNotesRepository,
@Inject(DI.pagesRepository)
private pagesRepository: PagesRepository,
@ -223,6 +220,7 @@ export class UserEntityService implements OnModuleInit {
@bindThis
public async getHasUnreadAntenna(userId: User['id']): Promise<boolean> {
/*
const myAntennas = (await this.antennaService.getAntennas()).filter(a => a.userId === userId);
const unread = myAntennas.length > 0 ? await this.antennaNotesRepository.findOneBy({
@ -231,6 +229,8 @@ export class UserEntityService implements OnModuleInit {
}) : null;
return unread != null;
*/
return false; // TODO
}
@bindThis

1
packages/backend/src/di-symbols.ts
View File

@ -54,7 +54,6 @@ export const DI = {
clipNotesRepository: Symbol('clipNotesRepository'),
clipFavoritesRepository: Symbol('clipFavoritesRepository'),
antennasRepository: Symbol('antennasRepository'),
antennaNotesRepository: Symbol('antennaNotesRepository'),
promoNotesRepository: Symbol('promoNotesRepository'),
promoReadsRepository: Symbol('promoReadsRepository'),
relaysRepository: Symbol('relaysRepository'),

10
packages/backend/src/models/RepositoryModule.ts
View File

@ -1,6 +1,6 @@
import { Module } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import { User, Note, Announcement, AnnouncementRead, App, NoteFavorite, NoteThreadMuting, NoteReaction, NoteUnread, Notification, Poll, PollVote, UserProfile, UserKeypair, UserPending, AttestationChallenge, UserSecurityKey, UserPublickey, UserList, UserListJoining, UserNotePining, UserIp, UsedUsername, Following, FollowRequest, Instance, Emoji, DriveFile, DriveFolder, Meta, Muting, RenoteMuting, Blocking, SwSubscription, Hashtag, AbuseUserReport, RegistrationTicket, AuthSession, AccessToken, Signin, Page, PageLike, GalleryPost, GalleryLike, ModerationLog, Clip, ClipNote, Antenna, AntennaNote, PromoNote, PromoRead, Relay, MutedNote, Channel, ChannelFollowing, ChannelFavorite, RegistryItem, Webhook, Ad, PasswordResetRequest, RetentionAggregation, FlashLike, Flash, Role, RoleAssignment, ClipFavorite } from './index.js';
import { User, Note, Announcement, AnnouncementRead, App, NoteFavorite, NoteThreadMuting, NoteReaction, NoteUnread, Notification, Poll, PollVote, UserProfile, UserKeypair, UserPending, AttestationChallenge, UserSecurityKey, UserPublickey, UserList, UserListJoining, UserNotePining, UserIp, UsedUsername, Following, FollowRequest, Instance, Emoji, DriveFile, DriveFolder, Meta, Muting, RenoteMuting, Blocking, SwSubscription, Hashtag, AbuseUserReport, RegistrationTicket, AuthSession, AccessToken, Signin, Page, PageLike, GalleryPost, GalleryLike, ModerationLog, Clip, ClipNote, Antenna, PromoNote, PromoRead, Relay, MutedNote, Channel, ChannelFollowing, ChannelFavorite, RegistryItem, Webhook, Ad, PasswordResetRequest, RetentionAggregation, FlashLike, Flash, Role, RoleAssignment, ClipFavorite } from './index.js';
import type { DataSource } from 'typeorm';
import type { Provider } from '@nestjs/common';
@ -298,12 +298,6 @@ const $antennasRepository: Provider = {
inject: [DI.db],
};
const $antennaNotesRepository: Provider = {
provide: DI.antennaNotesRepository,
useFactory: (db: DataSource) => db.getRepository(AntennaNote),
inject: [DI.db],
};
const $promoNotesRepository: Provider = {
provide: DI.promoNotesRepository,
useFactory: (db: DataSource) => db.getRepository(PromoNote),
@ -453,7 +447,6 @@ const $roleAssignmentsRepository: Provider = {
$clipNotesRepository,
$clipFavoritesRepository,
$antennasRepository,
$antennaNotesRepository,
$promoNotesRepository,
$promoReadsRepository,
$relaysRepository,
@ -521,7 +514,6 @@ const $roleAssignmentsRepository: Provider = {
$clipNotesRepository,
$clipFavoritesRepository,
$antennasRepository,
$antennaNotesRepository,
$promoNotesRepository,
$promoReadsRepository,
$relaysRepository,

43
packages/backend/src/models/entities/AntennaNote.ts
View File

@ -1,43 +0,0 @@
import { Entity, Index, JoinColumn, Column, ManyToOne, PrimaryColumn } from 'typeorm';
import { id } from '../id.js';
import { Note } from './Note.js';
import { Antenna } from './Antenna.js';
@Entity()
@Index(['noteId', 'antennaId'], { unique: true })
export class AntennaNote {
@PrimaryColumn(id())
public id: string;
@Index()
@Column({
...id(),
comment: 'The note ID.',
})
public noteId: Note['id'];
@ManyToOne(type => Note, {
onDelete: 'CASCADE',
})
@JoinColumn()
public note: Note | null;
@Index()
@Column({
...id(),
comment: 'The antenna ID.',
})
public antennaId: Antenna['id'];
@ManyToOne(type => Antenna, {
onDelete: 'CASCADE',
})
@JoinColumn()
public antenna: Antenna | null;
@Index()
@Column('boolean', {
default: false,
})
public read: boolean;
}

3
packages/backend/src/models/index.ts
View File

@ -4,7 +4,6 @@ import { Ad } from '@/models/entities/Ad.js';
import { Announcement } from '@/models/entities/Announcement.js';
import { AnnouncementRead } from '@/models/entities/AnnouncementRead.js';
import { Antenna } from '@/models/entities/Antenna.js';
import { AntennaNote } from '@/models/entities/AntennaNote.js';
import { App } from '@/models/entities/App.js';
import { AttestationChallenge } from '@/models/entities/AttestationChallenge.js';
import { AuthSession } from '@/models/entities/AuthSession.js';
@ -73,7 +72,6 @@ export {
Announcement,
AnnouncementRead,
Antenna,
AntennaNote,
App,
AttestationChallenge,
AuthSession,
@ -141,7 +139,6 @@ export type AdsRepository = Repository<Ad>;
export type AnnouncementsRepository = Repository<Announcement>;
export type AnnouncementReadsRepository = Repository<AnnouncementRead>;
export type AntennasRepository = Repository<Antenna>;
export type AntennaNotesRepository = Repository<AntennaNote>;
export type AppsRepository = Repository<App>;
export type AttestationChallengesRepository = Repository<AttestationChallenge>;
export type AuthSessionsRepository = Repository<AuthSession>;

2
packages/backend/src/postgres.ts
View File

@ -12,7 +12,6 @@ import { Ad } from '@/models/entities/Ad.js';
import { Announcement } from '@/models/entities/Announcement.js';
import { AnnouncementRead } from '@/models/entities/AnnouncementRead.js';
import { Antenna } from '@/models/entities/Antenna.js';
import { AntennaNote } from '@/models/entities/AntennaNote.js';
import { App } from '@/models/entities/App.js';
import { AttestationChallenge } from '@/models/entities/AttestationChallenge.js';
import { AuthSession } from '@/models/entities/AuthSession.js';
@ -168,7 +167,6 @@ export const entities = [
ClipNote,
ClipFavorite,
Antenna,
AntennaNote,
PromoNote,
PromoRead,
Relay,

5
packages/backend/src/queue/processors/CleanProcessorService.ts
View File

@ -1,7 +1,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { In, LessThan } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { AntennaNotesRepository, AntennasRepository, MutedNotesRepository, NotificationsRepository, RoleAssignmentsRepository, UserIpsRepository } from '@/models/index.js';
import type { AntennasRepository, MutedNotesRepository, NotificationsRepository, RoleAssignmentsRepository, UserIpsRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
@ -29,9 +29,6 @@ export class CleanProcessorService {
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
@Inject(DI.antennaNotesRepository)
private antennaNotesRepository: AntennaNotesRepository,
@Inject(DI.roleAssignmentsRepository)
private roleAssignmentsRepository: RoleAssignmentsRepository,

40
packages/backend/src/server/api/endpoints/antennas/notes.ts
View File

@ -1,10 +1,12 @@
import { Inject, Injectable } from '@nestjs/common';
import Redis from 'ioredis';
import { Endpoint } from '@/server/api/endpoint-base.js';
import type { NotesRepository, AntennaNotesRepository, AntennasRepository } from '@/models/index.js';
import type { NotesRepository, AntennasRepository } from '@/models/index.js';
import { QueryService } from '@/core/QueryService.js';
import { NoteReadService } from '@/core/NoteReadService.js';
import { DI } from '@/di-symbols.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { IdService } from '@/core/IdService.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -50,15 +52,16 @@ export const paramDef = {
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
@Inject(DI.antennaNotesRepository)
private antennaNotesRepository: AntennaNotesRepository,
private idService: IdService,
private noteEntityService: NoteEntityService,
private queryService: QueryService,
private noteReadService: NoteReadService,
@ -73,9 +76,24 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
throw new ApiError(meta.errors.noSuchAntenna);
}
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'),
ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.innerJoin(this.antennaNotesRepository.metadata.targetName, 'antennaNote', 'antennaNote.noteId = note.id')
const noteIdsRes = await this.redisClient.xrevrange(
`antennaTimeline:${antenna.id}`,
ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : '+',
'-',
'COUNT', ps.limit + 1); // untilIdに指定したものも含まれるため+1
if (noteIdsRes.length === 0) {
return [];
}
const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId);
if (noteIds.length === 0) {
return [];
}
const query = this.notesRepository.createQueryBuilder('note')
.where('note.id IN (:...noteIds)', { noteIds: noteIds })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('user.avatar', 'avatar')
.leftJoinAndSelect('user.banner', 'banner')
@ -86,16 +104,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
.leftJoinAndSelect('replyUser.banner', 'replyUserBanner')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('renoteUser.avatar', 'renoteUserAvatar')
.leftJoinAndSelect('renoteUser.banner', 'renoteUserBanner')
.andWhere('antennaNote.antennaId = :antennaId', { antennaId: antenna.id });
.leftJoinAndSelect('renoteUser.banner', 'renoteUserBanner');
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateMutedUserQuery(query, me);
this.queryService.generateBlockedUserQuery(query, me);
const notes = await query
.take(ps.limit)
.getMany();
const notes = await query.getMany();
notes.sort((a, b) => a.id > b.id ? -1 : 1);
if (notes.length > 0) {
this.noteReadService.read(me.id, notes);