From e414737179bff4b744f49e431d7f3620be999e46 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 1 Jan 2023 16:53:10 +0900 Subject: [PATCH] feat: make possible to export favorited notes #9331 --- CHANGELOG.md | 1 + locales/ja-JP.yml | 1 + packages/backend/src/core/QueueService.ts | 12 +- .../src/queue/DbQueueProcessorsService.ts | 5 +- .../backend/src/queue/QueueProcessorModule.ts | 2 + .../ExportFavoritesProcessorService.ts | 162 ++++++++++++++++++ .../backend/src/server/api/EndpointsModule.ts | 4 + packages/backend/src/server/api/endpoints.ts | 2 + .../api/endpoints/i/export-favorites.ts | 31 ++++ .../src/pages/settings/import-export.vue | 12 ++ 10 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts create mode 100644 packages/backend/src/server/api/endpoints/i/export-favorites.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 550a5fcf9..25e175fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ You should also include the user name that made the change. - AVIF support @tamaina - Add Cloudflare Turnstile CAPTCHA support @CyberRex0 - Introduce retention-rate aggregation @syuilo +- Make possible to export favorited notes @syuilo - Server: signToActivityPubGet is set to true by default @syuilo - Server: improve syslog performance @syuilo - Server: improve note scoring for featured notes @CyberRex0 diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 2764a85ab..a07fb5ff9 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1421,6 +1421,7 @@ _profile: _exportOrImport: allNotes: "全てのノート" + favoritedNotes: "お気に入りにしたノート" followingList: "フォロー" muteList: "ミュート" blockingList: "ブロック" diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 7956a3a8f..4bf41e0ac 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -5,10 +5,10 @@ import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; -import { bindThis } from '@/decorators.js'; @Injectable() export class QueueService { @@ -97,6 +97,16 @@ export class QueueService { }); } + @bindThis + public createExportFavoritesJob(user: ThinUser) { + return this.dbQueue.add('exportFavorites', { + user: user, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + @bindThis public createExportFollowingJob(user: ThinUser, excludeMuting = false, excludeInactive = false) { return this.dbQueue.add('exportFollowing', { diff --git a/packages/backend/src/queue/DbQueueProcessorsService.ts b/packages/backend/src/queue/DbQueueProcessorsService.ts index e5568ab9b..df337ad81 100644 --- a/packages/backend/src/queue/DbQueueProcessorsService.ts +++ b/packages/backend/src/queue/DbQueueProcessorsService.ts @@ -2,6 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { DbJobData } from '@/queue/types.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { bindThis } from '@/decorators.js'; import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; @@ -15,8 +16,8 @@ import { ImportBlockingProcessorService } from './processors/ImportBlockingProce import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js'; import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js'; import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js'; +import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js'; import type Bull from 'bull'; -import { bindThis } from '@/decorators.js'; @Injectable() export class DbQueueProcessorsService { @@ -27,6 +28,7 @@ export class DbQueueProcessorsService { private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService, private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService, private exportNotesProcessorService: ExportNotesProcessorService, + private exportFavoritesProcessorService: ExportFavoritesProcessorService, private exportFollowingProcessorService: ExportFollowingProcessorService, private exportMutingProcessorService: ExportMutingProcessorService, private exportBlockingProcessorService: ExportBlockingProcessorService, @@ -45,6 +47,7 @@ export class DbQueueProcessorsService { q.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done)); q.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done)); q.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done)); + q.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done)); q.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done)); q.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done)); q.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done)); diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index 620296498..034e9cc5a 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -30,6 +30,7 @@ import { ImportUserListsProcessorService } from './processors/ImportUserListsPro import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js'; import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; +import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js'; @Module({ imports: [ @@ -45,6 +46,7 @@ import { AggregateRetentionProcessorService } from './processors/AggregateRetent DeleteDriveFilesProcessorService, ExportCustomEmojisProcessorService, ExportNotesProcessorService, + ExportFavoritesProcessorService, ExportFollowingProcessorService, ExportMutingProcessorService, ExportBlockingProcessorService, diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts new file mode 100644 index 000000000..3820705e5 --- /dev/null +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -0,0 +1,162 @@ +import * as fs from 'node:fs'; +import { Inject, Injectable } from '@nestjs/common'; +import { IsNull, MoreThan } from 'typeorm'; +import { format as dateFormat } from 'date-fns'; +import { DI } from '@/di-symbols.js'; +import type { NoteFavorite, NoteFavoritesRepository, NotesRepository, PollsRepository, User, UsersRepository } from '@/models/index.js'; +import type { Config } from '@/config.js'; +import type Logger from '@/logger.js'; +import { DriveService } from '@/core/DriveService.js'; +import { createTemp } from '@/misc/create-temp.js'; +import type { Poll } from '@/models/entities/Poll.js'; +import type { Note } from '@/models/entities/Note.js'; +import { bindThis } from '@/decorators.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type Bull from 'bull'; +import type { DbUserJobData } from '../types.js'; + +@Injectable() +export class ExportFavoritesProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.config) + private config: Config, + + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.pollsRepository) + private pollsRepository: PollsRepository, + + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + @Inject(DI.noteFavoritesRepository) + private noteFavoritesRepository: NoteFavoritesRepository, + + private driveService: DriveService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('export-favorites'); + } + + @bindThis + public async process(job: Bull.Job, done: () => void): Promise { + this.logger.info(`Exporting favorites of ${job.data.user.id} ...`); + + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + done(); + return; + } + + // Create temp file + const [path, cleanup] = await createTemp(); + + this.logger.info(`Temp file is ${path}`); + + try { + const stream = fs.createWriteStream(path, { flags: 'a' }); + + const write = (text: string): Promise => { + return new Promise((res, rej) => { + stream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await write('['); + + let exportedFavoritesCount = 0; + let cursor: NoteFavorite['id'] | null = null; + + while (true) { + const favorites = await this.noteFavoritesRepository.find({ + where: { + userId: user.id, + ...(cursor ? { id: MoreThan(cursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + relations: ['note', 'note.user'], + }) as (NoteFavorite & { note: Note & { user: User } })[]; + + if (favorites.length === 0) { + job.progress(100); + break; + } + + cursor = favorites[favorites.length - 1].id; + + for (const favorite of favorites) { + let poll: Poll | undefined; + if (favorite.note.hasPoll) { + poll = await this.pollsRepository.findOneByOrFail({ noteId: favorite.note.id }); + } + const content = JSON.stringify(serialize(favorite, poll)); + const isFirst = exportedFavoritesCount === 0; + await write(isFirst ? content : ',\n' + content); + exportedFavoritesCount++; + } + + const total = await this.noteFavoritesRepository.countBy({ + userId: user.id, + }); + + job.progress(exportedFavoritesCount / total); + } + + await write(']'); + + stream.end(); + this.logger.succ(`Exported to: ${path}`); + + const fileName = 'favorites-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true }); + + this.logger.succ(`Exported to: ${driveFile.id}`); + } finally { + cleanup(); + } + + done(); + } +} + +function serialize(favorite: NoteFavorite & { note: Note & { user: User } }, poll: Poll | null = null): Record { + return { + id: favorite.id, + createdAt: favorite.createdAt, + note: { + id: favorite.note.id, + text: favorite.note.text, + createdAt: favorite.note.createdAt, + fileIds: favorite.note.fileIds, + replyId: favorite.note.replyId, + renoteId: favorite.note.renoteId, + poll: poll, + cw: favorite.note.cw, + visibility: favorite.note.visibility, + visibleUserIds: favorite.note.visibleUserIds, + localOnly: favorite.note.localOnly, + uri: favorite.note.uri, + url: favorite.note.url, + user: { + id: favorite.note.user.id, + name: favorite.note.user.name, + username: favorite.note.user.username, + host: favorite.note.user.host, + uri: favorite.note.user.uri, + }, + }, + }; +} diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 1f96647e7..18ba16ac7 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -176,6 +176,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; @@ -495,6 +496,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default }; const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default }; const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default }; +const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default }; const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default }; const $i_favorites: Provider = { provide: 'ep:i/favorites', useClass: ep___i_favorites.default }; const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep___i_gallery_likes.default }; @@ -818,6 +820,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportFavorites, $i_exportUserLists, $i_favorites, $i_gallery_likes, @@ -1135,6 +1138,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $i_exportFollowing, $i_exportMute, $i_exportNotes, + $i_exportFavorites, $i_exportUserLists, $i_favorites, $i_gallery_likes, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index e8dc5abfa..a09ffa832 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -175,6 +175,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; +import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; @@ -492,6 +493,7 @@ const eps = [ ['i/export-following', ep___i_exportFollowing], ['i/export-mute', ep___i_exportMute], ['i/export-notes', ep___i_exportNotes], + ['i/export-favorites', ep___i_exportFavorites], ['i/export-user-lists', ep___i_exportUserLists], ['i/favorites', ep___i_favorites], ['i/gallery/likes', ep___i_gallery_likes], diff --git a/packages/backend/src/server/api/endpoints/i/export-favorites.ts b/packages/backend/src/server/api/endpoints/i/export-favorites.ts new file mode 100644 index 000000000..b32f39d3e --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/export-favorites.ts @@ -0,0 +1,31 @@ +import { Inject, Injectable } from '@nestjs/common'; +import ms from 'ms'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueueService } from '@/core/QueueService.js'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1day'), + max: 1, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; + +// eslint-disable-next-line import/no-default-export +@Injectable() +export default class extends Endpoint { + constructor( + private queueService: QueueService, + ) { + super(meta, paramDef, async (ps, me) => { + this.queueService.createExportFavoritesJob(me); + }); + } +} diff --git a/packages/frontend/src/pages/settings/import-export.vue b/packages/frontend/src/pages/settings/import-export.vue index 7db267c14..ca3bea4d1 100644 --- a/packages/frontend/src/pages/settings/import-export.vue +++ b/packages/frontend/src/pages/settings/import-export.vue @@ -8,6 +8,14 @@ {{ i18n.ts.export }} + + + + + + {{ i18n.ts.export }} + + @@ -108,6 +116,10 @@ const exportNotes = () => { os.api('i/export-notes', {}).then(onExportSuccess).catch(onError); }; +const exportFavorites = () => { + os.api('i/export-favorites', {}).then(onExportSuccess).catch(onError); +}; + const exportFollowing = () => { os.api('i/export-following', { excludeMuting: excludeMutingUsers.value,