From e9b6ed941bdfb3b94a97a4a8ee55ddf62ff9abef Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Tue, 7 Nov 2023 03:31:47 +0100 Subject: [PATCH 1/3] add: endpoint and processor for account data export --- packages/backend/src/core/QueueService.ts | 10 + .../backend/src/queue/QueueProcessorModule.ts | 2 + .../src/queue/QueueProcessorService.ts | 3 + .../ExportAccountDataProcessorService.ts | 767 ++++++++++++++++++ packages/backend/src/queue/types.ts | 1 + .../backend/src/server/api/EndpointsModule.ts | 4 + packages/backend/src/server/api/endpoints.ts | 2 + .../src/server/api/endpoints/i/export-data.ts | 35 + 8 files changed, 824 insertions(+) create mode 100644 packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts create mode 100644 packages/backend/src/server/api/endpoints/i/export-data.ts diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index be378a899..c5830168b 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -164,6 +164,16 @@ export class QueueService { }); } + @bindThis + public createExportAccountDataJob(user: ThinUser) { + return this.dbQueue.add('exportAccountData', { + user: { id: user.id }, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + @bindThis public createExportNotesJob(user: ThinUser) { return this.dbQueue.add('exportNotes', { diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index e6327002c..5c61eb9e9 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -19,6 +19,7 @@ import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesP import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js'; import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js'; +import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js'; import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js'; import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js'; @@ -51,6 +52,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor CheckExpiredMutingsProcessorService, CleanProcessorService, DeleteDriveFilesProcessorService, + ExportAccountDataProcessorService, ExportCustomEmojisProcessorService, ExportNotesProcessorService, ExportFavoritesProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 5201bfed8..7e45509fb 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -14,6 +14,7 @@ import { EndedPollNotificationProcessorService } from './processors/EndedPollNot import { DeliverProcessorService } from './processors/DeliverProcessorService.js'; import { InboxProcessorService } from './processors/InboxProcessorService.js'; import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; +import { ExportAccountDataProcessorService } from './processors/ExportAccountDataProcessorService.js'; import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js'; @@ -89,6 +90,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private deliverProcessorService: DeliverProcessorService, private inboxProcessorService: InboxProcessorService, private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService, + private exportAccountDataProcessorService: ExportAccountDataProcessorService, private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService, private exportNotesProcessorService: ExportNotesProcessorService, private exportFavoritesProcessorService: ExportFavoritesProcessorService, @@ -162,6 +164,7 @@ export class QueueProcessorService implements OnApplicationShutdown { this.dbQueueWorker = new Bull.Worker(QUEUE.DB, (job) => { switch (job.name) { case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job); + case 'exportAccountData': return this.exportAccountDataProcessorService.process(job); case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job); case 'exportNotes': return this.exportNotesProcessorService.process(job); case 'exportFavorites': return this.exportFavoritesProcessorService.process(job); diff --git a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts new file mode 100644 index 000000000..f85f757fa --- /dev/null +++ b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts @@ -0,0 +1,767 @@ +/* eslint-disable no-constant-condition */ +/* eslint-disable @typescript-eslint/no-unnecessary-condition */ + +import * as fs from 'node:fs'; +import { Inject, Injectable } from '@nestjs/common'; +import { In, IsNull, MoreThan, Not } from 'typeorm'; +import { format as dateFormat } from 'date-fns'; +import mime from 'mime-types'; +import archiver from 'archiver'; +import { DI } from '@/di-symbols.js'; +import type { AntennasRepository, BlockingsRepository, DriveFilesRepository, FollowingsRepository, MiBlocking, MiFollowing, MiMuting, MiNote, MiNoteFavorite, MiPoll, MiUser, MutingsRepository, NoteFavoritesRepository, NotesRepository, PollsRepository, UserListMembershipsRepository, UserListsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { Config } from '@/config.js'; +import type Logger from '@/logger.js'; +import { DriveService } from '@/core/DriveService.js'; +import { IdService } from '@/core/IdService.js'; +import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import { createTemp, createTempDir } from '@/misc/create-temp.js'; +import { bindThis } from '@/decorators.js'; +import { Packed } from '@/misc/json-schema.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { DownloadService } from '@/core/DownloadService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; + +@Injectable() +export class ExportAccountDataProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.config) + private config: Config, + + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.userProfilesRepository) + private userProfilesRepository: UserProfilesRepository, + + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + @Inject(DI.noteFavoritesRepository) + private noteFavoritesRepository: NoteFavoritesRepository, + + @Inject(DI.pollsRepository) + private pollsRepository: PollsRepository, + + @Inject(DI.followingsRepository) + private followingsRepository: FollowingsRepository, + + @Inject(DI.mutingsRepository) + private mutingsRepository: MutingsRepository, + + @Inject(DI.blockingsRepository) + private blockingsRepository: BlockingsRepository, + + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + + @Inject(DI.antennasRepository) + private antennasRepository: AntennasRepository, + + @Inject(DI.userListsRepository) + private userListsRepository: UserListsRepository, + + @Inject(DI.userListMembershipsRepository) + private userListMembershipsRepository: UserListMembershipsRepository, + + private utilityService: UtilityService, + private driveService: DriveService, + private idService: IdService, + private driveFileEntityService: DriveFileEntityService, + private downloadService: DownloadService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('export-account-data'); + } + + @bindThis + public async process(job: Bull.Job): Promise { + this.logger.info('Exporting Account Data...'); + + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const profile = await this.userProfilesRepository.findOneBy({ userId: job.data.user.id }); + if (profile == null) { + return; + } + + const [path, cleanup] = await createTempDir(); + + this.logger.info(`Temp dir is ${path}`); + + // User Export + + const userPath = path + '/user.json'; + + fs.writeFileSync(userPath, '', 'utf-8'); + + const userStream = fs.createWriteStream(userPath, { flags: 'a' }); + + const writeUser = (text: string): Promise => { + return new Promise((res, rej) => { + userStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeUser(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","user":[`); + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { host, uri, sharedInbox, followersUri, lastFetchedAt, inbox, ...userTrimmed } = user; + + await writeUser(JSON.stringify(userTrimmed)); + + await writeUser(']}'); + + userStream.end(); + + // Profile Export + + const profilePath = path + '/profile.json'; + + fs.writeFileSync(profilePath, '', 'utf-8'); + + const profileStream = fs.createWriteStream(profilePath, { flags: 'a' }); + + const writeProfile = (text: string): Promise => { + return new Promise((res, rej) => { + profileStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { emailVerifyCode, twoFactorBackupSecret, twoFactorSecret, password, twoFactorTempSecret, userHost, ...profileTrimmed } = profile; + + await writeProfile(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","profile":[`); + + await writeProfile(JSON.stringify(profileTrimmed)); + + await writeProfile(']}'); + + profileStream.end(); + + // Note Export + + const notesPath = path + '/notes.json'; + + fs.writeFileSync(notesPath, '', 'utf-8'); + + const notesStream = fs.createWriteStream(notesPath, { flags: 'a' }); + + const writeNotes = (text: string): Promise => { + return new Promise((res, rej) => { + notesStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeNotes(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","notes":[`); + + let noteCursor: MiNote['id'] | null = null; + let exportedNotesCount = 0; + + while (true) { + const notes = await this.notesRepository.find({ + where: { + userId: user.id, + ...(noteCursor ? { id: MoreThan(noteCursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }) as MiNote[]; + + if (notes.length === 0) { + break; + } + + noteCursor = notes.at(-1)?.id ?? null; + + for (const note of notes) { + let poll: MiPoll | undefined; + if (note.hasPoll) { + poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id }); + } + const files = await this.driveFileEntityService.packManyByIds(note.fileIds); + const content = JSON.stringify(this.noteSerialize(note, poll, files)); + const isFirst = exportedNotesCount === 0; + await writeNotes(isFirst ? content : ',\n' + content); + exportedNotesCount++; + } + } + + await writeNotes(']}'); + + notesStream.end(); + + // Following Export + + const followingsPath = path + '/followings.json'; + + fs.writeFileSync(followingsPath, '', 'utf-8'); + + const followingStream = fs.createWriteStream(followingsPath, { flags: 'a' }); + + const writeFollowing = (text: string): Promise => { + return new Promise((res, rej) => { + followingStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeFollowing(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followings":[`); + + let followingsCursor: MiFollowing['id'] | null = null; + let exportedFollowingsCount = 0; + + const mutings = await this.mutingsRepository.findBy({ + muterId: user.id, + }); + + while (true) { + const followings = await this.followingsRepository.find({ + where: { + followerId: user.id, + ...(mutings.length > 0 ? { followeeId: Not(In(mutings.map(x => x.muteeId))) } : {}), + ...(followingsCursor ? { id: MoreThan(followingsCursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }) as MiFollowing[]; + + if (followings.length === 0) { + break; + } + + followingsCursor = followings.at(-1)?.id ?? null; + + for (const following of followings) { + const u = await this.usersRepository.findOneBy({ id: following.followeeId }); + if (u == null) { + continue; + } + + if (u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { + continue; + } + + const isFirst = exportedFollowingsCount === 0; + const content = this.utilityService.getFullApAccount(u.username, u.host); + await writeFollowing(isFirst ? `"${content}"` : ',\n' + `"${content}"`); + exportedFollowingsCount++; + } + } + + await writeFollowing(']}'); + + followingStream.end(); + + // Followers Export + + const followersPath = path + '/followers.json'; + + fs.writeFileSync(followersPath, '', 'utf-8'); + + const followerStream = fs.createWriteStream(followersPath, { flags: 'a' }); + + const writeFollowers = (text: string): Promise => { + return new Promise((res, rej) => { + followerStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeFollowers(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followers":[`); + + let followersCursor: MiFollowing['id'] | null = null; + let exportedFollowersCount = 0; + + while (true) { + const followers = await this.followingsRepository.find({ + where: { + followeeId: user.id, + ...(followersCursor ? { id: MoreThan(followersCursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }) as MiFollowing[]; + + if (followers.length === 0) { + break; + } + + followersCursor = followers.at(-1)?.id ?? null; + + for (const follower of followers) { + const u = await this.usersRepository.findOneBy({ id: follower.followerId }); + if (u == null) { + continue; + } + + const isFirst = exportedFollowersCount === 0; + const content = this.utilityService.getFullApAccount(u.username, u.host); + await writeFollowers(isFirst ? `"${content}"` : ',\n' + `"${content}"`); + exportedFollowersCount++; + } + } + + await writeFollowers(']}'); + + followerStream.end(); + + // Drive Export + + const filesPath = path + '/drive.json'; + + fs.writeFileSync(filesPath, '', 'utf-8'); + + const filesStream = fs.createWriteStream(filesPath, { flags: 'a' }); + + const writeDrive = (text: string): Promise => { + return new Promise((res, rej) => { + filesStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + fs.mkdirSync(`${path}/files`); + + await writeDrive(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","drive":[`); + + const driveFiles = await this.driveFilesRepository.find({ where: { userId: user.id } }); + + for (const file of driveFiles) { + const ext = mime.extension(file.type); + const fileName = file.name + '.' + ext; + const filePath = path + '/files/' + fileName; + fs.writeFileSync(filePath, '', 'binary'); + let downloaded = false; + + try { + await this.downloadService.downloadUrl(file.url, filePath); + downloaded = true; + } catch (e) { + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + + if (!downloaded) { + fs.unlinkSync(filePath); + } + + const content = JSON.stringify({ + fileName: fileName, + file: file, + }); + const isFirst = driveFiles.indexOf(file) === 0; + + await writeDrive(isFirst ? content : ',\n' + content); + } + + await writeDrive(']}'); + + filesStream.end(); + + // Muting Export + + const mutingPath = path + '/mutings.json'; + + fs.writeFileSync(mutingPath, '', 'utf-8'); + + const mutingStream = fs.createWriteStream(mutingPath, { flags: 'a' }); + + const writeMuting = (text: string): Promise => { + return new Promise((res, rej) => { + mutingStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeMuting(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","mutings":[`); + + let exportedMutingCount = 0; + let mutingCursor: MiMuting['id'] | null = null; + + while (true) { + const mutes = await this.mutingsRepository.find({ + where: { + muterId: user.id, + expiresAt: IsNull(), + ...(mutingCursor ? { id: MoreThan(mutingCursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); + + if (mutes.length === 0) { + break; + } + + mutingCursor = mutes.at(-1)?.id ?? null; + + for (const mute of mutes) { + const u = await this.usersRepository.findOneBy({ id: mute.muteeId }); + + if (u == null) { + exportedMutingCount++; continue; + } + + const content = this.utilityService.getFullApAccount(u.username, u.host); + const isFirst = exportedMutingCount === 0; + await writeMuting(isFirst ? `"${content}"` : ',\n' + `"${content}"`); + exportedMutingCount++; + } + } + + await writeMuting(']}'); + + mutingStream.end(); + + // Blockings Export + + const blockingPath = path + '/blockings.json'; + + fs.writeFileSync(blockingPath, '', 'utf-8'); + + const blockingStream = fs.createWriteStream(blockingPath, { flags: 'a' }); + + const writeBlocking = (text: string): Promise => { + return new Promise((res, rej) => { + blockingStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeBlocking(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","blockings":[`); + + let exportedBlockingCount = 0; + let blockingCursor: MiBlocking['id'] | null = null; + + while (true) { + const blockings = await this.blockingsRepository.find({ + where: { + blockerId: user.id, + ...(blockingCursor ? { id: MoreThan(blockingCursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + }); + + if (blockings.length === 0) { + break; + } + + blockingCursor = blockings.at(-1)?.id ?? null; + + for (const block of blockings) { + const u = await this.usersRepository.findOneBy({ id: block.blockeeId }); + + if (u == null) { + exportedBlockingCount++; continue; + } + + const content = this.utilityService.getFullApAccount(u.username, u.host); + const isFirst = exportedBlockingCount === 0; + await writeBlocking(isFirst ? `"${content}"` : ',\n' + `"${content}"`); + exportedBlockingCount++; + } + } + + await writeBlocking(']}'); + + blockingStream.end(); + + // Favorites export + + const favoritePath = path + '/favorites.json'; + + fs.writeFileSync(favoritePath, '', 'utf-8'); + + const favoriteStream = fs.createWriteStream(favoritePath, { flags: 'a' }); + + const writeFavorite = (text: string): Promise => { + return new Promise((res, rej) => { + favoriteStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeFavorite(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","favorites":[`); + + let exportedFavoritesCount = 0; + let favoriteCursor: MiNoteFavorite['id'] | null = null; + + while (true) { + const favorites = await this.noteFavoritesRepository.find({ + where: { + userId: user.id, + ...(favoriteCursor ? { id: MoreThan(favoriteCursor) } : {}), + }, + take: 100, + order: { + id: 1, + }, + relations: ['note', 'note.user'], + }) as (MiNoteFavorite & { note: MiNote & { user: MiUser } })[]; + + if (favorites.length === 0) { + break; + } + + favoriteCursor = favorites.at(-1)?.id ?? null; + + for (const favorite of favorites) { + let poll: MiPoll | undefined; + if (favorite.note.hasPoll) { + poll = await this.pollsRepository.findOneByOrFail({ noteId: favorite.note.id }); + } + const content = JSON.stringify(this.favoriteSerialize(favorite, poll)); + const isFirst = exportedFavoritesCount === 0; + await writeFavorite(isFirst ? content : ',\n' + content); + exportedFavoritesCount++; + } + } + + await writeFavorite(']}'); + + favoriteStream.end(); + + // Antennas export + + const antennaPath = path + '/antennas.json'; + + fs.writeFileSync(antennaPath, '', 'utf-8'); + + const antennaStream = fs.createWriteStream(antennaPath, { flags: 'a' }); + + const writeAntenna = (text: string): Promise => { + return new Promise((res, rej) => { + antennaStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + await writeAntenna(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","antennas":[`); + + const antennas = await this.antennasRepository.findBy({ userId: user.id }); + + for (const [index, antenna] of antennas.entries()) { + let users: MiUser[] | undefined; + if (antenna.userListId !== null) { + const memberships = await this.userListMembershipsRepository.findBy({ userListId: antenna.userListId }); + users = await this.usersRepository.findBy({ + id: In(memberships.map(j => j.userId)), + }); + } + + await writeAntenna(JSON.stringify({ + name: antenna.name, + src: antenna.src, + keywords: antenna.keywords, + excludeKeywords: antenna.excludeKeywords, + users: antenna.users, + userListAccts: typeof users !== 'undefined' ? users.map((u) => { + return this.utilityService.getFullApAccount(u.username, u.host); // acct + }) : null, + caseSensitive: antenna.caseSensitive, + localOnly: antenna.localOnly, + withReplies: antenna.withReplies, + withFile: antenna.withFile, + notify: antenna.notify, + })); + + if (antennas.length - 1 !== index) { + await writeAntenna(', '); + } + } + + await writeAntenna(']}'); + + antennaStream.end(); + + // Lists export + + const listPath = path + '/lists.csv'; + + fs.writeFileSync(listPath, '', 'utf-8'); + + const listStream = fs.createWriteStream(listPath, { flags: 'a' }); + + const writeList = (text: string): Promise => { + return new Promise((res, rej) => { + listStream.write(text, err => { + if (err) { + this.logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + }; + + const lists = await this.userListsRepository.findBy({ + userId: user.id, + }); + + for (const list of lists) { + const memberships = await this.userListMembershipsRepository.findBy({ userListId: list.id }); + const users = await this.usersRepository.findBy({ + id: In(memberships.map(j => j.userId)), + }); + + for (const u of users) { + const acct = this.utilityService.getFullApAccount(u.username, u.host); + const content = `${list.name},${acct}`; + await writeList(content + '\n'); + } + } + + listStream.end(); + + // Create archive + await new Promise(async (resolve) => { + const [archivePath, archiveCleanup] = await createTemp(); + const archiveStream = fs.createWriteStream(archivePath); + const archive = archiver('zip', { + zlib: { level: 0 }, + }); + archiveStream.on('close', async () => { + this.logger.succ(`Exported to: ${archivePath}`); + + const fileName = 'data-request-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; + const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); + + this.logger.succ(`Exported to: ${driveFile.id}`); + cleanup(); + archiveCleanup(); + resolve(); + }); + archive.pipe(archiveStream); + archive.directory(path, false); + archive.finalize(); + }); + } + + private noteSerialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record { + return { + id: note.id, + text: note.text, + createdAt: this.idService.parse(note.id).date.toISOString(), + fileIds: note.fileIds, + files: files, + replyId: note.replyId, + renoteId: note.renoteId, + poll: poll, + cw: note.cw, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + localOnly: note.localOnly, + reactionAcceptance: note.reactionAcceptance, + }; + } + + private favoriteSerialize(favorite: MiNoteFavorite & { note: MiNote & { user: MiUser } }, poll: MiPoll | null = null): Record { + return { + id: favorite.id, + createdAt: this.idService.parse(favorite.id).date.toISOString(), + note: { + id: favorite.note.id, + text: favorite.note.text, + createdAt: this.idService.parse(favorite.note.id).date.toISOString(), + fileIds: favorite.note.fileIds, + replyId: favorite.note.replyId, + renoteId: favorite.note.renoteId, + poll: poll, + cw: favorite.note.cw, + visibility: favorite.note.visibility, + visibleUserIds: favorite.note.visibleUserIds, + localOnly: favorite.note.localOnly, + reactionAcceptance: favorite.note.reactionAcceptance, + uri: favorite.note.uri, + url: favorite.note.url, + user: { + id: favorite.note.user.id, + name: favorite.note.user.name, + username: favorite.note.user.username, + host: favorite.note.user.host, + uri: favorite.note.user.uri, + }, + }, + }; + } +} diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 9330c0152..94a95d8b9 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -39,6 +39,7 @@ export type DbJobData = DbJobMap[T]; export type DbJobMap = { deleteDriveFiles: DbJobDataWithUser; + exportAccountData: DbJobDataWithUser; exportCustomEmojis: DbJobDataWithUser; exportAntennas: DBExportAntennasData; exportNotes: DbJobDataWithUser; diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index fde35ffd3..09a8d8c37 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js'; import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js'; import * as ep___i_changePassword from './endpoints/i/change-password.js'; import * as ep___i_deleteAccount from './endpoints/i/delete-account.js'; +import * as ep___i_exportData from './endpoints/i/export-data.js'; import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; @@ -573,6 +574,7 @@ const $i_authorizedApps: Provider = { provide: 'ep:i/authorized-apps', useClass: const $i_claimAchievement: Provider = { provide: 'ep:i/claim-achievement', useClass: ep___i_claimAchievement.default }; const $i_changePassword: Provider = { provide: 'ep:i/change-password', useClass: ep___i_changePassword.default }; const $i_deleteAccount: Provider = { provide: 'ep:i/delete-account', useClass: ep___i_deleteAccount.default }; +const $i_exportData: Provider = { provide: 'ep:i/export-data', useClass: ep___i_exportData.default }; const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass: ep___i_exportBlocking.default }; const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default }; const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default }; @@ -944,6 +946,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de $i_claimAchievement, $i_changePassword, $i_deleteAccount, + $i_exportData, $i_exportBlocking, $i_exportFollowing, $i_exportMute, @@ -1309,6 +1312,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de $i_claimAchievement, $i_changePassword, $i_deleteAccount, + $i_exportData, $i_exportBlocking, $i_exportFollowing, $i_exportMute, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index c1cabd33e..527235264 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -206,6 +206,7 @@ import * as ep___i_authorizedApps from './endpoints/i/authorized-apps.js'; import * as ep___i_claimAchievement from './endpoints/i/claim-achievement.js'; import * as ep___i_changePassword from './endpoints/i/change-password.js'; import * as ep___i_deleteAccount from './endpoints/i/delete-account.js'; +import * as ep___i_exportData from './endpoints/i/export-data.js'; import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js'; @@ -571,6 +572,7 @@ const eps = [ ['i/claim-achievement', ep___i_claimAchievement], ['i/change-password', ep___i_changePassword], ['i/delete-account', ep___i_deleteAccount], + ['i/export-data', ep___i_exportData], ['i/export-blocking', ep___i_exportBlocking], ['i/export-following', ep___i_exportFollowing], ['i/export-mute', ep___i_exportMute], diff --git a/packages/backend/src/server/api/endpoints/i/export-data.ts b/packages/backend/src/server/api/endpoints/i/export-data.ts new file mode 100644 index 000000000..d9a1e087b --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/export-data.ts @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import ms from 'ms'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueueService } from '@/core/QueueService.js'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('3days'), + max: 1, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + private queueService: QueueService, + ) { + super(meta, paramDef, async (ps, me) => { + this.queueService.createExportAccountDataJob(me); + }); + } +} From fabb8762567b61367cfa535130f3b635a62d425d Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Tue, 7 Nov 2023 11:31:11 +0100 Subject: [PATCH 2/3] upd: send email on export completion --- .../processors/ExportAccountDataProcessorService.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts index f85f757fa..97902f858 100644 --- a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts @@ -21,6 +21,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; +import { EmailService } from '@/core/EmailService.js'; @Injectable() export class ExportAccountDataProcessorService { @@ -71,6 +72,7 @@ export class ExportAccountDataProcessorService { private idService: IdService, private driveFileEntityService: DriveFileEntityService, private downloadService: DownloadService, + private emailService: EmailService, private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-account-data'); @@ -709,6 +711,13 @@ export class ExportAccountDataProcessorService { this.logger.succ(`Exported to: ${driveFile.id}`); cleanup(); archiveCleanup(); + if (profile.email) { + this.emailService.sendEmail(profile.email, + 'Your data archive is ready', + `Click the following link to download the archive: ${driveFile.url}
It is also available in your drive.`, + `Click the following link to download the archive: ${driveFile.url}\r\n\r\nIt is also available in your drive.`, + ); + } resolve(); }); archive.pipe(archiveStream); From e409abdd400111e5ddef4c8c6f0ef3550f81d8f5 Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Tue, 7 Nov 2023 11:38:50 +0100 Subject: [PATCH 3/3] upd: add frontend option to export data --- locales/en-US.yml | 6 +++++ locales/index.d.ts | 6 +++++ locales/ja-JP.yml | 6 +++++ .../frontend/src/pages/settings/other.vue | 25 +++++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/locales/en-US.yml b/locales/en-US.yml index 041ac2178..730e7450d 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -2401,3 +2401,9 @@ _animatedMFM: _alert: text: "Animated MFMs could include flashing lights and fast moving text/emojis." confirm: "Animate" + +_dataRequest: + title: "Request Data" + warn: "Data requests are only possible every 3 days." + text: "Once the data is ready to download, an email will be sent to the email address registered to this account." + button: "Request" diff --git a/locales/index.d.ts b/locales/index.d.ts index 56a9f9bc2..d4934b77a 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -2509,6 +2509,12 @@ export interface Locale { "confirm": string; }; }; + "_dataRequest": { + "title": string; + "warn": string; + "text": string; + "button": string; + }; } declare const locales: { [lang: string]: Locale; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index c45a820e3..36521c881 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -2395,3 +2395,9 @@ _animatedMFM: _alert: text: "アニメーションMFMには、点滅するライトや高速で動くテキスト／絵文字を含めることができる。" confirm: "アニメイト" + +_dataRequest: + title: "リクエストデータ" + warn: "データのリクエストは3日ごとにしかできない。" + text: "データのダウンロードが完了すると、このアカウントに登録されているEメールアドレスにEメールが送信されます。" + button: "リクエスト" diff --git a/packages/frontend/src/pages/settings/other.vue b/packages/frontend/src/pages/settings/other.vue index 3bd3ddbe9..eb2ee2760 100644 --- a/packages/frontend/src/pages/settings/other.vue +++ b/packages/frontend/src/pages/settings/other.vue @@ -34,6 +34,17 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + +
+ {{ i18n.ts._dataRequest.warn }} + {{ i18n.ts._dataRequest.text }} + {{ i18n.ts._dataRequest.button }} +
+ + @@ -156,6 +167,20 @@ async function updateRepliesAll(withReplies: boolean) { await os.api('following/update-all', { withReplies }); } +const exportData = () => { + os.api('i/export-data', {}).then(() => { + os.alert({ + type: 'info', + text: i18n.ts.exportRequested, + }); + }).catch((ev) => { + os.alert({ + type: 'error', + text: ev.message, + }); + }); +}; + watch([ enableCondensedLineForAcct, ], async () => {