feat: make possible to export favorited notes

#9331
This commit is contained in:
syuilo 2023-01-01 16:53:10 +09:00
parent 0d34d28c56
commit e414737179
10 changed files with 230 additions and 2 deletions

View file

@ -33,6 +33,7 @@ You should also include the user name that made the change.
- AVIF support @tamaina - AVIF support @tamaina
- Add Cloudflare Turnstile CAPTCHA support @CyberRex0 - Add Cloudflare Turnstile CAPTCHA support @CyberRex0
- Introduce retention-rate aggregation @syuilo - Introduce retention-rate aggregation @syuilo
- Make possible to export favorited notes @syuilo
- Server: signToActivityPubGet is set to true by default @syuilo - Server: signToActivityPubGet is set to true by default @syuilo
- Server: improve syslog performance @syuilo - Server: improve syslog performance @syuilo
- Server: improve note scoring for featured notes @CyberRex0 - Server: improve note scoring for featured notes @CyberRex0

View file

@ -1421,6 +1421,7 @@ _profile:
_exportOrImport: _exportOrImport:
allNotes: "全てのノート" allNotes: "全てのノート"
favoritedNotes: "お気に入りにしたノート"
followingList: "フォロー" followingList: "フォロー"
muteList: "ミュート" muteList: "ミュート"
blockingList: "ブロック" blockingList: "ブロック"

View file

@ -5,10 +5,10 @@ import type { DriveFile } from '@/models/entities/DriveFile.js';
import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js';
import type { ThinUser } from '../queue/types.js'; import type { ThinUser } from '../queue/types.js';
import type httpSignature from '@peertube/http-signature'; import type httpSignature from '@peertube/http-signature';
import { bindThis } from '@/decorators.js';
@Injectable() @Injectable()
export class QueueService { export class QueueService {
@ -97,6 +97,16 @@ export class QueueService {
}); });
} }
@bindThis
public createExportFavoritesJob(user: ThinUser) {
return this.dbQueue.add('exportFavorites', {
user: user,
}, {
removeOnComplete: true,
removeOnFail: true,
});
}
@bindThis @bindThis
public createExportFollowingJob(user: ThinUser, excludeMuting = false, excludeInactive = false) { public createExportFollowingJob(user: ThinUser, excludeMuting = false, excludeInactive = false) {
return this.dbQueue.add('exportFollowing', { return this.dbQueue.add('exportFollowing', {

View file

@ -2,6 +2,7 @@ import { Inject, Injectable } from '@nestjs/common';
import type { DbJobData } from '@/queue/types.js'; import type { DbJobData } from '@/queue/types.js';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js'; import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js'; import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js'; import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
@ -15,8 +16,8 @@ import { ImportBlockingProcessorService } from './processors/ImportBlockingProce
import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js'; import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js';
import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js'; import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js';
import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js'; import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js';
import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js';
import type Bull from 'bull'; import type Bull from 'bull';
import { bindThis } from '@/decorators.js';
@Injectable() @Injectable()
export class DbQueueProcessorsService { export class DbQueueProcessorsService {
@ -27,6 +28,7 @@ export class DbQueueProcessorsService {
private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService, private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService, private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
private exportNotesProcessorService: ExportNotesProcessorService, private exportNotesProcessorService: ExportNotesProcessorService,
private exportFavoritesProcessorService: ExportFavoritesProcessorService,
private exportFollowingProcessorService: ExportFollowingProcessorService, private exportFollowingProcessorService: ExportFollowingProcessorService,
private exportMutingProcessorService: ExportMutingProcessorService, private exportMutingProcessorService: ExportMutingProcessorService,
private exportBlockingProcessorService: ExportBlockingProcessorService, private exportBlockingProcessorService: ExportBlockingProcessorService,
@ -45,6 +47,7 @@ export class DbQueueProcessorsService {
q.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done)); q.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done));
q.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done)); q.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done));
q.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done)); q.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done));
q.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done));
q.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done)); q.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done));
q.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done)); q.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done));
q.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done)); q.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done));

View file

@ -30,6 +30,7 @@ import { ImportUserListsProcessorService } from './processors/ImportUserListsPro
import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js'; import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js';
import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js'; import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js';
import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js';
@Module({ @Module({
imports: [ imports: [
@ -45,6 +46,7 @@ import { AggregateRetentionProcessorService } from './processors/AggregateRetent
DeleteDriveFilesProcessorService, DeleteDriveFilesProcessorService,
ExportCustomEmojisProcessorService, ExportCustomEmojisProcessorService,
ExportNotesProcessorService, ExportNotesProcessorService,
ExportFavoritesProcessorService,
ExportFollowingProcessorService, ExportFollowingProcessorService,
ExportMutingProcessorService, ExportMutingProcessorService,
ExportBlockingProcessorService, ExportBlockingProcessorService,

View file

@ -0,0 +1,162 @@
import * as fs from 'node:fs';
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, MoreThan } from 'typeorm';
import { format as dateFormat } from 'date-fns';
import { DI } from '@/di-symbols.js';
import type { NoteFavorite, NoteFavoritesRepository, NotesRepository, PollsRepository, User, UsersRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js';
import type { Poll } from '@/models/entities/Poll.js';
import type { Note } from '@/models/entities/Note.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
@Injectable()
export class ExportFavoritesProcessorService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.pollsRepository)
private pollsRepository: PollsRepository,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@Inject(DI.noteFavoritesRepository)
private noteFavoritesRepository: NoteFavoritesRepository,
private driveService: DriveService,
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('export-favorites');
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
this.logger.info(`Exporting favorites of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`Temp file is ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
const write = (text: string): Promise<void> => {
return new Promise<void>((res, rej) => {
stream.write(text, err => {
if (err) {
this.logger.error(err);
rej(err);
} else {
res();
}
});
});
};
await write('[');
let exportedFavoritesCount = 0;
let cursor: NoteFavorite['id'] | null = null;
while (true) {
const favorites = await this.noteFavoritesRepository.find({
where: {
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
relations: ['note', 'note.user'],
}) as (NoteFavorite & { note: Note & { user: User } })[];
if (favorites.length === 0) {
job.progress(100);
break;
}
cursor = favorites[favorites.length - 1].id;
for (const favorite of favorites) {
let poll: Poll | undefined;
if (favorite.note.hasPoll) {
poll = await this.pollsRepository.findOneByOrFail({ noteId: favorite.note.id });
}
const content = JSON.stringify(serialize(favorite, poll));
const isFirst = exportedFavoritesCount === 0;
await write(isFirst ? content : ',\n' + content);
exportedFavoritesCount++;
}
const total = await this.noteFavoritesRepository.countBy({
userId: user.id,
});
job.progress(exportedFavoritesCount / total);
}
await write(']');
stream.end();
this.logger.succ(`Exported to: ${path}`);
const fileName = 'favorites-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true });
this.logger.succ(`Exported to: ${driveFile.id}`);
} finally {
cleanup();
}
done();
}
}
function serialize(favorite: NoteFavorite & { note: Note & { user: User } }, poll: Poll | null = null): Record<string, unknown> {
return {
id: favorite.id,
createdAt: favorite.createdAt,
note: {
id: favorite.note.id,
text: favorite.note.text,
createdAt: favorite.note.createdAt,
fileIds: favorite.note.fileIds,
replyId: favorite.note.replyId,
renoteId: favorite.note.renoteId,
poll: poll,
cw: favorite.note.cw,
visibility: favorite.note.visibility,
visibleUserIds: favorite.note.visibleUserIds,
localOnly: favorite.note.localOnly,
uri: favorite.note.uri,
url: favorite.note.url,
user: {
id: favorite.note.user.id,
name: favorite.note.user.name,
username: favorite.note.user.username,
host: favorite.note.user.host,
uri: favorite.note.user.uri,
},
},
};
}

View file

@ -176,6 +176,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js';
import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_favorites from './endpoints/i/favorites.js';
import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js';
@ -495,6 +496,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass:
const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default }; const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default };
const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default }; const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default };
const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default }; const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default };
const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default };
const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default }; const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default };
const $i_favorites: Provider = { provide: 'ep:i/favorites', useClass: ep___i_favorites.default }; const $i_favorites: Provider = { provide: 'ep:i/favorites', useClass: ep___i_favorites.default };
const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep___i_gallery_likes.default }; const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep___i_gallery_likes.default };
@ -818,6 +820,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$i_exportFollowing, $i_exportFollowing,
$i_exportMute, $i_exportMute,
$i_exportNotes, $i_exportNotes,
$i_exportFavorites,
$i_exportUserLists, $i_exportUserLists,
$i_favorites, $i_favorites,
$i_gallery_likes, $i_gallery_likes,
@ -1135,6 +1138,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$i_exportFollowing, $i_exportFollowing,
$i_exportMute, $i_exportMute,
$i_exportNotes, $i_exportNotes,
$i_exportFavorites,
$i_exportUserLists, $i_exportUserLists,
$i_favorites, $i_favorites,
$i_gallery_likes, $i_gallery_likes,

View file

@ -175,6 +175,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
import * as ep___i_exportFollowing from './endpoints/i/export-following.js'; import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
import * as ep___i_exportMute from './endpoints/i/export-mute.js'; import * as ep___i_exportMute from './endpoints/i/export-mute.js';
import * as ep___i_exportNotes from './endpoints/i/export-notes.js'; import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js'; import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
import * as ep___i_favorites from './endpoints/i/favorites.js'; import * as ep___i_favorites from './endpoints/i/favorites.js';
import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js';
@ -492,6 +493,7 @@ const eps = [
['i/export-following', ep___i_exportFollowing], ['i/export-following', ep___i_exportFollowing],
['i/export-mute', ep___i_exportMute], ['i/export-mute', ep___i_exportMute],
['i/export-notes', ep___i_exportNotes], ['i/export-notes', ep___i_exportNotes],
['i/export-favorites', ep___i_exportFavorites],
['i/export-user-lists', ep___i_exportUserLists], ['i/export-user-lists', ep___i_exportUserLists],
['i/favorites', ep___i_favorites], ['i/favorites', ep___i_favorites],
['i/gallery/likes', ep___i_gallery_likes], ['i/gallery/likes', ep___i_gallery_likes],

View file

@ -0,0 +1,31 @@
import { Inject, Injectable } from '@nestjs/common';
import ms from 'ms';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { QueueService } from '@/core/QueueService.js';
export const meta = {
secure: true,
requireCredential: true,
limit: {
duration: ms('1day'),
max: 1,
},
} as const;
export const paramDef = {
type: 'object',
properties: {},
required: [],
} as const;
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> {
constructor(
private queueService: QueueService,
) {
super(meta, paramDef, async (ps, me) => {
this.queueService.createExportFavoritesJob(me);
});
}
}

View file

@ -8,6 +8,14 @@
<MkButton primary :class="$style.button" inline @click="exportNotes()"><i class="ti ti-download"></i> {{ i18n.ts.export }}</MkButton> <MkButton primary :class="$style.button" inline @click="exportNotes()"><i class="ti ti-download"></i> {{ i18n.ts.export }}</MkButton>
</FormFolder> </FormFolder>
</FormSection> </FormSection>
<FormSection>
<template #label>{{ i18n.ts._exportOrImport.favoritedNotes }}</template>
<FormFolder>
<template #label>{{ i18n.ts.export }}</template>
<template #icon><i class="ti ti-download"></i></template>
<MkButton primary :class="$style.button" inline @click="exportFavorites()"><i class="ti ti-download"></i> {{ i18n.ts.export }}</MkButton>
</FormFolder>
</FormSection>
<FormSection> <FormSection>
<template #label>{{ i18n.ts._exportOrImport.followingList }}</template> <template #label>{{ i18n.ts._exportOrImport.followingList }}</template>
<FormFolder class="_formBlock"> <FormFolder class="_formBlock">
@ -108,6 +116,10 @@ const exportNotes = () => {
os.api('i/export-notes', {}).then(onExportSuccess).catch(onError); os.api('i/export-notes', {}).then(onExportSuccess).catch(onError);
}; };
const exportFavorites = () => {
os.api('i/export-favorites', {}).then(onExportSuccess).catch(onError);
};
const exportFollowing = () => { const exportFollowing = () => {
os.api('i/export-following', { os.api('i/export-following', {
excludeMuting: excludeMutingUsers.value, excludeMuting: excludeMutingUsers.value,