From 5db5bbd1cd25f83640d4dd01de14e7774d9370db Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 5 Feb 2019 19:50:14 +0900 Subject: [PATCH] =?UTF-8?q?=E8=87=AA=E5=88=86=E3=81=AE=E6=8A=95=E7=A8=BF?= =?UTF-8?q?=E6=83=85=E5=A0=B1=E3=82=92=E3=82=A8=E3=82=AF=E3=82=B9=E3=83=9D?= =?UTF-8?q?=E3=83=BC=E3=83=88=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=20(#4144)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * 正しいJSONを生成するように * データを整形 --- locales/ja-JP.yml | 3 + .../views/components/profile-editor.vue | 17 +++ src/queue/index.ts | 18 ++- src/queue/logger.ts | 3 + src/queue/processors/export-notes.ts | 128 ++++++++++++++++++ src/queue/processors/http/deliver.ts | 2 +- src/queue/processors/{http => }/index.ts | 8 +- src/server/api/endpoints/i/export-notes.ts | 18 +++ 8 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 src/queue/logger.ts create mode 100644 src/queue/processors/export-notes.ts rename src/queue/processors/{http => }/index.ts (57%) create mode 100644 src/server/api/endpoints/i/export-notes.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index f1517bf0c1..43f6651b24 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -557,6 +557,9 @@ common/views/components/profile-editor.vue: email-address: "メールアドレス" email-verified: "メールアドレスが確認されました" email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。" + export: "エクスポート" + export-notes: "すべての投稿のエクスポート" + export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。" common/views/components/user-list-editor.vue: users: "ユーザー" diff --git a/src/client/app/common/views/components/profile-editor.vue b/src/client/app/common/views/components/profile-editor.vue index feffd25437..d745e7d291 100644 --- a/src/client/app/common/views/components/profile-editor.vue +++ b/src/client/app/common/views/components/profile-editor.vue @@ -87,6 +87,14 @@ {{ $t('save') }} + +
+
{{ $t('export') }}
+ +
+ {{ $t('export-notes') }} +
+
@@ -252,6 +260,15 @@ export default Vue.extend({ email: this.email == '' ? null : this.email }); }); + }, + + exportNotes() { + this.$root.api('i/export-notes', {}); + + this.$root.dialog({ + type: 'info', + text: this.$t('export-requested') + }); } } }); diff --git a/src/queue/index.ts b/src/queue/index.ts index 65c52d864c..cf8af17a48 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,9 +1,9 @@ import * as Queue from 'bee-queue'; import config from '../config'; -import http from './processors/http'; + import { ILocalUser } from '../models/user'; -import Logger from '../misc/logger'; import { program } from '../argv'; +import handler from './processors'; const enableQueue = config.redis != null && !program.disableQueue; @@ -36,7 +36,7 @@ export function createHttpJob(data: any) { .backoff('exponential', 16384) // 16s .save(); } else { - return http({ data }, () => {}); + return handler({ data }, () => {}); } } @@ -51,10 +51,18 @@ export function deliver(user: ILocalUser, content: any, to: any) { }); } -export const queueLogger = new Logger('queue'); +export function createExportNotesJob(user: ILocalUser) { + if (!enableQueue) throw 'queue disabled'; + + return queue.createJob({ + type: 'exportNotes', + user: user + }) + .save(); +} export default function() { if (enableQueue) { - queue.process(128, http); + queue.process(128, handler); } } diff --git a/src/queue/logger.ts b/src/queue/logger.ts new file mode 100644 index 0000000000..99d88bd63b --- /dev/null +++ b/src/queue/logger.ts @@ -0,0 +1,3 @@ +import Logger from '../misc/logger'; + +export const queueLogger = new Logger('queue', 'orange'); diff --git a/src/queue/processors/export-notes.ts b/src/queue/processors/export-notes.ts new file mode 100644 index 0000000000..52845a5a9c --- /dev/null +++ b/src/queue/processors/export-notes.ts @@ -0,0 +1,128 @@ +import * as bq from 'bee-queue'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as mongo from 'mongodb'; + +import { queueLogger } from '../logger'; +import Note, { INote } from '../../models/note'; +import addFile from '../../services/drive/add-file'; +import User from '../../models/user'; +import dateFormat = require('dateformat'); + +const logger = queueLogger.createSubLogger('export-notes'); + +export async function exportNotes(job: bq.Job, done: any): Promise { + logger.info(`Exporting notes of ${job.data.user._id} ...`); + + const user = await User.findOne({ + _id: new mongo.ObjectID(job.data.user._id.toString()) + }); + + // Create temp file + const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp file is ${path}`); + + const stream = fs.createWriteStream(path, { flags: 'a' }); + + await new Promise((res, rej) => { + stream.write('[', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + let exportedNotesCount = 0; + let ended = false; + let cursor: any = null; + + while (!ended) { + const notes = await Note.find({ + userId: user._id, + ...(cursor ? { _id: { $gt: cursor } } : {}) + }, { + limit: 100, + sort: { + _id: 1 + } + }); + + if (notes.length === 0) { + ended = true; + job.reportProgress(100); + break; + } + + cursor = notes[notes.length - 1]._id; + + for (const note of notes) { + const content = JSON.stringify(serialize(note)); + await new Promise((res, rej) => { + stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + exportedNotesCount++; + } + + const total = await Note.count({ + userId: user._id, + }); + + job.reportProgress(exportedNotesCount / total); + } + + await new Promise((res, rej) => { + stream.write(']', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + stream.end(); + logger.succ(`Exported to: ${path}`); + + const fileName = dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json'; + const driveFile = await addFile(user, path, fileName); + + logger.succ(`Exported to: ${driveFile._id}`); + cleanup(); + done(); +} + +function serialize(note: INote): any { + return { + id: note._id, + text: note.text, + createdAt: note.createdAt, + fileIds: note.fileIds, + replyId: note.replyId, + renoteId: note.renoteId, + poll: note.poll, + cw: note.cw, + viaMobile: note.viaMobile, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + appId: note.appId, + geo: note.geo, + localOnly: note.localOnly + }; +} diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index d8d90a2773..d1dad55cd7 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -1,7 +1,7 @@ import * as bq from 'bee-queue'; import request from '../../../remote/activitypub/request'; -import { queueLogger } from '../..'; +import { queueLogger } from '../../logger'; export default async (job: bq.Job, done: any): Promise => { try { diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/index.ts similarity index 57% rename from src/queue/processors/http/index.ts rename to src/queue/processors/index.ts index 74ed723bd3..3f08fe29fb 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/index.ts @@ -1,10 +1,12 @@ -import deliver from './deliver'; -import processInbox from './process-inbox'; -import { queueLogger } from '../..'; +import deliver from './http/deliver'; +import processInbox from './http/process-inbox'; +import { exportNotes } from './export-notes'; +import { queueLogger } from '../logger'; const handlers: any = { deliver, processInbox, + exportNotes, }; export default (job: any, done: any) => { diff --git a/src/server/api/endpoints/i/export-notes.ts b/src/server/api/endpoints/i/export-notes.ts new file mode 100644 index 0000000000..6da6e68b9d --- /dev/null +++ b/src/server/api/endpoints/i/export-notes.ts @@ -0,0 +1,18 @@ +import define from '../../define'; +import { createExportNotesJob } from '../../../../queue'; +import ms = require('ms'); + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1day'), + max: 1, + }, +}; + +export default define(meta, (ps, user) => new Promise(async (res, rej) => { + createExportNotesJob(user); + + res(); +}));