From 335200c31e0c1f623f3b16d277ef14ffcab64384 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 18 Aug 2018 23:56:44 +0900 Subject: [PATCH] wip --- cli/clean-cached-remote-files.js | 101 ------------------------------ cli/clean-unused-drive-files.js | 80 ----------------------- docs/manage.en.md | 11 ---- docs/manage.ja.md | 11 ---- src/services/drive/add-file.ts | 4 +- src/services/drive/delete-file.ts | 4 ++ src/services/note/create.ts | 4 ++ src/services/note/delete.ts | 4 ++ src/services/update-chart.ts | 4 +- 9 files changed, 17 insertions(+), 206 deletions(-) delete mode 100644 cli/clean-cached-remote-files.js delete mode 100644 cli/clean-unused-drive-files.js diff --git a/cli/clean-cached-remote-files.js b/cli/clean-cached-remote-files.js deleted file mode 100644 index 5b388c73b4..0000000000 --- a/cli/clean-cached-remote-files.js +++ /dev/null @@ -1,101 +0,0 @@ -const chalk = require('chalk'); -const log = require('single-line-log').stdout; -const sequential = require('promise-sequential'); -const { default: DriveFile, DriveFileChunk } = require('../built/models/drive-file'); -const { default: DriveFileThumbnail, DriveFileThumbnailChunk } = require('../built/models/drive-file-thumbnail'); -const { default: User } = require('../built/models/user'); - -const q = { - 'metadata._user.host': { - $ne: null - }, - 'metadata.withoutChunks': false -}; - -async function main() { - const promiseGens = []; - - const count = await DriveFile.count(q); - - let prev; - - for (let i = 0; i < count; i++) { - promiseGens.push(() => { - const promise = new Promise(async (res, rej) => { - const file = await DriveFile.findOne(prev ? Object.assign({ - _id: { $lt: prev._id } - }, q) : q, { - sort: { - _id: -1 - } - }); - - prev = file; - - function skip() { - res([i, file, false]); - } - - if (file == null) return skip(); - - log(chalk`{gray ${i}} scanning {bold ${file._id}} ${file.filename} ...`); - - const attachingUsersCount = await User.count({ - $or: [{ - avatarId: file._id - }, { - bannerId: file._id - }] - }, { limit: 1 }); - if (attachingUsersCount !== 0) return skip(); - - Promise.all([ - // チャンクをすべて削除 - DriveFileChunk.remove({ - files_id: file._id - }), - - DriveFile.update({ _id: file._id }, { - $set: { - 'metadata.withoutChunks': true - } - }) - ]).then(async () => { - res([i, file, true]); - - //#region サムネイルもあれば削除 - const thumbnail = await DriveFileThumbnail.findOne({ - 'metadata.originalId': file._id - }); - - if (thumbnail) { - DriveFileThumbnailChunk.remove({ - files_id: thumbnail._id - }); - - DriveFileThumbnail.remove({ _id: thumbnail._id }); - } - //#endregion - }); - }); - - promise.then(([i, file, deleted]) => { - if (deleted) { - log(chalk`{gray ${i}} {red deleted: {bold ${file._id}} ${file.filename}}`); - } else { - log(chalk`{gray ${i}} {green skipped: {bold ${file._id}} ${file.filename}}`); - } - log.clear(); - console.log(); - }); - - return promise; - }); - } - - return await sequential(promiseGens); -} - -main().then(() => { - console.log('ALL DONE'); -}).catch(console.error); diff --git a/cli/clean-unused-drive-files.js b/cli/clean-unused-drive-files.js deleted file mode 100644 index 87b158b9ee..0000000000 --- a/cli/clean-unused-drive-files.js +++ /dev/null @@ -1,80 +0,0 @@ -const chalk = require('chalk'); -const log = require('single-line-log').stdout; -const sequential = require('promise-sequential'); -const { default: DriveFile, deleteDriveFile } = require('../built/models/drive-file'); -const { default: Note } = require('../built/models/note'); -const { default: MessagingMessage } = require('../built/models/messaging-message'); -const { default: User } = require('../built/models/user'); - -async function main() { - const promiseGens = []; - - const count = await DriveFile.count({}); - - let prev; - - for (let i = 0; i < count; i++) { - promiseGens.push(() => { - const promise = new Promise(async (res, rej) => { - const file = await DriveFile.findOne(prev ? { - _id: { $lt: prev._id } - } : {}, { - sort: { - _id: -1 - } - }); - - prev = file; - - function skip() { - res([i, file, false]); - } - - if (file == null) return skip(); - - log(chalk`{gray ${i}} scanning {bold ${file._id}} ${file.filename} ...`); - - const attachingUsersCount = await User.count({ - $or: [{ - avatarId: file._id - }, { - bannerId: file._id - }] - }, { limit: 1 }); - if (attachingUsersCount !== 0) return skip(); - - const attachingNotesCount = await Note.count({ - mediaIds: file._id - }, { limit: 1 }); - if (attachingNotesCount !== 0) return skip(); - - const attachingMessagesCount = await MessagingMessage.count({ - fileId: file._id - }, { limit: 1 }); - if (attachingMessagesCount !== 0) return skip(); - - deleteDriveFile(file).then(() => { - res([i, file, true]); - }).catch(rej); - }); - - promise.then(([i, file, deleted]) => { - if (deleted) { - log(chalk`{gray ${i}} {red deleted: {bold ${file._id}} ${file.filename}}`); - } else { - log(chalk`{gray ${i}} {green skipped: {bold ${file._id}} ${file.filename}}`); - } - log.clear(); - console.log(); - }); - - return promise; - }); - } - - return await sequential(promiseGens); -} - -main().then(() => { - console.log('done'); -}).catch(console.error); diff --git a/docs/manage.en.md b/docs/manage.en.md index a7296ce479..713070517c 100644 --- a/docs/manage.en.md +++ b/docs/manage.en.md @@ -33,14 +33,3 @@ node cli/suspend @syuilo@misskey.xyz ``` shell node cli/reset-password (User-ID or Username) ``` - -## Clean up cached remote files -``` shell -node cli/clean-cached-remote-files -``` - -## Clean up unused drive files -``` shell -node cli/clean-unused-drive-files -``` -> We recommend that you announce a user that unused drive files will be deleted before performing this operation, as it may delete the user's important files. diff --git a/docs/manage.ja.md b/docs/manage.ja.md index f289037ad3..897fae7ec2 100644 --- a/docs/manage.ja.md +++ b/docs/manage.ja.md @@ -33,14 +33,3 @@ node cli/suspend @syuilo@misskey.xyz ``` shell node cli/reset-password (ユーザーID または ユーザー名) ``` - -## キャッシュされたリモートファイルをクリーンアップする -``` shell -node cli/clean-cached-remote-files -``` - -## 使われていないドライブのファイルをクリーンアップする -``` shell -node cli/clean-unused-drive-files -``` -> ユーザーの大事なファイルを削除する可能性があるので、この操作を実行する前にユーザーに告知することをお勧めします。 diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index da0d3fd82f..b090d56cee 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -17,6 +17,7 @@ import { isLocalUser, IUser, IRemoteUser } from '../../models/user'; import delFile from './delete-file'; import config from '../../config'; import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; +import { updateDriveStats } from '../update-chart'; const log = debug('misskey:drive:add-file'); @@ -377,7 +378,8 @@ export default async function( publishDriveStream(user._id, 'file_created', packedFile); }); - // TODO: サムネイル生成 + // 統計を更新 + updateDriveStats(driveFile, true); return driveFile; } diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 445d231d66..73532a2953 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -2,6 +2,7 @@ import * as Minio from 'minio'; import DriveFile, { DriveFileChunk, IDriveFile } from '../../models/drive-file'; import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-file-thumbnail'; import config from '../../config'; +import { updateDriveStats } from '../update-chart'; export default async function(file: IDriveFile, isExpired = false) { if (file.metadata.storage == 'minio') { @@ -45,4 +46,7 @@ export default async function(file: IDriveFile, isExpired = false) { await DriveFileThumbnail.remove({ _id: thumbnail._id }); } //#endregion + + // 統計を更新 + updateDriveStats(file, false); } diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 521750dc84..d8f0f57b63 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -23,6 +23,7 @@ import registerHashtag from '../register-hashtag'; import isQuote from '../../misc/is-quote'; import { TextElementMention } from '../../mfm/parse/elements/mention'; import { TextElementHashtag } from '../../mfm/parse/elements/hashtag'; +import { updateNoteStats } from '../update-chart'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -142,6 +143,9 @@ export default async (user: IUser, data: Option, silent = false) => new Promise< return; } + // 統計を更新 + updateNoteStats(note, true); + // ハッシュタグ登録 tags.map(tag => registerHashtag(user, tag)); diff --git a/src/services/note/delete.ts b/src/services/note/delete.ts index 7f245958b0..d444b13a8b 100644 --- a/src/services/note/delete.ts +++ b/src/services/note/delete.ts @@ -6,6 +6,7 @@ import pack from '../../remote/activitypub/renderer'; import { deliver } from '../../queue'; import Following from '../../models/following'; import renderNote from '../../remote/activitypub/renderer/note'; +import { updateNoteStats } from '../update-chart'; /** * 投稿を削除します。 @@ -43,4 +44,7 @@ export default async function(user: IUser, note: INote) { }); } //#endregion + + // 統計を更新 + updateNoteStats(note, false); } diff --git a/src/services/update-chart.ts b/src/services/update-chart.ts index 64ba9e53cf..9175d61f74 100644 --- a/src/services/update-chart.ts +++ b/src/services/update-chart.ts @@ -201,13 +201,13 @@ export async function updateNoteStats(note: INote, isAdditional: boolean) { await update(inc); } -export async function updateDriveStats(user: IUser, file: IDriveFile, isAdditional: boolean) { +export async function updateDriveStats(file: IDriveFile, isAdditional: boolean) { const inc = {} as any; const amount = isAdditional ? 1 : -1; const size = isAdditional ? file.length : -file.length; - if (isLocalUser(user)) { + if (isLocalUser(file.metadata._user)) { inc['drive.local.totalCount'] = amount; inc['drive.local.diffCount'] = amount; inc['drive.local.totalSize'] = size;