Improve drive management

This commit is contained in:
syuilo 2019-05-27 16:54:47 +09:00
parent 9403ee6495
commit 72fb23f4d5
No known key found for this signature in database
GPG Key ID: BDC4C49D06AB9D69
14 changed files with 152 additions and 54 deletions

View File

@ -1417,6 +1417,9 @@ admin/views/drive.vue:
unmark-as-sensitive: "閲覧注意を解除"
marked-as-sensitive: "閲覧注意に設定しました"
unmarked-as-sensitive: "閲覧注意を解除しました"
clean-remote-files: "リモートファイルのキャッシュを削除"
clean-remote-files-are-you-sure: "すべてのリモートファイルのキャッシュを削除してもよろしいですか?"
clean-up: "クリーンアップ"
admin/views/users.vue:
operation: "操作"

View File

@ -14,6 +14,10 @@
<ui-button @click="show()"><fa :icon="faSearch"/> {{ $t('lookup') }}</ui-button>
<ui-textarea v-if="file" :value="file | json5" readonly tall style="margin-top:16px;"></ui-textarea>
</section>
<section>
<ui-button @click="cleanUp()"><fa :icon="faTrashAlt"/> {{ $t('clean-up') }}</ui-button>
<ui-button @click="cleanRemoteFiles()"><fa :icon="faTrashAlt"/> {{ $t('clean-remote-files') }}</ui-button>
</section>
</ui-card>
<ui-card>
@ -227,6 +231,29 @@ export default Vue.extend({
});
});
},
cleanRemoteFiles() {
this.$root.dialog({
type: 'warning',
text: this.$t('clean-remote-files-are-you-sure'),
showCancelButton: true
}).then(({ canceled }) => {
if (canceled) return;
this.$root.api('admin/drive/clean-remote-files');
this.$root.dialog({
type: 'success',
splash: true
});
});
},
cleanUp() {
this.$root.api('admin/drive/cleanup');
this.$root.dialog({
type: 'success',
splash: true
});
}
}
});
</script>

View File

@ -8,6 +8,7 @@ import { program } from '../argv';
import processDeliver from './processors/deliver';
import processInbox from './processors/inbox';
import processDb from './processors/db';
import procesObjectStorage from './processors/object-storage';
import { queueLogger } from './logger';
import { DriveFile } from '../models/entities/drive-file';
@ -34,9 +35,12 @@ function renderError(e: Error): any {
export const deliverQueue = initializeQueue('deliver');
export const inboxQueue = initializeQueue('inbox');
export const dbQueue = initializeQueue('db');
export const objectStorageQueue = initializeQueue('objectStorage');
const deliverLogger = queueLogger.createSubLogger('deliver');
const inboxLogger = queueLogger.createSubLogger('inbox');
const dbLogger = queueLogger.createSubLogger('db');
const objectStorageLogger = queueLogger.createSubLogger('objectStorage');
deliverQueue
.on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`))
@ -54,6 +58,22 @@ inboxQueue
.on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) }))
.on('stalled', (job) => inboxLogger.warn(`stalled id=${job.id} activity=${job.data.activity ? job.data.activity.id : 'none'}`));
dbQueue
.on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`))
.on('active', (job) => dbLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
.on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) }))
.on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`));
objectStorageQueue
.on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`))
.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
.on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
.on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) }))
.on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`));
export function deliver(user: ILocalUser, content: any, to: any) {
if (content == null) return null;
@ -165,11 +185,21 @@ export function createImportUserListsJob(user: ILocalUser, fileId: DriveFile['id
});
}
export function createDeleteObjectStorageFileJob(key: string) {
return objectStorageQueue.add('deleteFile', {
key: key
}, {
removeOnComplete: true,
removeOnFail: true
});
}
export default function() {
if (!program.onlyServer) {
deliverQueue.process(128, processDeliver);
inboxQueue.process(128, processInbox);
processDb(dbQueue);
procesObjectStorage(objectStorageQueue);
}
}

View File

@ -1,7 +1,7 @@
import * as Bull from 'bull';
import { queueLogger } from '../../logger';
import deleteFile from '../../../services/drive/delete-file';
import { deleteFile } from '../../../services/drive/delete-file';
import { Users, DriveFiles } from '../../../models';
import { MoreThan } from 'typeorm';

View File

@ -0,0 +1,22 @@
import * as Bull from 'bull';
import * as Minio from 'minio';
import { fetchMeta } from '../../../misc/fetch-meta';
export default async (job: Bull.Job) => {
const meta = await fetchMeta();
const minio = new Minio.Client({
endPoint: meta.objectStorageEndpoint!,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
useSSL: meta.objectStorageUseSSL,
accessKey: meta.objectStorageAccessKey!,
secretKey: meta.objectStorageSecretKey!,
});
const key: string = job.data.key;
await minio.removeObject(meta.objectStorageBucket!, key);
return 'Success';
};

View File

@ -0,0 +1,12 @@
import * as Bull from 'bull';
import deleteFile from './delete-file';
const jobs = {
deleteFile,
} as any;
export default function(q: Bull.Queue) {
for (const [k, v] of Object.entries(jobs)) {
q.process(k, v as any);
}
}

View File

@ -1,6 +1,6 @@
import $ from 'cafy';
import define from '../../define';
import del from '../../../../services/drive/delete-file';
import { deleteFile } from '../../../../services/drive/delete-file';
import { DriveFiles } from '../../../../models';
import { ID } from '../../../../misc/cafy-id';
@ -27,6 +27,6 @@ export default define(meta, async (ps, me) => {
});
for (const file of files) {
del(file);
deleteFile(file);
}
});

View File

@ -0,0 +1,21 @@
import { Not, IsNull } from 'typeorm';
import define from '../../../define';
import { deleteFile } from '../../../../../services/drive/delete-file';
import { DriveFiles } from '../../../../../models';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireModerator: true,
};
export default define(meta, async (ps, me) => {
const files = await DriveFiles.find({
userHost: Not(IsNull())
});
for (const file of files) {
deleteFile(file, true);
}
});

View File

@ -0,0 +1,21 @@
import { IsNull } from 'typeorm';
import define from '../../../define';
import { deleteFile } from '../../../../../services/drive/delete-file';
import { DriveFiles } from '../../../../../models';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireModerator: true,
};
export default define(meta, async (ps, me) => {
const files = await DriveFiles.find({
userId: IsNull()
});
for (const file of files) {
deleteFile(file);
}
});

View File

@ -1,6 +1,6 @@
import $ from 'cafy';
import define from '../../../define';
import del from '../../../../../services/drive/delete-file';
import { deleteFile } from '../../../../../services/drive/delete-file';
import { DriveFiles } from '../../../../../models';
export const meta = {
@ -22,6 +22,6 @@ export default define(meta, async (ps, me) => {
});
for (const file of files) {
del(file);
deleteFile(file);
}
});

View File

@ -1,6 +1,6 @@
import $ from 'cafy';
import { ID } from '../../../../../misc/cafy-id';
import del from '../../../../../services/drive/delete-file';
import { deleteFile } from '../../../../../services/drive/delete-file';
import { publishDriveStream } from '../../../../../services/stream';
import define from '../../../define';
import { ApiError } from '../../../error';
@ -57,7 +57,7 @@ export default define(meta, async (ps, user) => {
}
// Delete
await del(file);
await deleteFile(file);
// Publish fileDeleted event
publishDriveStream(user.id, 'fileDeleted', file.id);

View File

@ -7,7 +7,7 @@ import * as uuid from 'uuid';
import * as sharp from 'sharp';
import { publishMainStream, publishDriveStream } from '../stream';
import delFile from './delete-file';
import { deleteFile } from './delete-file';
import { fetchMeta } from '../../misc/fetch-meta';
import { GenerateVideoThumbnail } from './generate-video-thumbnail';
import { driveLogger } from './logger';
@ -233,7 +233,7 @@ async function deleteOldFile(user: IRemoteUser) {
const oldFile = await q.getOne();
if (oldFile) {
delFile(oldFile, true);
deleteFile(oldFile, true);
}
}

View File

@ -1,11 +1,10 @@
import * as Minio from 'minio';
import { DriveFile } from '../../models/entities/drive-file';
import { InternalStorage } from './internal-storage';
import { DriveFiles, Instances, Notes } from '../../models';
import { driveChart, perUserDriveChart, instanceChart } from '../chart';
import { fetchMeta } from '../../misc/fetch-meta';
import { createDeleteObjectStorageFileJob } from '../../queue';
export default async function(file: DriveFile, isExpired = false) {
export async function deleteFile(file: DriveFile, isExpired = false) {
if (file.storedInternal) {
InternalStorage.del(file.accessKey!);
@ -17,25 +16,14 @@ export default async function(file: DriveFile, isExpired = false) {
InternalStorage.del(file.webpublicAccessKey!);
}
} else if (!file.isLink) {
const meta = await fetchMeta();
const minio = new Minio.Client({
endPoint: meta.objectStorageEndpoint!,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
useSSL: meta.objectStorageUseSSL,
accessKey: meta.objectStorageAccessKey!,
secretKey: meta.objectStorageSecretKey!,
});
await minio.removeObject(meta.objectStorageBucket!, file.accessKey!);
createDeleteObjectStorageFileJob(file.accessKey!);
if (file.thumbnailUrl) {
await minio.removeObject(meta.objectStorageBucket!, file.thumbnailAccessKey!);
createDeleteObjectStorageFileJob(file.thumbnailAccessKey!);
}
if (file.webpublicUrl) {
await minio.removeObject(meta.objectStorageBucket!, file.webpublicAccessKey!);
createDeleteObjectStorageFileJob(file.webpublicAccessKey!);
}
}
@ -44,8 +32,8 @@ export default async function(file: DriveFile, isExpired = false) {
DriveFiles.update(file.id, {
isLink: true,
url: file.uri,
thumbnailUrl: null,
webpublicUrl: null
thumbnailUrl: file.uri,
webpublicUrl: file.uri
});
} else {
DriveFiles.delete(file.id);

View File

@ -1,26 +0,0 @@
import * as promiseLimit from 'promise-limit';
import del from '../services/drive/delete-file';
import { DriveFiles } from '../models';
import { Not, IsNull } from 'typeorm';
import { DriveFile } from '../models/entities/drive-file';
import { ensure } from '../prelude/ensure';
const limit = promiseLimit(16);
DriveFiles.find({
userHost: Not(IsNull())
}).then(async files => {
console.log(`there is ${files.length} files`);
await Promise.all(files.map(file => limit(() => job(file))));
console.log('ALL DONE');
});
async function job(file: DriveFile): Promise<any> {
file = await DriveFiles.findOne(file.id).then(ensure);
await del(file, true);
console.log('done', file.id);
}