From 5458b107743fcb6fa52b9527f08f36342339753f Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 24 Jul 2018 01:58:11 +0900 Subject: [PATCH 1/5] wip --- .config/example.yml | 16 ++++++++ package.json | 2 + src/config/types.ts | 8 ++++ src/services/drive/add-file.ts | 68 ++++++++++++++++++---------------- 4 files changed, 63 insertions(+), 31 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 6c939a4f32..28a850182f 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -53,6 +53,22 @@ remoteDriveCapacityMb: 8 # Users cannot see remote images when they turn off "Show media from a remote server" setting. preventCache: false +drive: + storage: 'db' + + # OR + + # storage: 'object-storage' + # service: 'minio' + # bucket: + # prefix: + # config: + # endPoint: + # port: + # secure: + # accessKey: + # secretKey: + # # Below settings are optional # diff --git a/package.json b/package.json index 845a6166b1..849ff7a62b 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "@types/koa-views": "2.0.3", "@types/koa__cors": "2.2.2", "@types/kue": "0.11.9", + "@types/minio": "6.0.2", "@types/mkdirp": "0.5.2", "@types/mocha": "5.2.3", "@types/mongodb": "3.1.2", @@ -147,6 +148,7 @@ "kue": "0.11.6", "loader-utils": "1.1.0", "mecab-async": "0.1.2", + "minio": "6.0.0", "mkdirp": "0.5.1", "mocha": "5.2.0", "moji": "0.5.1", diff --git a/src/config/types.ts b/src/config/types.ts index c26ff9db9a..b2e28f7306 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -49,6 +49,14 @@ export type Source = { remoteDriveCapacityMb: number; preventCacheRemoteFiles: boolean; + drive?: { + storage: string; + bucket: string; + prefix: string; + service?: string; + config?: any; + }; + /** * ゴーストアカウントのID */ diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index f7c8922b58..a1b9686956 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -8,14 +8,14 @@ import * as _gm from 'gm'; import * as debug from 'debug'; import fileType = require('file-type'); const prominence = require('prominence'); +import * as Minio from 'minio'; +import * as uuid from 'uuid'; import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file'; import DriveFolder from '../../models/drive-folder'; import { pack } from '../../models/drive-file'; import event, { publishDriveStream } from '../../stream'; import { isLocalUser, IUser, IRemoteUser } from '../../models/user'; -import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; -import genThumbnail from '../../drive/gen-thumbnail'; import delFile from './delete-file'; import config from '../../config'; @@ -25,28 +25,43 @@ const gm = _gm.subClass({ const log = debug('misskey:drive:add-file'); -const writeChunks = (name: string, readable: stream.Readable, type: string, metadata: any) => - getDriveFileBucket() - .then(bucket => new Promise((resolve, reject) => { +async function save(readable: stream.Readable, name: string, type: string, hash: string, size: number, metadata: any): Promise { + if (config.drive && config.drive.storage == 'object-storage') { + if (config.drive.service == 'minio') { + + const minio = new Minio.Client(config.drive.config); + const id = uuid.v4(); + const obj = `${config.drive.prefix}/${id}`; + await minio.putObject(config.drive.bucket, obj, readable); + + Object.assign(metadata, { + obj: id, + url: `${ config.drive.config.secure ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? ':' + config.drive.config.port : '' }/${ config.drive.bucket }/${ obj }` + }); + + const file = await DriveFile.insert({ + length: size, + uploadDate: new Date(), + md5: hash, + filename: name, + metadata: metadata, + contentType: type + }); + + return file; + } + } else { + // Get MongoDB GridFS bucket + const bucket = await getDriveFileBucket(); + + return new Promise((resolve, reject) => { const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); writeStream.once('finish', resolve); writeStream.on('error', reject); readable.pipe(writeStream); - })); - -const writeThumbnailChunks = (name: string, readable: stream.Readable, originalId: mongodb.ObjectID) => - getDriveFileThumbnailBucket() - .then(bucket => new Promise((resolve, reject) => { - const writeStream = bucket.openUploadStream(name, { - contentType: 'image/jpeg', - metadata: { - originalId - } - }); - writeStream.once('finish', resolve); - writeStream.on('error', reject); - readable.pipe(writeStream); - })); + }); + } +} async function deleteOldFile(user: IRemoteUser) { const oldFile = await DriveFile.findOne({ @@ -283,7 +298,7 @@ export default async function( metadata: metadata, contentType: mime }) - : await (writeChunks(detectedName, fs.createReadStream(path), mime, metadata) as Promise); + : await (save(fs.createReadStream(path), detectedName, mime, hash, size, metadata)); log(`drive file has been created ${driveFile._id}`); @@ -293,16 +308,7 @@ export default async function( publishDriveStream(user._id, 'file_created', packedFile); }); - if (!metaOnly) { - try { - const thumb = await genThumbnail(driveFile); - if (thumb) { - await writeThumbnailChunks(detectedName, thumb, driveFile._id); - } - } catch (e) { - // noop - } - } + // TODO: サムネイル生成 return driveFile; } From adc3c16ef3bd53a4f1bb84ab7ca0fcc64384fe8f Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 24 Jul 2018 05:04:43 +0900 Subject: [PATCH 2/5] wip --- cli/clean-cached-remote-files.js | 4 ++-- src/models/drive-file.ts | 7 ++++--- src/remote/activitypub/models/person.ts | 8 ++++---- src/server/file/send-drive-file.ts | 2 +- src/services/drive/add-file.ts | 16 +++++++++++----- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/cli/clean-cached-remote-files.js b/cli/clean-cached-remote-files.js index a9c38a4cdf..5b388c73b4 100644 --- a/cli/clean-cached-remote-files.js +++ b/cli/clean-cached-remote-files.js @@ -9,7 +9,7 @@ const q = { 'metadata._user.host': { $ne: null }, - 'metadata.isMetaOnly': false + 'metadata.withoutChunks': false }; async function main() { @@ -57,7 +57,7 @@ async function main() { DriveFile.update({ _id: file._id }, { $set: { - 'metadata.isMetaOnly': true + 'metadata.withoutChunks': true } }) ]).then(async () => { diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index 3a0390f792..0b7212665d 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -31,8 +31,9 @@ export type IMetadata = { comment: string; uri?: string; url?: string; + src?: string; deletedAt?: Date; - isMetaOnly?: boolean; + withoutChunks?: boolean; isSensitive?: boolean; }; @@ -155,9 +156,9 @@ export const pack = ( _target = Object.assign(_target, _file.metadata); + _target.url = _file.metadata.url ? _file.metadata.url : `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`; _target.src = _file.metadata.url; - _target.url = _file.metadata.isMetaOnly ? _file.metadata.url : `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`; - _target.isRemote = _file.metadata.isMetaOnly; + _target.isRemote = _file.metadata.withoutChunks; if (_target.properties == null) _target.properties = {}; diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index eee4aa1bfa..4ff8d23be7 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -152,8 +152,8 @@ export async function createPerson(value: any, resolver?: Resolver): Promise Date: Tue, 24 Jul 2018 06:21:21 +0900 Subject: [PATCH 3/5] wip --- src/models/drive-file.ts | 2 ++ src/services/drive/add-file.ts | 6 ++-- src/services/drive/delete-file.ts | 52 ++++++++++++++++++------------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index 0b7212665d..f197f86d46 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -34,6 +34,8 @@ export type IMetadata = { src?: string; deletedAt?: Date; withoutChunks?: boolean; + storage?: string; + storageProps?: any; isSensitive?: boolean; }; diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 4f379a25be..ab9353c9fc 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -35,9 +35,11 @@ async function save(readable: stream.Readable, name: string, type: string, hash: await minio.putObject(config.drive.bucket, obj, readable); Object.assign(metadata, { - storage: 'object-storage', withoutChunks: true, - obj: id, + storage: 'object-storage', + storageProps: { + id: id + }, url: `${ config.drive.config.secure ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? ':' + config.drive.config.port : '' }/${ config.drive.bucket }/${ obj }` }); diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 4ac60439b7..bf9c80f485 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -1,30 +1,40 @@ +import * as Minio from 'minio'; import DriveFile, { DriveFileChunk, IDriveFile } from '../../models/drive-file'; import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-file-thumbnail'; +import config from '../../config'; export default async function(file: IDriveFile, isExpired = false) { - // チャンクをすべて削除 - await DriveFileChunk.remove({ - files_id: file._id - }); - - await DriveFile.update({ _id: file._id }, { - $set: { - 'metadata.deletedAt': new Date(), - 'metadata.isExpired': isExpired + if (file.metadata.withoutChunks) { + if (file.metadata.storage == 'object-storage') { + const minio = new Minio.Client(config.drive.config); + const obj = `${config.drive.prefix}/${file.metadata.storageProps.id}`; + await minio.removeObject(config.drive.bucket, obj); } - }); - - //#region サムネイルもあれば削除 - const thumbnail = await DriveFileThumbnail.findOne({ - 'metadata.originalId': file._id - }); - - if (thumbnail) { - await DriveFileThumbnailChunk.remove({ - files_id: thumbnail._id + } else { + // チャンクをすべて削除 + await DriveFileChunk.remove({ + files_id: file._id }); - await DriveFileThumbnail.remove({ _id: thumbnail._id }); + await DriveFile.update({ _id: file._id }, { + $set: { + 'metadata.deletedAt': new Date(), + 'metadata.isExpired': isExpired + } + }); + + //#region サムネイルもあれば削除 + const thumbnail = await DriveFileThumbnail.findOne({ + 'metadata.originalId': file._id + }); + + if (thumbnail) { + await DriveFileThumbnailChunk.remove({ + files_id: thumbnail._id + }); + + await DriveFileThumbnail.remove({ _id: thumbnail._id }); + } + //#endregion } - //#endregion } From 5e9fb8bd848d97da2e00f4e7f7fbe5e77e82b9b8 Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 24 Jul 2018 23:35:19 +0900 Subject: [PATCH 4/5] wip --- {migration => cli/migration}/2.0.0.js | 4 +-- {migration => cli/migration}/2.4.0.js | 4 +-- cli/migration/5.0.0.js | 10 ++++++++ migration/README.md | 11 -------- migration/init-migration-file.sh | 37 --------------------------- package.json | 2 +- 6 files changed, 15 insertions(+), 53 deletions(-) rename {migration => cli/migration}/2.0.0.js (88%) rename {migration => cli/migration}/2.4.0.js (91%) create mode 100644 cli/migration/5.0.0.js delete mode 100644 migration/README.md delete mode 100644 migration/init-migration-file.sh diff --git a/migration/2.0.0.js b/cli/migration/2.0.0.js similarity index 88% rename from migration/2.0.0.js rename to cli/migration/2.0.0.js index eb8f5730c7..f7298972e5 100644 --- a/migration/2.0.0.js +++ b/cli/migration/2.0.0.js @@ -3,8 +3,8 @@ const chalk = require('chalk'); const sequential = require('promise-sequential'); -const { default: User } = require('../built/models/user'); -const { default: DriveFile } = require('../built/models/drive-file'); +const { default: User } = require('../../built/models/user'); +const { default: DriveFile } = require('../../built/models/drive-file'); async function main() { const promiseGens = []; diff --git a/migration/2.4.0.js b/cli/migration/2.4.0.js similarity index 91% rename from migration/2.4.0.js rename to cli/migration/2.4.0.js index e9584a1dfc..aa37849aa1 100644 --- a/migration/2.4.0.js +++ b/cli/migration/2.4.0.js @@ -3,8 +3,8 @@ const chalk = require('chalk'); const sequential = require('promise-sequential'); -const { default: User } = require('../built/models/user'); -const { default: DriveFile } = require('../built/models/drive-file'); +const { default: User } = require('../../built/models/user'); +const { default: DriveFile } = require('../../built/models/drive-file'); async function main() { const promiseGens = []; diff --git a/cli/migration/5.0.0.js b/cli/migration/5.0.0.js new file mode 100644 index 0000000000..8c6f49ed85 --- /dev/null +++ b/cli/migration/5.0.0.js @@ -0,0 +1,10 @@ +const { default: DriveFile } = require('../../built/models/drive-file'); + +DriveFile.update({}, { + $rename: { + 'metadata.url': 'metadata.src', + 'metadata.isMetaOnly': 'metadata.withoutChunks', + } +}, { + multi: true +}); diff --git a/migration/README.md b/migration/README.md deleted file mode 100644 index d52e84b35a..0000000000 --- a/migration/README.md +++ /dev/null @@ -1,11 +0,0 @@ -Misskeyの破壊的変更に対応するいくつかのスニペットがあります。 -MongoDBシェルで実行する必要のあるものとnodeで直接実行する必要のあるものがあります。 -ファイル名が `shell.` から始まるものは前者、 `node.` から始まるものは後者です。 - -MongoDBシェルで実行する場合、`use`でデータベースを選択しておく必要があります。 - -nodeで実行するいくつかのスニペットは、並列処理させる数を引数で設定できるものがあります。 -処理中にエラーで落ちる場合は、メモリが足りていない可能性があるので、少ない数に設定してみてください。 -※デフォルトは`5`です。 - -ファイルを作成する際は `../init-migration-file.sh -t _type_ -n _name_` を実行すると _type_._unixtime_._name_.js が生成されます diff --git a/migration/init-migration-file.sh b/migration/init-migration-file.sh deleted file mode 100644 index c6a2b862e6..0000000000 --- a/migration/init-migration-file.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -usage() { - echo "$0 [-t type] [-n name]" - echo " type: [node | shell]" - echo " name: if no present, set untitled" - exit 0 -} - -while getopts :t:n:h OPT -do - case $OPT in - t) type=$OPTARG - ;; - n) name=$OPTARG - ;; - h) usage - ;; - \?) usage - ;; - :) usage - ;; - esac -done - -if [ "$type" = "" ] -then - echo "no type present!!!" - usage -fi - -if [ "$name" = "" ] -then - name="untitled" -fi - -touch "$(realpath $(dirname $BASH_SOURCE))/migration/$type.$(date +%s).$name.js" diff --git a/package.json b/package.json index 849ff7a62b..3ca36923f2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "misskey", "author": "syuilo ", - "version": "4.27.0", + "version": "5.0.0", "clientVersion": "1.0.7487", "codename": "nighthike", "main": "./built/index.js", From 03ce87d710b9194a43370dd1736e38b2dde2aa55 Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 24 Jul 2018 23:43:14 +0900 Subject: [PATCH 5/5] wip --- src/client/app/auth/views/form.vue | 2 +- .../app/common/scripts/compose-notification.ts | 14 +++++++------- .../app/common/views/components/autocomplete.vue | 2 +- src/client/app/common/views/components/avatar.vue | 2 +- .../components/games/reversi/reversi.game.vue | 4 ++-- .../app/common/views/widgets/photo-stream.vue | 2 +- src/client/app/common/views/widgets/slideshow.vue | 2 +- .../app/desktop/views/components/drive.file.vue | 2 +- .../desktop/views/components/followers-window.vue | 2 +- .../desktop/views/components/following-window.vue | 2 +- .../app/desktop/views/components/media-image.vue | 2 +- .../app/desktop/views/components/media-video.vue | 2 +- .../app/desktop/views/components/post-form.vue | 2 +- .../desktop/views/components/settings.profile.vue | 2 +- .../app/desktop/views/components/user-preview.vue | 2 +- .../views/pages/user/user.followers-you-know.vue | 2 +- .../app/desktop/views/pages/user/user.photos.vue | 2 +- src/client/app/desktop/views/widgets/profile.vue | 2 +- .../app/mobile/views/components/drive.file.vue | 2 +- .../app/mobile/views/components/media-image.vue | 2 +- .../app/mobile/views/components/media-video.vue | 2 +- .../app/mobile/views/components/note-card.vue | 2 +- .../app/mobile/views/components/post-form.vue | 2 +- src/client/app/mobile/views/components/ui.nav.vue | 2 +- .../app/mobile/views/components/user-card.vue | 2 +- src/client/app/mobile/views/pages/followers.vue | 2 +- src/client/app/mobile/views/pages/following.vue | 2 +- src/client/app/mobile/views/pages/user.vue | 2 +- .../views/pages/user/home.followers-you-know.vue | 2 +- .../app/mobile/views/pages/user/home.photos.vue | 2 +- src/client/app/mobile/views/widgets/profile.vue | 4 ++-- 31 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/client/app/auth/views/form.vue b/src/client/app/auth/views/form.vue index 152b900429..80086e3861 100644 --- a/src/client/app/auth/views/form.vue +++ b/src/client/app/auth/views/form.vue @@ -2,7 +2,7 @@

{{ app.name }}があなたのアカウントにアクセスすることを許可しますか？

- +
diff --git a/src/client/app/common/scripts/compose-notification.ts b/src/client/app/common/scripts/compose-notification.ts index 47499e5490..c93609bc59 100644 --- a/src/client/app/common/scripts/compose-notification.ts +++ b/src/client/app/common/scripts/compose-notification.ts @@ -17,21 +17,21 @@ export default function(type, data): Notification { return { title: 'ファイルがアップロードされました', body: data.name, - icon: data.url + '?thumbnail&size=64' + icon: data.url }; case 'unread_messaging_message': return { title: `${getUserName(data.user)}さんからメッセージ:`, body: data.text, // TODO: getMessagingMessageSummary(data), - icon: data.user.avatarUrl + '?thumbnail&size=64' + icon: data.user.avatarUrl }; case 'reversi_invited': return { title: '対局への招待があります', body: `${getUserName(data.parent)}さんから`, - icon: data.parent.avatarUrl + '?thumbnail&size=64' + icon: data.parent.avatarUrl }; case 'notification': @@ -40,28 +40,28 @@ export default function(type, data): Notification { return { title: `${getUserName(data.user)}さんから:`, body: getNoteSummary(data), - icon: data.user.avatarUrl + '?thumbnail&size=64' + icon: data.user.avatarUrl }; case 'reply': return { title: `${getUserName(data.user)}さんから返信:`, body: getNoteSummary(data), - icon: data.user.avatarUrl + '?thumbnail&size=64' + icon: data.user.avatarUrl }; case 'quote': return { title: `${getUserName(data.user)}さんが引用:`, body: getNoteSummary(data), - icon: data.user.avatarUrl + '?thumbnail&size=64' + icon: data.user.avatarUrl }; case 'reaction': return { title: `${getUserName(data.user)}: ${getReactionEmoji(data.reaction)}:`, body: getNoteSummary(data.note), - icon: data.user.avatarUrl + '?thumbnail&size=64' + icon: data.user.avatarUrl }; default: diff --git a/src/client/app/common/views/components/autocomplete.vue b/src/client/app/common/views/components/autocomplete.vue index 4a130a04ec..cd6066877c 100644 --- a/src/client/app/common/views/components/autocomplete.vue +++ b/src/client/app/common/views/components/autocomplete.vue @@ -2,7 +2,7 @@
  1. - + {{ user | userName }} @{{ user | acct }}
    2. diff --git a/src/client/app/common/views/components/avatar.vue b/src/client/app/common/views/components/avatar.vue index a65b62882f..a924b62e64 100644 --- a/src/client/app/common/views/components/avatar.vue +++ b/src/client/app/common/views/components/avatar.vue @@ -31,7 +31,7 @@ export default Vue.extend({ : this.user.avatarColor && this.user.avatarColor.length == 3 ? `rgb(${ this.user.avatarColor.join(',') })` : null, - backgroundImage: this.lightmode ? null : `url(${ this.user.avatarUrl }?thumbnail)`, + backgroundImage: this.lightmode ? null : `url(${ this.user.avatarUrl })`, borderRadius: this.$store.state.settings.circleIcons ? '100%' : null }; } diff --git a/src/client/app/common/views/components/games/reversi/reversi.game.vue b/src/client/app/common/views/components/games/reversi/reversi.game.vue index ed9e718497..303070ffd8 100644 --- a/src/client/app/common/views/components/games/reversi/reversi.game.vue +++ b/src/client/app/common/views/components/games/reversi/reversi.game.vue @@ -26,8 +26,8 @@ :class="{ empty: stone == null, none: o.map[i] == 'null', isEnded: game.isEnded, myTurn: !game.isEnded && isMyTurn, can: turnUser ? o.canPut(turnUser.id == blackUser.id, i) : null, prev: o.prevPos == i }" @click="set(i)" :title="`${String.fromCharCode(65 + o.transformPosToXy(i)[0])}${o.transformPosToXy(i)[1] + 1}`"> - - + +
diff --git a/src/client/app/common/views/widgets/photo-stream.vue b/src/client/app/common/views/widgets/photo-stream.vue index ae5924bb10..3e24c58e8e 100644 --- a/src/client/app/common/views/widgets/photo-stream.vue +++ b/src/client/app/common/views/widgets/photo-stream.vue @@ -5,7 +5,7 @@

%fa:spinner .pulse .fw%%i18n:common.loading%

-
+

%i18n:@no-photos%

diff --git a/src/client/app/common/views/widgets/slideshow.vue b/src/client/app/common/views/widgets/slideshow.vue index e1c28f5115..2eede03786 100644 --- a/src/client/app/common/views/widgets/slideshow.vue +++ b/src/client/app/common/views/widgets/slideshow.vue @@ -72,7 +72,7 @@ export default define({ if (this.images.length == 0) return; const index = Math.floor(Math.random() * this.images.length); - const img = `url(${ this.images[index].url }?thumbnail&size=1024)`; + const img = `url(${ this.images[index].url })`; (this.$refs.slideB as any).style.backgroundImage = img; diff --git a/src/client/app/desktop/views/components/drive.file.vue b/src/client/app/desktop/views/components/drive.file.vue index 11700d4966..e1db36f5cf 100644 --- a/src/client/app/desktop/views/components/drive.file.vue +++ b/src/client/app/desktop/views/components/drive.file.vue @@ -16,7 +16,7 @@

%i18n:@banner%

- +

{{ file.name.lastIndexOf('.') != -1 ? file.name.substr(0, file.name.lastIndexOf('.')) : file.name }} diff --git a/src/client/app/desktop/views/components/followers-window.vue b/src/client/app/desktop/views/components/followers-window.vue index 7ed31315f1..fdab7bc1ce 100644 --- a/src/client/app/desktop/views/components/followers-window.vue +++ b/src/client/app/desktop/views/components/followers-window.vue @@ -1,7 +1,7 @@