From b5c235f8375c9e6ae59c1421103039882470797d Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 28 Feb 2019 15:20:40 +0900 Subject: [PATCH 1/8] Refactor drive --- src/misc/get-drive-file-url.ts | 31 ------------------ src/models/drive-file.ts | 25 +++----------- src/remote/activitypub/models/person.ts | 9 +++--- src/remote/activitypub/renderer/document.ts | 3 +- src/remote/activitypub/renderer/image.ts | 3 +- src/server/api/endpoints/drive/files/show.ts | 34 ++++++-------------- src/server/api/endpoints/i/update.ts | 7 ++-- src/server/file/index.ts | 3 +- src/server/file/send-drive-file.ts | 34 +++++++++----------- src/server/web/feed.ts | 5 ++- src/services/drive/add-file.ts | 9 ++++-- 11 files changed, 47 insertions(+), 116 deletions(-) delete mode 100644 src/misc/get-drive-file-url.ts diff --git a/src/misc/get-drive-file-url.ts b/src/misc/get-drive-file-url.ts deleted file mode 100644 index 067db8a5d..000000000 --- a/src/misc/get-drive-file-url.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { IDriveFile } from '../models/drive-file'; -import config from '../config'; - -export default function(file: IDriveFile, thumbnail = false): string { - if (file == null) return null; - - const isImage = file.contentType && file.contentType.startsWith('image/'); - - if (file.metadata.withoutChunks) { - if (thumbnail) { - return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || (isImage ? file.metadata.url : null); - } else { - return file.metadata.webpublicUrl || file.metadata.url; - } - } else { - if (thumbnail) { - return `${config.driveUrl}/${file._id}?thumbnail`; - } else { - return `${config.driveUrl}/${file._id}?web`; - } - } -} - -export function getOriginalUrl(file: IDriveFile) { - if (file.metadata && file.metadata.url) { - return file.metadata.url; - } - - const accessKey = file.metadata ? file.metadata.accessKey : null; - return `${config.driveUrl}/${file._id}${accessKey ? '?original=' + accessKey : ''}`; -} diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index f3e21f209..2b1f59fb6 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -4,7 +4,6 @@ import { pack as packFolder } from './drive-folder'; import { pack as packUser } from './user'; import monkDb, { nativeDbConn, dbLogger } from '../db/mongodb'; import isObjectId from '../misc/is-objectid'; -import getDriveFileUrl, { getOriginalUrl } from '../misc/get-drive-file-url'; const DriveFile = monkDb.get('driveFiles.files'); DriveFile.createIndex('md5'); @@ -36,26 +35,10 @@ export type IMetadata = { */ uri?: string; - /** - * URL for web(生成されている場合) or original - * * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ - */ - url?: string; - - /** - * URL for thumbnail (thumbnailがなければなし) - * * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ - */ + url: string; thumbnailUrl?: string; - - /** - * URL for original (web用が生成されてない場合はurlがoriginalを指す) - * * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ - */ webpublicUrl?: string; - accessKey?: string; - src?: string; deletedAt?: Date; @@ -188,8 +171,8 @@ export const pack = ( _target = Object.assign(_target, _file.metadata); - _target.url = getDriveFileUrl(_file); - _target.thumbnailUrl = getDriveFileUrl(_file, true); + _target.url = _file.metadata.webpublicUrl; + _target.thumbnailUrl = _file.metadata.thumbnailUrl; _target.isRemote = _file.metadata.isRemote; if (_target.properties == null) _target.properties = {}; @@ -224,7 +207,7 @@ export const pack = ( delete _target._user; if (opts.self) { - _target.url = getOriginalUrl(_file); + _target.url = _file.metadata.url; } resolve(_target); diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index c2b99dfb0..7749468b9 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -16,7 +16,6 @@ import { URL } from 'url'; import { resolveNote, extractEmojis } from './note'; import { registerOrFetchInstanceDoc } from '../../../services/register-or-fetch-instance-doc'; import Instance from '../../../models/instance'; -import getDriveFileUrl from '../../../misc/get-drive-file-url'; import { IEmoji } from '../../../models/emoji'; import { ITag, extractHashtags } from './tag'; import Following from '../../../models/following'; @@ -227,8 +226,8 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise ({ type: 'Document', mediaType: file.contentType, - url: getDriveFileUrl(file) + url: file.metadata.webpublicUrl }); diff --git a/src/remote/activitypub/renderer/image.ts b/src/remote/activitypub/renderer/image.ts index ec637b952..e4b2eed25 100644 --- a/src/remote/activitypub/renderer/image.ts +++ b/src/remote/activitypub/renderer/image.ts @@ -1,8 +1,7 @@ import { IDriveFile } from '../../../models/drive-file'; -import getDriveFileUrl from '../../../misc/get-drive-file-url'; export default (file: IDriveFile) => ({ type: 'Image', - url: getDriveFileUrl(file), + url: file.metadata.webpublicUrl, sensitive: file.metadata.isSensitive }); diff --git a/src/server/api/endpoints/drive/files/show.ts b/src/server/api/endpoints/drive/files/show.ts index 6d63a8605..c0159af72 100644 --- a/src/server/api/endpoints/drive/files/show.ts +++ b/src/server/api/endpoints/drive/files/show.ts @@ -1,9 +1,7 @@ import $ from 'cafy'; -import * as mongo from 'mongodb'; import ID, { transform } from '../../../../../misc/cafy-id'; import DriveFile, { pack, IDriveFile } from '../../../../../models/drive-file'; import define from '../../../define'; -import config from '../../../../../config'; import { ApiError } from '../../../error'; export const meta = { @@ -73,28 +71,16 @@ export default define(meta, async (ps, user) => { 'metadata.deletedAt': { $exists: false } }); } else if (ps.url) { - const isInternalStorageUrl = ps.url.startsWith(config.driveUrl); - if (isInternalStorageUrl) { - // Extract file ID from url - // e.g. - // http://misskey.local/files/foo?original=bar --> foo - const fileId = new mongo.ObjectID(ps.url.replace(config.driveUrl, '').replace(/\?(.*)$/, '').replace(/\//g, '')); - file = await DriveFile.findOne({ - _id: fileId, - 'metadata.deletedAt': { $exists: false } - }); - } else { - file = await DriveFile.findOne({ - $or: [{ - 'metadata.url': ps.url - }, { - 'metadata.webpublicUrl': ps.url - }, { - 'metadata.thumbnailUrl': ps.url - }], - 'metadata.deletedAt': { $exists: false } - }); - } + file = await DriveFile.findOne({ + $or: [{ + 'metadata.url': ps.url + }, { + 'metadata.webpublicUrl': ps.url + }, { + 'metadata.thumbnailUrl': ps.url + }], + 'metadata.deletedAt': { $exists: false } + }); } else { throw new ApiError(meta.errors.fileIdOrUrlRequired); } diff --git a/src/server/api/endpoints/i/update.ts b/src/server/api/endpoints/i/update.ts index 099ef3399..1eef7dc93 100644 --- a/src/server/api/endpoints/i/update.ts +++ b/src/server/api/endpoints/i/update.ts @@ -6,7 +6,6 @@ import DriveFile from '../../../../models/drive-file'; import acceptAllFollowRequests from '../../../../services/following/requests/accept-all'; import { publishToFollowers } from '../../../../services/i/update'; import define from '../../define'; -import getDriveFileUrl from '../../../../misc/get-drive-file-url'; import { parse, parsePlain } from '../../../../mfm/parse'; import extractEmojis from '../../../../misc/extract-emojis'; import extractHashtags from '../../../../misc/extract-hashtags'; @@ -195,7 +194,7 @@ export default define(meta, async (ps, user, app) => { if (avatar.metadata.deletedAt) { updates.avatarUrl = null; } else { - updates.avatarUrl = getDriveFileUrl(avatar, true); + updates.avatarUrl = avatar.metadata.thumbnailUrl; if (avatar.metadata.properties.avgColor) { updates.avatarColor = avatar.metadata.properties.avgColor; @@ -214,7 +213,7 @@ export default define(meta, async (ps, user, app) => { if (banner.metadata.deletedAt) { updates.bannerUrl = null; } else { - updates.bannerUrl = getDriveFileUrl(banner, false); + updates.bannerUrl = banner.metadata.webpublicUrl; if (banner.metadata.properties.avgColor) { updates.bannerColor = banner.metadata.properties.avgColor; @@ -236,7 +235,7 @@ export default define(meta, async (ps, user, app) => { if (wallpaper.metadata.deletedAt) { updates.wallpaperUrl = null; } else { - updates.wallpaperUrl = getDriveFileUrl(wallpaper); + updates.wallpaperUrl = wallpaper.metadata.webpublicUrl; if (wallpaper.metadata.properties.avgColor) { updates.wallpaperColor = wallpaper.metadata.properties.avgColor; diff --git a/src/server/file/index.ts b/src/server/file/index.ts index 973528da3..acb8322e3 100644 --- a/src/server/file/index.ts +++ b/src/server/file/index.ts @@ -33,8 +33,7 @@ router.get('/app-default.jpg', ctx => { ctx.body = file; }); -router.get('/:id', sendDriveFile); -router.get('/:id/*', sendDriveFile); +router.get('/*', sendDriveFile); // Register router app.use(router.routes()); diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts index 691d3bf84..afc3aa3ce 100644 --- a/src/server/file/send-drive-file.ts +++ b/src/server/file/send-drive-file.ts @@ -1,6 +1,5 @@ import * as Koa from 'koa'; import * as send from 'koa-send'; -import * as mongodb from 'mongodb'; import DriveFile, { getDriveFileBucket } from '../../models/drive-file'; import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic'; @@ -14,16 +13,18 @@ const commonReadableHandlerGenerator = (ctx: Koa.BaseContext) => (e: Error): voi }; export default async function(ctx: Koa.BaseContext) { - // Validate id - if (!mongodb.ObjectID.isValid(ctx.params.id)) { - ctx.throw(400, 'incorrect id'); - return; - } - - const fileId = new mongodb.ObjectID(ctx.params.id); + const url = ctx.href; // Fetch drive file - const file = await DriveFile.findOne({ _id: fileId }); + const file = await DriveFile.findOne({ + $or: [{ + 'metadata.url': url + }, { + 'metadata.webpublicUrl': url + }, { + 'metadata.thumbnailUrl': url + }], + }); if (file == null) { ctx.status = 404; @@ -43,21 +44,16 @@ export default async function(ctx: Koa.BaseContext) { } const sendRaw = async () => { - if (file.metadata && file.metadata.accessKey && file.metadata.accessKey != ctx.query['original']) { - ctx.status = 403; - return; - } - const bucket = await getDriveFileBucket(); - const readable = bucket.openDownloadStream(fileId); + const readable = bucket.openDownloadStream(file._id); readable.on('error', commonReadableHandlerGenerator(ctx)); ctx.set('Content-Type', file.contentType); ctx.body = readable; }; - if ('thumbnail' in ctx.query) { + if (file.metadata.thumbnailUrl === url) { const thumb = await DriveFileThumbnail.findOne({ - 'metadata.originalId': fileId + 'metadata.originalId': file._id }); if (thumb != null) { @@ -72,9 +68,9 @@ export default async function(ctx: Koa.BaseContext) { await send(ctx as any, '/dummy.png', { root: assets }); } } - } else if ('web' in ctx.query) { + } else if (file.metadata.webpublicUrl === url) { const web = await DriveFileWebpublic.findOne({ - 'metadata.originalId': fileId + 'metadata.originalId': file._id }); if (web != null) { diff --git a/src/server/web/feed.ts b/src/server/web/feed.ts index 09ac10c57..e8d06837e 100644 --- a/src/server/web/feed.ts +++ b/src/server/web/feed.ts @@ -2,7 +2,6 @@ import { Feed } from 'feed'; import config from '../../config'; import Note from '../../models/note'; import { IUser } from '../../models/user'; -import { getOriginalUrl } from '../../misc/get-drive-file-url'; export default async function(user: IUser) { const author: Author = { @@ -38,7 +37,7 @@ export default async function(user: IUser) { } as FeedOptions); for (const note of notes) { - const file = note._files && note._files.find(file => file.contentType.startsWith('image/')); + const file = note._files ? note._files.find(file => file.contentType.startsWith('image/')) : null; feed.addItem({ title: `New note by ${author.name}`, @@ -46,7 +45,7 @@ export default async function(user: IUser) { date: note.createdAt, description: note.cw, content: note.text, - image: file && getOriginalUrl(file) + image: file ? file.metadata.webpublicUrl : null }); } diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 9f3805f94..782e8e999 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -117,12 +117,15 @@ async function save(path: string, name: string, type: string, hash: string, size return file; } else { // use MongoDB GridFS + Object.assign(metadata, { + url: `${config.driveUrl}/${uuid.v4()}`, + webpublicUrl: `${config.driveUrl}/${uuid.v4()}?web`, + thumbnailUrl: `${config.driveUrl}/${uuid.v4()}?thumbnail`, + } as IMetadata); + // #region store original const originalDst = await getDriveFileBucket(); - // web用(Exif削除済み)がある場合はオリジナルにアクセス制限 - if (alts.webpublic) metadata.accessKey = uuid.v4(); - const originalFile = await storeOriginal(originalDst, name, path, type, metadata); logger.info(`original stored to ${originalFile._id}`); From d6db5e46e8a989102695aa2fc3b25bbe51267d57 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 28 Feb 2019 15:44:23 +0900 Subject: [PATCH 2/8] Add migration --- src/models/drive-file.ts | 34 +++++++++++++++++++ src/queue/processors/export-blocking.ts | 4 +-- src/queue/processors/export-following.ts | 4 +-- src/queue/processors/export-mute.ts | 4 +-- src/queue/processors/export-notes.ts | 4 +-- .../api/endpoints/drive/files/create.ts | 4 +-- src/services/drive/add-file.ts | 2 +- src/services/drive/delete-file.ts | 4 +-- src/services/drive/upload-from-url.ts | 4 +-- 9 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index 2b1f59fb6..9bd83b7ee 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -4,15 +4,49 @@ import { pack as packFolder } from './drive-folder'; import { pack as packUser } from './user'; import monkDb, { nativeDbConn, dbLogger } from '../db/mongodb'; import isObjectId from '../misc/is-objectid'; +import config from '../config'; +import uuid = require('uuid'); const DriveFile = monkDb.get('driveFiles.files'); DriveFile.createIndex('md5'); DriveFile.createIndex('metadata.uri'); +DriveFile.createIndex('metadata.url'); +DriveFile.createIndex('metadata.webpublicUrl'); +DriveFile.createIndex('metadata.thumbnailUrl'); DriveFile.createIndex('metadata.userId'); DriveFile.createIndex('metadata.folderId'); DriveFile.createIndex('metadata._user.host'); export default DriveFile; +// 後方互換性のため +DriveFile.findOne({ + $or: [{ + 'metadata.url': { $exists: false } + }, { + 'metadata.url': null + }] +}).then(x => { + if (x != null) { + DriveFile.find({ + $or: [{ + 'metadata.url': { $exists: false } + }, { + 'metadata.url': null + }] + }, { fields: { _id: true } }).then(xs => { + for (const x of xs) { + DriveFile.update({ _id: x._id }, { + $set: { + 'metadata.url': `${config.driveUrl}/${uuid.v4()}`, + 'metadata.webpublicUrl': `${config.driveUrl}/${uuid.v4()}?web`, + 'metadata.thumbnailUrl': `${config.driveUrl}/${uuid.v4()}?thumbnail`, + } + }); + } + }); + } +}); + export const DriveFileChunk = monkDb.get('driveFiles.chunks'); export const getDriveFileBucket = async (): Promise => { diff --git a/src/queue/processors/export-blocking.ts b/src/queue/processors/export-blocking.ts index b30d8e3bc..ad91b8846 100644 --- a/src/queue/processors/export-blocking.ts +++ b/src/queue/processors/export-blocking.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import * as mongo from 'mongodb'; import { queueLogger } from '../logger'; -import addFile from '../../services/drive/add-file'; +import { addDriveFile } from '../../services/drive/add-file'; import User from '../../models/user'; import dateFormat = require('dateformat'); import Blocking from '../../models/blocking'; @@ -81,7 +81,7 @@ export async function exportBlocking(job: bq.Job, done: any): Promise { logger.succ(`Exported to: ${path}`); const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; - const driveFile = await addFile(user, path, fileName); + const driveFile = await addDriveFile(user, path, fileName); logger.succ(`Exported to: ${driveFile._id}`); cleanup(); diff --git a/src/queue/processors/export-following.ts b/src/queue/processors/export-following.ts index e6521d065..a29e4af00 100644 --- a/src/queue/processors/export-following.ts +++ b/src/queue/processors/export-following.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import * as mongo from 'mongodb'; import { queueLogger } from '../logger'; -import addFile from '../../services/drive/add-file'; +import { addDriveFile } from '../../services/drive/add-file'; import User from '../../models/user'; import dateFormat = require('dateformat'); import Following from '../../models/following'; @@ -81,7 +81,7 @@ export async function exportFollowing(job: bq.Job, done: any): Promise { logger.succ(`Exported to: ${path}`); const fileName = 'following-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; - const driveFile = await addFile(user, path, fileName); + const driveFile = await addDriveFile(user, path, fileName); logger.succ(`Exported to: ${driveFile._id}`); cleanup(); diff --git a/src/queue/processors/export-mute.ts b/src/queue/processors/export-mute.ts index 74456c1da..8de6b41f9 100644 --- a/src/queue/processors/export-mute.ts +++ b/src/queue/processors/export-mute.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import * as mongo from 'mongodb'; import { queueLogger } from '../logger'; -import addFile from '../../services/drive/add-file'; +import { addDriveFile } from '../../services/drive/add-file'; import User from '../../models/user'; import dateFormat = require('dateformat'); import Mute from '../../models/mute'; @@ -81,7 +81,7 @@ export async function exportMute(job: bq.Job, done: any): Promise { logger.succ(`Exported to: ${path}`); const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv'; - const driveFile = await addFile(user, path, fileName); + const driveFile = await addDriveFile(user, path, fileName); logger.succ(`Exported to: ${driveFile._id}`); cleanup(); diff --git a/src/queue/processors/export-notes.ts b/src/queue/processors/export-notes.ts index 32e4cd1d6..86cff3aa5 100644 --- a/src/queue/processors/export-notes.ts +++ b/src/queue/processors/export-notes.ts @@ -5,7 +5,7 @@ import * as mongo from 'mongodb'; import { queueLogger } from '../logger'; import Note, { INote } from '../../models/note'; -import addFile from '../../services/drive/add-file'; +import { addDriveFile } from '../../services/drive/add-file'; import User from '../../models/user'; import dateFormat = require('dateformat'); @@ -101,7 +101,7 @@ export async function exportNotes(job: bq.Job, done: any): Promise { logger.succ(`Exported to: ${path}`); const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json'; - const driveFile = await addFile(user, path, fileName); + const driveFile = await addDriveFile(user, path, fileName); logger.succ(`Exported to: ${driveFile._id}`); cleanup(); diff --git a/src/server/api/endpoints/drive/files/create.ts b/src/server/api/endpoints/drive/files/create.ts index b2979c488..1f2769cbe 100644 --- a/src/server/api/endpoints/drive/files/create.ts +++ b/src/server/api/endpoints/drive/files/create.ts @@ -2,7 +2,7 @@ import * as ms from 'ms'; import $ from 'cafy'; import ID, { transform } from '../../../../../misc/cafy-id'; import { validateFileName, pack } from '../../../../../models/drive-file'; -import create from '../../../../../services/drive/add-file'; +import { addDriveFile } from '../../../../../services/drive/add-file'; import define from '../../../define'; import { apiLogger } from '../../../logger'; import { ApiError } from '../../../error'; @@ -87,7 +87,7 @@ export default define(meta, async (ps, user, app, file, cleanup) => { try { // Create file - const driveFile = await create(user, file.path, name, null, ps.folderId, ps.force, false, null, null, ps.isSensitive); + const driveFile = await addDriveFile(user, file.path, name, null, ps.folderId, ps.force, false, null, null, ps.isSensitive); return pack(driveFile, { self: true }); } catch (e) { apiLogger.error(e); diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 782e8e999..091eb6f6f 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -276,7 +276,7 @@ async function deleteOldFile(user: IRemoteUser) { * @param sensitive Mark file as sensitive * @return Created drive file */ -export default async function( +export async function addDriveFile( user: IUser, path: string, name: string = null, diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index c5c15ca20..dfe736807 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -48,8 +48,8 @@ export default async function(file: IDriveFile, isExpired = false) { set.metadata.withoutChunks = true; set.metadata.isRemote = true; set.metadata.url = file.metadata.uri; - set.metadata.thumbnailUrl = undefined; - set.metadata.webpublicUrl = undefined; + set.metadata.thumbnailUrl = file.metadata.uri; + set.metadata.webpublicUrl = file.metadata.uri; } await DriveFile.update({ _id: file._id }, { diff --git a/src/services/drive/upload-from-url.ts b/src/services/drive/upload-from-url.ts index 1651afa27..91f6a287f 100644 --- a/src/services/drive/upload-from-url.ts +++ b/src/services/drive/upload-from-url.ts @@ -4,7 +4,7 @@ import * as tmp from 'tmp'; import * as request from 'request'; import { IDriveFile, validateFileName } from '../../models/drive-file'; -import create from './add-file'; +import { addDriveFile } from './add-file'; import config from '../../config'; import { IUser } from '../../models/user'; import * as mongodb from 'mongodb'; @@ -83,7 +83,7 @@ export default async ( let error; try { - driveFile = await create(user, path, name, null, folderId, force, link, url, uri, sensitive); + driveFile = await addDriveFile(user, path, name, null, folderId, force, link, url, uri, sensitive); logger.succ(`Got: ${driveFile._id}`); } catch (e) { error = e; From 7cd2a9c9f268c76b0140763ed5015e3cbbf711f2 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 1 Mar 2019 11:35:04 +0900 Subject: [PATCH 3/8] Add migration --- src/server/api/endpoints/users/show.ts | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/server/api/endpoints/users/show.ts b/src/server/api/endpoints/users/show.ts index 4e59945eb..9d08a48e7 100644 --- a/src/server/api/endpoints/users/show.ts +++ b/src/server/api/endpoints/users/show.ts @@ -5,6 +5,7 @@ import resolveRemoteUser from '../../../../remote/resolve-user'; import define from '../../define'; import { apiLogger } from '../../logger'; import { ApiError } from '../../error'; +import DriveFile from '../../../../models/drive-file'; const cursorOption = { fields: { data: false } }; @@ -103,6 +104,33 @@ export default define(meta, async (ps, me) => { } } + // 後方互換性のため + // TODO: そのうち削除 + if (user.avatarId) { + const avatar = await DriveFile.findOne({ _id: user.avatarId }); + User.update({ _id: user._id }, { + $set: { + avatarUrl: avatar.metadata.thumbnailUrl, + } + }); + } + if (user.bannerId) { + const banner = await DriveFile.findOne({ _id: user.bannerId }); + User.update({ _id: user._id }, { + $set: { + bannerUrl: banner.metadata.webpublicUrl, + } + }); + } + if (user.wallpaperId) { + const wallpaper = await DriveFile.findOne({ _id: user.wallpaperId }); + User.update({ _id: user._id }, { + $set: { + wallpaperUrl: wallpaper.metadata.webpublicUrl, + } + }); + } + return await pack(user, me, { detail: true }); From 1eece5ab3dafe7e9940b343f68c8029bb7a8b0e7 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 2 Mar 2019 07:41:31 +0900 Subject: [PATCH 4/8] =?UTF-8?q?URL=E3=81=AB=E6=8B=A1=E5=BC=B5=E5=AD=90?= =?UTF-8?q?=E3=82=92=E4=BB=98=E3=81=91=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/models/drive-file.ts | 16 ++++++++++++---- src/services/drive/add-file.ts | 22 +++++++++++----------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index 9bd83b7ee..6c554c117 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -33,13 +33,21 @@ DriveFile.findOne({ }, { 'metadata.url': null }] - }, { fields: { _id: true } }).then(xs => { + }, { fields: { _id: true, filename: true, contentType: true } }).then(xs => { for (const x of xs) { + let [ext] = (x.filename.match(/\.([a-zA-Z0-9_-]+)$/) || ['']); + + if (ext === '') { + if (x.contentType === 'image/jpeg') ext = '.jpg'; + if (x.contentType === 'image/png') ext = '.png'; + if (x.contentType === 'image/webp') ext = '.webp'; + } + DriveFile.update({ _id: x._id }, { $set: { - 'metadata.url': `${config.driveUrl}/${uuid.v4()}`, - 'metadata.webpublicUrl': `${config.driveUrl}/${uuid.v4()}?web`, - 'metadata.thumbnailUrl': `${config.driveUrl}/${uuid.v4()}?thumbnail`, + 'metadata.url': `${config.driveUrl}/${uuid.v4()}${ext}`, + 'metadata.webpublicUrl': `${config.driveUrl}/${uuid.v4()}.jpg`, + 'metadata.thumbnailUrl': `${config.driveUrl}/${uuid.v4()}.jpg`, } }); } diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 091eb6f6f..7b28113bd 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -42,16 +42,16 @@ async function save(path: string, name: string, type: string, hash: string, size // thunbnail, webpublic を必要なら生成 const alts = await generateAlts(path, type, !metadata.uri); + let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']); + + if (ext === '') { + if (type === 'image/jpeg') ext = '.jpg'; + if (type === 'image/png') ext = '.png'; + if (type === 'image/webp') ext = '.webp'; + } + if (config.drive && config.drive.storage == 'minio') { //#region ObjectStorage params - let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']); - - if (ext === '') { - if (type === 'image/jpeg') ext = '.jpg'; - if (type === 'image/png') ext = '.png'; - if (type === 'image/webp') ext = '.webp'; - } - const baseUrl = config.drive.baseUrl || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; @@ -118,9 +118,9 @@ async function save(path: string, name: string, type: string, hash: string, size return file; } else { // use MongoDB GridFS Object.assign(metadata, { - url: `${config.driveUrl}/${uuid.v4()}`, - webpublicUrl: `${config.driveUrl}/${uuid.v4()}?web`, - thumbnailUrl: `${config.driveUrl}/${uuid.v4()}?thumbnail`, + url: `${config.driveUrl}/${uuid.v4()}${ext}`, + webpublicUrl: `${config.driveUrl}/${uuid.v4()}.${alts.webpublic.ext}`, + thumbnailUrl: `${config.driveUrl}/${uuid.v4()}.${alts.thumbnail.ext}`, } as IMetadata); // #region store original From 7945224a742195866d7648a4a207cfcdc767ea50 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 2 Mar 2019 07:41:42 +0900 Subject: [PATCH 5/8] =?UTF-8?q?=E3=83=95=E3=82=A9=E3=83=BC=E3=83=AB?= =?UTF-8?q?=E3=83=90=E3=83=83=E3=82=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/models/drive-file.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index 6c554c117..c0ef957a7 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -213,8 +213,8 @@ export const pack = ( _target = Object.assign(_target, _file.metadata); - _target.url = _file.metadata.webpublicUrl; - _target.thumbnailUrl = _file.metadata.thumbnailUrl; + _target.url = _file.metadata.webpublicUrl || _file.metadata.url; + _target.thumbnailUrl = _file.metadata.thumbnailUrl || _file.metadata.webpublicUrl; _target.isRemote = _file.metadata.isRemote; if (_target.properties == null) _target.properties = {}; From b9b81e6665e9de05a45fbbd13c2ac8e4004e0083 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 2 Mar 2019 07:43:17 +0900 Subject: [PATCH 6/8] Update drive-file.ts --- src/models/drive-file.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index c0ef957a7..96276962c 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -213,8 +213,10 @@ export const pack = ( _target = Object.assign(_target, _file.metadata); + const isImage = file.contentType && file.contentType.startsWith('image/'); + _target.url = _file.metadata.webpublicUrl || _file.metadata.url; - _target.thumbnailUrl = _file.metadata.thumbnailUrl || _file.metadata.webpublicUrl; + _target.thumbnailUrl = _file.metadata.thumbnailUrl || _file.metadata.webpublicUrl || (isImage ? _file.metadata.url : '/assets/thumbnail-not-available.png'); _target.isRemote = _file.metadata.isRemote; if (_target.properties == null) _target.properties = {}; From 3cfcb89c0412ff76a3f5bca9cbe9576f74f25feb Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 2 Mar 2019 11:29:18 +0900 Subject: [PATCH 7/8] update --- src/models/drive-file.ts | 2 +- src/server/file/send-drive-file.ts | 4 +++- src/services/drive/delete-file.ts | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index 96276962c..487717ef1 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -46,7 +46,7 @@ DriveFile.findOne({ DriveFile.update({ _id: x._id }, { $set: { 'metadata.url': `${config.driveUrl}/${uuid.v4()}${ext}`, - 'metadata.webpublicUrl': `${config.driveUrl}/${uuid.v4()}.jpg`, + 'metadata.webpublicUrl': `${config.driveUrl}/${uuid.v4()}${ext}`, 'metadata.thumbnailUrl': `${config.driveUrl}/${uuid.v4()}.jpg`, } }); diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts index afc3aa3ce..a9bc6f03e 100644 --- a/src/server/file/send-drive-file.ts +++ b/src/server/file/send-drive-file.ts @@ -4,6 +4,7 @@ import DriveFile, { getDriveFileBucket } from '../../models/drive-file'; import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic'; import { serverLogger } from '..'; +import config from '../../config'; const assets = `${__dirname}/../../server/file/assets/`; @@ -13,7 +14,8 @@ const commonReadableHandlerGenerator = (ctx: Koa.BaseContext) => (e: Error): voi }; export default async function(ctx: Koa.BaseContext) { - const url = ctx.href; + let url = ctx.href; + if (url.startsWith('http://') && config.url.startsWith('https://')) url = url.replace('http://', 'https://'); // Fetch drive file const file = await DriveFile.findOne({ diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index dfe736807..c5c15ca20 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -48,8 +48,8 @@ export default async function(file: IDriveFile, isExpired = false) { set.metadata.withoutChunks = true; set.metadata.isRemote = true; set.metadata.url = file.metadata.uri; - set.metadata.thumbnailUrl = file.metadata.uri; - set.metadata.webpublicUrl = file.metadata.uri; + set.metadata.thumbnailUrl = undefined; + set.metadata.webpublicUrl = undefined; } await DriveFile.update({ _id: file._id }, { From f87232178ab49d30c2a77e4a475777b9abb87079 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 2 Mar 2019 11:34:34 +0900 Subject: [PATCH 8/8] Fix --- src/models/drive-file.ts | 2 +- src/server/file/index.ts | 8 +++++++- src/server/file/send-drive-file.ts | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts index 487717ef1..11037ac2a 100644 --- a/src/models/drive-file.ts +++ b/src/models/drive-file.ts @@ -216,7 +216,7 @@ export const pack = ( const isImage = file.contentType && file.contentType.startsWith('image/'); _target.url = _file.metadata.webpublicUrl || _file.metadata.url; - _target.thumbnailUrl = _file.metadata.thumbnailUrl || _file.metadata.webpublicUrl || (isImage ? _file.metadata.url : '/assets/thumbnail-not-available.png'); + _target.thumbnailUrl = _file.metadata.thumbnailUrl || _file.metadata.webpublicUrl || (isImage ? _file.metadata.url : '/files/thumbnail-not-available.png'); _target.isRemote = _file.metadata.isRemote; if (_target.properties == null) _target.properties = {}; diff --git a/src/server/file/index.ts b/src/server/file/index.ts index acb8322e3..5dbb5a4b5 100644 --- a/src/server/file/index.ts +++ b/src/server/file/index.ts @@ -29,7 +29,13 @@ router.get('/default-avatar.jpg', ctx => { router.get('/app-default.jpg', ctx => { const file = fs.createReadStream(`${__dirname}/assets/dummy.png`); - ctx.set('Content-Type', 'image/jpeg'); + ctx.set('Content-Type', 'image/png'); + ctx.body = file; +}); + +router.get('/thumbnail-not-available.png', ctx => { + const file = fs.createReadStream(`${__dirname}/assets/thumbnail-not-available.png`); + ctx.set('Content-Type', 'image/png'); ctx.body = file; }); diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts index a9bc6f03e..7ccc52bae 100644 --- a/src/server/file/send-drive-file.ts +++ b/src/server/file/send-drive-file.ts @@ -67,7 +67,7 @@ export default async function(ctx: Koa.BaseContext) { await sendRaw(); } else { ctx.status = 404; - await send(ctx as any, '/dummy.png', { root: assets }); + await send(ctx as any, '/thumbnail-not-available.png', { root: assets }); } } } else if (file.metadata.webpublicUrl === url) {