Compare commits

...

10 commits

Author SHA1 Message Date
syuilo
f87232178a
Fix 2019-03-02 11:34:34 +09:00
syuilo
3cfcb89c04
update 2019-03-02 11:29:18 +09:00
syuilo
b9b81e6665
Update drive-file.ts 2019-03-02 07:43:17 +09:00
syuilo
7945224a74
フォールバック 2019-03-02 07:41:42 +09:00
syuilo
1eece5ab3d
URLに拡張子を付けた 2019-03-02 07:41:31 +09:00
syuilo
1d3342e641
Merge branch 'develop' into refactor-drive 2019-03-02 07:18:01 +09:00
syuilo
7cd2a9c9f2
Add migration 2019-03-01 11:35:04 +09:00
syuilo
528be133ff
Merge branch 'develop' into refactor-drive 2019-03-01 11:17:28 +09:00
syuilo
d6db5e46e8
Add migration 2019-02-28 15:44:23 +09:00
syuilo
b5c235f837
Refactor drive 2019-02-28 15:20:40 +09:00
18 changed files with 150 additions and 139 deletions

View file

@ -1,31 +0,0 @@
import { IDriveFile } from '../models/drive-file';
import config from '../config';
export default function(file: IDriveFile, thumbnail = false): string {
if (file == null) return null;
const isImage = file.contentType && file.contentType.startsWith('image/');
if (file.metadata.withoutChunks) {
if (thumbnail) {
return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || (isImage ? file.metadata.url : null);
} else {
return file.metadata.webpublicUrl || file.metadata.url;
}
} else {
if (thumbnail) {
return `${config.driveUrl}/${file._id}?thumbnail`;
} else {
return `${config.driveUrl}/${file._id}?web`;
}
}
}
export function getOriginalUrl(file: IDriveFile) {
if (file.metadata && file.metadata.url) {
return file.metadata.url;
}
const accessKey = file.metadata ? file.metadata.accessKey : null;
return `${config.driveUrl}/${file._id}${accessKey ? '?original=' + accessKey : ''}`;
}

View file

@ -4,16 +4,57 @@ import { pack as packFolder } from './drive-folder';
import { pack as packUser } from './user';
import monkDb, { nativeDbConn, dbLogger } from '../db/mongodb';
import isObjectId from '../misc/is-objectid';
import getDriveFileUrl, { getOriginalUrl } from '../misc/get-drive-file-url';
import config from '../config';
import uuid = require('uuid');
const DriveFile = monkDb.get<IDriveFile>('driveFiles.files');
DriveFile.createIndex('md5');
DriveFile.createIndex('metadata.uri');
DriveFile.createIndex('metadata.url');
DriveFile.createIndex('metadata.webpublicUrl');
DriveFile.createIndex('metadata.thumbnailUrl');
DriveFile.createIndex('metadata.userId');
DriveFile.createIndex('metadata.folderId');
DriveFile.createIndex('metadata._user.host');
export default DriveFile;
// 後方互換性のため
DriveFile.findOne({
$or: [{
'metadata.url': { $exists: false }
}, {
'metadata.url': null
}]
}).then(x => {
if (x != null) {
DriveFile.find({
$or: [{
'metadata.url': { $exists: false }
}, {
'metadata.url': null
}]
}, { fields: { _id: true, filename: true, contentType: true } }).then(xs => {
for (const x of xs) {
let [ext] = (x.filename.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
if (ext === '') {
if (x.contentType === 'image/jpeg') ext = '.jpg';
if (x.contentType === 'image/png') ext = '.png';
if (x.contentType === 'image/webp') ext = '.webp';
}
DriveFile.update({ _id: x._id }, {
$set: {
'metadata.url': `${config.driveUrl}/${uuid.v4()}${ext}`,
'metadata.webpublicUrl': `${config.driveUrl}/${uuid.v4()}${ext}`,
'metadata.thumbnailUrl': `${config.driveUrl}/${uuid.v4()}.jpg`,
}
});
}
});
}
});
export const DriveFileChunk = monkDb.get('driveFiles.chunks');
export const getDriveFileBucket = async (): Promise<mongo.GridFSBucket> => {
@ -36,26 +77,10 @@ export type IMetadata = {
*/
uri?: string;
/**
* URL for web() or original
* * or
*/
url?: string;
/**
* URL for thumbnail (thumbnailがなければなし)
* * or
*/
url: string;
thumbnailUrl?: string;
/**
* URL for original (web用が生成されてない場合はurlがoriginalを指す)
* * or
*/
webpublicUrl?: string;
accessKey?: string;
src?: string;
deletedAt?: Date;
@ -188,8 +213,10 @@ export const pack = (
_target = Object.assign(_target, _file.metadata);
_target.url = getDriveFileUrl(_file);
_target.thumbnailUrl = getDriveFileUrl(_file, true);
const isImage = file.contentType && file.contentType.startsWith('image/');
_target.url = _file.metadata.webpublicUrl || _file.metadata.url;
_target.thumbnailUrl = _file.metadata.thumbnailUrl || _file.metadata.webpublicUrl || (isImage ? _file.metadata.url : '/files/thumbnail-not-available.png');
_target.isRemote = _file.metadata.isRemote;
if (_target.properties == null) _target.properties = {};
@ -224,7 +251,7 @@ export const pack = (
delete _target._user;
if (opts.self) {
_target.url = getOriginalUrl(_file);
_target.url = _file.metadata.url;
}
resolve(_target);

View file

@ -4,7 +4,7 @@ import * as fs from 'fs';
import * as mongo from 'mongodb';
import { queueLogger } from '../logger';
import addFile from '../../services/drive/add-file';
import { addDriveFile } from '../../services/drive/add-file';
import User from '../../models/user';
import dateFormat = require('dateformat');
import Blocking from '../../models/blocking';
@ -81,7 +81,7 @@ export async function exportBlocking(job: bq.Job, done: any): Promise<void> {
logger.succ(`Exported to: ${path}`);
const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv';
const driveFile = await addFile(user, path, fileName);
const driveFile = await addDriveFile(user, path, fileName);
logger.succ(`Exported to: ${driveFile._id}`);
cleanup();

View file

@ -4,7 +4,7 @@ import * as fs from 'fs';
import * as mongo from 'mongodb';
import { queueLogger } from '../logger';
import addFile from '../../services/drive/add-file';
import { addDriveFile } from '../../services/drive/add-file';
import User from '../../models/user';
import dateFormat = require('dateformat');
import Following from '../../models/following';
@ -81,7 +81,7 @@ export async function exportFollowing(job: bq.Job, done: any): Promise<void> {
logger.succ(`Exported to: ${path}`);
const fileName = 'following-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv';
const driveFile = await addFile(user, path, fileName);
const driveFile = await addDriveFile(user, path, fileName);
logger.succ(`Exported to: ${driveFile._id}`);
cleanup();

View file

@ -4,7 +4,7 @@ import * as fs from 'fs';
import * as mongo from 'mongodb';
import { queueLogger } from '../logger';
import addFile from '../../services/drive/add-file';
import { addDriveFile } from '../../services/drive/add-file';
import User from '../../models/user';
import dateFormat = require('dateformat');
import Mute from '../../models/mute';
@ -81,7 +81,7 @@ export async function exportMute(job: bq.Job, done: any): Promise<void> {
logger.succ(`Exported to: ${path}`);
const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.csv';
const driveFile = await addFile(user, path, fileName);
const driveFile = await addDriveFile(user, path, fileName);
logger.succ(`Exported to: ${driveFile._id}`);
cleanup();

View file

@ -5,7 +5,7 @@ import * as mongo from 'mongodb';
import { queueLogger } from '../logger';
import Note, { INote } from '../../models/note';
import addFile from '../../services/drive/add-file';
import { addDriveFile } from '../../services/drive/add-file';
import User from '../../models/user';
import dateFormat = require('dateformat');
@ -101,7 +101,7 @@ export async function exportNotes(job: bq.Job, done: any): Promise<void> {
logger.succ(`Exported to: ${path}`);
const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json';
const driveFile = await addFile(user, path, fileName);
const driveFile = await addDriveFile(user, path, fileName);
logger.succ(`Exported to: ${driveFile._id}`);
cleanup();

View file

@ -16,7 +16,6 @@ import { URL } from 'url';
import { resolveNote, extractEmojis } from './note';
import { registerOrFetchInstanceDoc } from '../../../services/register-or-fetch-instance-doc';
import Instance from '../../../models/instance';
import getDriveFileUrl from '../../../misc/get-drive-file-url';
import { IEmoji } from '../../../models/emoji';
import { ITag, extractHashtags } from './tag';
import Following from '../../../models/following';
@ -227,8 +226,8 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<IU
const avatarId = avatar ? avatar._id : null;
const bannerId = banner ? banner._id : null;
const avatarUrl = getDriveFileUrl(avatar, true);
const bannerUrl = getDriveFileUrl(banner, false);
const avatarUrl = avatar.metadata.thumbnailUrl;
const bannerUrl = banner.metadata.webpublicUrl;
const avatarColor = avatar && avatar.metadata.properties.avgColor ? avatar.metadata.properties.avgColor : null;
const bannerColor = banner && avatar.metadata.properties.avgColor ? banner.metadata.properties.avgColor : null;
@ -373,13 +372,13 @@ export async function updatePerson(uri: string, resolver?: Resolver, hint?: obje
if (avatar) {
updates.avatarId = avatar._id;
updates.avatarUrl = getDriveFileUrl(avatar, true);
updates.avatarUrl = avatar.metadata.thumbnailUrl;
updates.avatarColor = avatar.metadata.properties.avgColor ? avatar.metadata.properties.avgColor : null;
}
if (banner) {
updates.bannerId = banner._id;
updates.bannerUrl = getDriveFileUrl(banner, true);
updates.bannerUrl = banner.metadata.webpublicUrl;
updates.bannerColor = banner.metadata.properties.avgColor ? banner.metadata.properties.avgColor : null;
}

View file

@ -1,8 +1,7 @@
import { IDriveFile } from '../../../models/drive-file';
import getDriveFileUrl from '../../../misc/get-drive-file-url';
export default (file: IDriveFile) => ({
type: 'Document',
mediaType: file.contentType,
url: getDriveFileUrl(file)
url: file.metadata.webpublicUrl
});

View file

@ -1,8 +1,7 @@
import { IDriveFile } from '../../../models/drive-file';
import getDriveFileUrl from '../../../misc/get-drive-file-url';
export default (file: IDriveFile) => ({
type: 'Image',
url: getDriveFileUrl(file),
url: file.metadata.webpublicUrl,
sensitive: file.metadata.isSensitive
});

View file

@ -2,7 +2,7 @@ import * as ms from 'ms';
import $ from 'cafy';
import ID, { transform } from '../../../../../misc/cafy-id';
import { validateFileName, pack } from '../../../../../models/drive-file';
import create from '../../../../../services/drive/add-file';
import { addDriveFile } from '../../../../../services/drive/add-file';
import define from '../../../define';
import { apiLogger } from '../../../logger';
import { ApiError } from '../../../error';
@ -87,7 +87,7 @@ export default define(meta, async (ps, user, app, file, cleanup) => {
try {
// Create file
const driveFile = await create(user, file.path, name, null, ps.folderId, ps.force, false, null, null, ps.isSensitive);
const driveFile = await addDriveFile(user, file.path, name, null, ps.folderId, ps.force, false, null, null, ps.isSensitive);
return pack(driveFile, { self: true });
} catch (e) {
apiLogger.error(e);

View file

@ -1,9 +1,7 @@
import $ from 'cafy';
import * as mongo from 'mongodb';
import ID, { transform } from '../../../../../misc/cafy-id';
import DriveFile, { pack, IDriveFile } from '../../../../../models/drive-file';
import define from '../../../define';
import config from '../../../../../config';
import { ApiError } from '../../../error';
export const meta = {
@ -73,28 +71,16 @@ export default define(meta, async (ps, user) => {
'metadata.deletedAt': { $exists: false }
});
} else if (ps.url) {
const isInternalStorageUrl = ps.url.startsWith(config.driveUrl);
if (isInternalStorageUrl) {
// Extract file ID from url
// e.g.
// http://misskey.local/files/foo?original=bar --> foo
const fileId = new mongo.ObjectID(ps.url.replace(config.driveUrl, '').replace(/\?(.*)$/, '').replace(/\//g, ''));
file = await DriveFile.findOne({
_id: fileId,
'metadata.deletedAt': { $exists: false }
});
} else {
file = await DriveFile.findOne({
$or: [{
'metadata.url': ps.url
}, {
'metadata.webpublicUrl': ps.url
}, {
'metadata.thumbnailUrl': ps.url
}],
'metadata.deletedAt': { $exists: false }
});
}
file = await DriveFile.findOne({
$or: [{
'metadata.url': ps.url
}, {
'metadata.webpublicUrl': ps.url
}, {
'metadata.thumbnailUrl': ps.url
}],
'metadata.deletedAt': { $exists: false }
});
} else {
throw new ApiError(meta.errors.fileIdOrUrlRequired);
}

View file

@ -6,7 +6,6 @@ import DriveFile from '../../../../models/drive-file';
import acceptAllFollowRequests from '../../../../services/following/requests/accept-all';
import { publishToFollowers } from '../../../../services/i/update';
import define from '../../define';
import getDriveFileUrl from '../../../../misc/get-drive-file-url';
import { parse, parsePlain } from '../../../../mfm/parse';
import extractEmojis from '../../../../misc/extract-emojis';
import extractHashtags from '../../../../misc/extract-hashtags';
@ -195,7 +194,7 @@ export default define(meta, async (ps, user, app) => {
if (avatar.metadata.deletedAt) {
updates.avatarUrl = null;
} else {
updates.avatarUrl = getDriveFileUrl(avatar, true);
updates.avatarUrl = avatar.metadata.thumbnailUrl;
if (avatar.metadata.properties.avgColor) {
updates.avatarColor = avatar.metadata.properties.avgColor;
@ -214,7 +213,7 @@ export default define(meta, async (ps, user, app) => {
if (banner.metadata.deletedAt) {
updates.bannerUrl = null;
} else {
updates.bannerUrl = getDriveFileUrl(banner, false);
updates.bannerUrl = banner.metadata.webpublicUrl;
if (banner.metadata.properties.avgColor) {
updates.bannerColor = banner.metadata.properties.avgColor;
@ -236,7 +235,7 @@ export default define(meta, async (ps, user, app) => {
if (wallpaper.metadata.deletedAt) {
updates.wallpaperUrl = null;
} else {
updates.wallpaperUrl = getDriveFileUrl(wallpaper);
updates.wallpaperUrl = wallpaper.metadata.webpublicUrl;
if (wallpaper.metadata.properties.avgColor) {
updates.wallpaperColor = wallpaper.metadata.properties.avgColor;

View file

@ -5,6 +5,7 @@ import resolveRemoteUser from '../../../../remote/resolve-user';
import define from '../../define';
import { apiLogger } from '../../logger';
import { ApiError } from '../../error';
import DriveFile from '../../../../models/drive-file';
const cursorOption = { fields: { data: false } };
@ -103,6 +104,33 @@ export default define(meta, async (ps, me) => {
}
}
// 後方互換性のため
// TODO: そのうち削除
if (user.avatarId) {
const avatar = await DriveFile.findOne({ _id: user.avatarId });
User.update({ _id: user._id }, {
$set: {
avatarUrl: avatar.metadata.thumbnailUrl,
}
});
}
if (user.bannerId) {
const banner = await DriveFile.findOne({ _id: user.bannerId });
User.update({ _id: user._id }, {
$set: {
bannerUrl: banner.metadata.webpublicUrl,
}
});
}
if (user.wallpaperId) {
const wallpaper = await DriveFile.findOne({ _id: user.wallpaperId });
User.update({ _id: user._id }, {
$set: {
wallpaperUrl: wallpaper.metadata.webpublicUrl,
}
});
}
return await pack(user, me, {
detail: true
});

View file

@ -29,12 +29,17 @@ router.get('/default-avatar.jpg', ctx => {
router.get('/app-default.jpg', ctx => {
const file = fs.createReadStream(`${__dirname}/assets/dummy.png`);
ctx.set('Content-Type', 'image/jpeg');
ctx.set('Content-Type', 'image/png');
ctx.body = file;
});
router.get('/:id', sendDriveFile);
router.get('/:id/*', sendDriveFile);
router.get('/thumbnail-not-available.png', ctx => {
const file = fs.createReadStream(`${__dirname}/assets/thumbnail-not-available.png`);
ctx.set('Content-Type', 'image/png');
ctx.body = file;
});
router.get('/*', sendDriveFile);
// Register router
app.use(router.routes());

View file

@ -1,10 +1,10 @@
import * as Koa from 'koa';
import * as send from 'koa-send';
import * as mongodb from 'mongodb';
import DriveFile, { getDriveFileBucket } from '../../models/drive-file';
import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
import { serverLogger } from '..';
import config from '../../config';
const assets = `${__dirname}/../../server/file/assets/`;
@ -14,16 +14,19 @@ const commonReadableHandlerGenerator = (ctx: Koa.BaseContext) => (e: Error): voi
};
export default async function(ctx: Koa.BaseContext) {
// Validate id
if (!mongodb.ObjectID.isValid(ctx.params.id)) {
ctx.throw(400, 'incorrect id');
return;
}
const fileId = new mongodb.ObjectID(ctx.params.id);
let url = ctx.href;
if (url.startsWith('http://') && config.url.startsWith('https://')) url = url.replace('http://', 'https://');
// Fetch drive file
const file = await DriveFile.findOne({ _id: fileId });
const file = await DriveFile.findOne({
$or: [{
'metadata.url': url
}, {
'metadata.webpublicUrl': url
}, {
'metadata.thumbnailUrl': url
}],
});
if (file == null) {
ctx.status = 404;
@ -43,21 +46,16 @@ export default async function(ctx: Koa.BaseContext) {
}
const sendRaw = async () => {
if (file.metadata && file.metadata.accessKey && file.metadata.accessKey != ctx.query['original']) {
ctx.status = 403;
return;
}
const bucket = await getDriveFileBucket();
const readable = bucket.openDownloadStream(fileId);
const readable = bucket.openDownloadStream(file._id);
readable.on('error', commonReadableHandlerGenerator(ctx));
ctx.set('Content-Type', file.contentType);
ctx.body = readable;
};
if ('thumbnail' in ctx.query) {
if (file.metadata.thumbnailUrl === url) {
const thumb = await DriveFileThumbnail.findOne({
'metadata.originalId': fileId
'metadata.originalId': file._id
});
if (thumb != null) {
@ -69,12 +67,12 @@ export default async function(ctx: Koa.BaseContext) {
await sendRaw();
} else {
ctx.status = 404;
await send(ctx as any, '/dummy.png', { root: assets });
await send(ctx as any, '/thumbnail-not-available.png', { root: assets });
}
}
} else if ('web' in ctx.query) {
} else if (file.metadata.webpublicUrl === url) {
const web = await DriveFileWebpublic.findOne({
'metadata.originalId': fileId
'metadata.originalId': file._id
});
if (web != null) {

View file

@ -2,7 +2,6 @@ import { Feed } from 'feed';
import config from '../../config';
import Note from '../../models/note';
import { IUser } from '../../models/user';
import { getOriginalUrl } from '../../misc/get-drive-file-url';
export default async function(user: IUser) {
const author: Author = {
@ -38,7 +37,7 @@ export default async function(user: IUser) {
} as FeedOptions);
for (const note of notes) {
const file = note._files && note._files.find(file => file.contentType.startsWith('image/'));
const file = note._files ? note._files.find(file => file.contentType.startsWith('image/')) : null;
feed.addItem({
title: `New note by ${author.name}`,
@ -46,7 +45,7 @@ export default async function(user: IUser) {
date: note.createdAt,
description: note.cw,
content: note.text,
image: file && getOriginalUrl(file)
image: file ? file.metadata.webpublicUrl : null
});
}

View file

@ -42,16 +42,16 @@ async function save(path: string, name: string, type: string, hash: string, size
// thunbnail, webpublic を必要なら生成
const alts = await generateAlts(path, type, !metadata.uri);
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
if (ext === '') {
if (type === 'image/jpeg') ext = '.jpg';
if (type === 'image/png') ext = '.png';
if (type === 'image/webp') ext = '.webp';
}
if (config.drive && config.drive.storage == 'minio') {
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
if (ext === '') {
if (type === 'image/jpeg') ext = '.jpg';
if (type === 'image/png') ext = '.png';
if (type === 'image/webp') ext = '.webp';
}
const baseUrl = config.drive.baseUrl
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
@ -117,12 +117,15 @@ async function save(path: string, name: string, type: string, hash: string, size
return file;
} else { // use MongoDB GridFS
Object.assign(metadata, {
url: `${config.driveUrl}/${uuid.v4()}${ext}`,
webpublicUrl: `${config.driveUrl}/${uuid.v4()}.${alts.webpublic.ext}`,
thumbnailUrl: `${config.driveUrl}/${uuid.v4()}.${alts.thumbnail.ext}`,
} as IMetadata);
// #region store original
const originalDst = await getDriveFileBucket();
// web用(Exif削除済み)がある場合はオリジナルにアクセス制限
if (alts.webpublic) metadata.accessKey = uuid.v4();
const originalFile = await storeOriginal(originalDst, name, path, type, metadata);
logger.info(`original stored to ${originalFile._id}`);
@ -273,7 +276,7 @@ async function deleteOldFile(user: IRemoteUser) {
* @param sensitive Mark file as sensitive
* @return Created drive file
*/
export default async function(
export async function addDriveFile(
user: IUser,
path: string,
name: string = null,

View file

@ -4,7 +4,7 @@ import * as tmp from 'tmp';
import * as request from 'request';
import { IDriveFile, validateFileName } from '../../models/drive-file';
import create from './add-file';
import { addDriveFile } from './add-file';
import config from '../../config';
import { IUser } from '../../models/user';
import * as mongodb from 'mongodb';
@ -83,7 +83,7 @@ export default async (
let error;
try {
driveFile = await create(user, path, name, null, folderId, force, link, url, uri, sensitive);
driveFile = await addDriveFile(user, path, name, null, folderId, force, link, url, uri, sensitive);
logger.succ(`Got: ${driveFile._id}`);
} catch (e) {
error = e;