heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Image for web publish (#3402)

Browse source 
* Image for Web

* Add comment

* Make main to original
This commit is contained in:
MeiMei 2018-11-26 04:25:48 +09:00 committed by syuilo
parent 0863e5d379
commit bcb04924ff
14 changed files with 283 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/misc/get-drive-file-url.ts
View file

@ -6,15 +6,24 @@ export default function(file: IDriveFile, thumbnail = false): string {
if (file.metadata.withoutChunks) {
if (thumbnail) {
return file.metadata.thumbnailUrl || file.metadata.url;
return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || file.metadata.url;
} else {
return file.metadata.url;
return file.metadata.webpublicUrl || file.metadata.url;
}
} else {
if (thumbnail) {
return `${config.drive_url}/${file._id}?thumbnail`;
} else {
return `${config.drive_url}/${file._id}`;
return `${config.drive_url}/${file._id}?web`;
}
}
}
export function getOriginalUrl(file: IDriveFile) {
if (file.metadata && file.metadata.url) {
return file.metadata.url;
}
const accessKey = file.metadata ? file.metadata.accessKey : null;
return `${config.drive_url}/${file._id}${accessKey ? '?original=' + accessKey : ''}`;
}

29
src/models/drive-file-webpublic.ts Normal file
View file

@ -0,0 +1,29 @@
import * as mongo from 'mongodb';
import monkDb, { nativeDbConn } from '../db/mongodb';
const DriveFileWebpublic = monkDb.get<IDriveFileWebpublic>('driveFileWebpublics.files');
DriveFileWebpublic.createIndex('metadata.originalId', { sparse: true, unique: true });
export default DriveFileWebpublic;
export const DriveFileWebpublicChunk = monkDb.get('driveFileWebpublics.chunks');
export const getDriveFileWebpublicBucket = async (): Promise<mongo.GridFSBucket> => {
const db = await nativeDbConn();
const bucket = new mongo.GridFSBucket(db, {
bucketName: 'driveFileWebpublics'
});
return bucket;
};
export type IMetadata = {
originalId: mongo.ObjectID;
};
export type IDriveFileWebpublic = {
_id: mongo.ObjectID;
uploadDate: Date;
md5: string;
filename: string;
contentType: string;
metadata: IMetadata;
};

67
src/models/drive-file.ts
View file

@ -3,7 +3,7 @@ const deepcopy = require('deepcopy');
import { pack as packFolder } from './drive-folder';
import monkDb, { nativeDbConn } from '../db/mongodb';
import isObjectId from '../misc/is-objectid';
import getDriveFileUrl from '../misc/get-drive-file-url';
import getDriveFileUrl, { getOriginalUrl } from '../misc/get-drive-file-url';
const DriveFile = monkDb.get<IDriveFile>('driveFiles.files');
DriveFile.createIndex('md5');
@ -28,21 +28,48 @@ export type IMetadata = {
_user: any;
folderId: mongo.ObjectID;
comment: string;
/**
* リモートインスタンスから取得した場合の元URL
*/
uri?: string;
/**
* URL for web(生成されている場合) or original
* * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
*/
url?: string;
/**
* URL for thumbnail (thumbnailがなければなし)
* * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
*/
thumbnailUrl?: string;
/**
* URL for original (web用が生成されてない場合はurlがoriginalを指す)
* * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
*/
webpublicUrl?: string;
accessKey?: string;
src?: string;
deletedAt?: Date;
/**
* このファイルの中身データがMongoDB内に保存されているのか否か
* このファイルの中身データがMongoDB内に保存されていないか否か
* オブジェクトストレージを利用している or リモートサーバーへの直リンクである
* な場合は false になります
* な場合は true になります
*/
withoutChunks?: boolean;
storage?: string;
storageProps?: any;
/***
* ObjectStorage の格納先の情報
*/
storageProps?: IStorageProps;
isSensitive?: boolean;
/**
@ -56,6 +83,25 @@ export type IMetadata = {
isRemote?: boolean;
};
export type IStorageProps = {
/**
* ObjectStorage key for original
*/
key: string;
/***
* ObjectStorage key for thumbnail (thumbnailがなければなし)
*/
thumbnailKey?: string;
/***
* ObjectStorage key for webpublic (webpublicがなければなし)
*/
webpublicKey?: string;
id?: string;
};
export type IDriveFile = {
_id: mongo.ObjectID;
uploadDate: Date;
@ -83,7 +129,8 @@ export function validateFileName(name: string): boolean {
export const packMany = (
files: any[],
options?: {
detail: boolean
detail?: boolean
self?: boolean,
}
) => {
return Promise.all(files.map(f => pack(f, options)));
@ -95,11 +142,13 @@ export const packMany = (
export const pack = (
file: any,
options?: {
detail: boolean
detail?: boolean,
self?: boolean,
}
) => new Promise<any>(async (resolve, reject) => {
const opts = Object.assign({
detail: false
detail: false,
self: false
}, options);
let _file: any;
@ -165,5 +214,9 @@ export const pack = (
delete _target.isRemote;
delete _target._user;
if (opts.self) {
_target.url = getOriginalUrl(_file);
}
resolve(_target);
});

2
src/server/api/endpoints/drive/files.ts
View file

@ -77,5 +77,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
sort: sort
});
res(await packMany(files));
res(await packMany(files, { detail: false, self: true }));
}));

2
src/server/api/endpoints/drive/files/check_existence.ts
View file

@ -32,6 +32,6 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
if (file === null) {
res({ file: null });
} else {
res({ file: await pack(file) });
res({ file: await pack(file, { self: true }) });
}
}));

2
src/server/api/endpoints/drive/files/create.ts
View file

@ -74,7 +74,7 @@ export default define(meta, (ps, user, app, file, cleanup) => new Promise(async
cleanup();
res(pack(driveFile));
res(pack(driveFile, { self: true }));
} catch (e) {
console.error(e);

2
src/server/api/endpoints/drive/files/find.ts
View file

@ -31,5 +31,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
'metadata.folderId': ps.folderId
});
res(await Promise.all(files.map(file => pack(file))));
res(await Promise.all(files.map(file => pack(file, { self: true }))));
}));

3
src/server/api/endpoints/drive/files/show.ts
View file

@ -41,7 +41,8 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
// Serialize
const _file = await pack(file, {
detail: true
detail: true,
self: true
});
res(_file);

2
src/server/api/endpoints/drive/files/update.ts
View file

@ -111,7 +111,7 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
});
// Serialize
const fileObj = await pack(file);
const fileObj = await pack(file, { self: true });
// Response
res(fileObj);

2
src/server/api/endpoints/drive/files/upload_from_url.ts
View file

@ -50,5 +50,5 @@ export const meta = {
};
export default define(meta, (ps, user) => new Promise(async (res, rej) => {
res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force)));
res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force), { self: true }));
}));

2
src/server/api/endpoints/drive/stream.ts
View file

@ -65,5 +65,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
sort: sort
});
res(await packMany(files));
res(await packMany(files, { self: true }));
}));

19
src/server/file/send-drive-file.ts
View file

@ -3,6 +3,7 @@ import * as send from 'koa-send';
import * as mongodb from 'mongodb';
import DriveFile, { getDriveFileBucket } from '../../models/drive-file';
import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
const assets = `${__dirname}/../../server/file/assets/`;
@ -41,6 +42,11 @@ export default async function(ctx: Koa.Context) {
}
const sendRaw = async () => {
if (file.metadata && file.metadata.accessKey && file.metadata.accessKey != ctx.query['original']) {
ctx.status = 403;
return;
}
const bucket = await getDriveFileBucket();
const readable = bucket.openDownloadStream(fileId);
readable.on('error', commonReadableHandlerGenerator(ctx));
@ -60,6 +66,19 @@ export default async function(ctx: Koa.Context) {
} else {
await sendRaw();
}
} else if ('web' in ctx.query) {
const web = await DriveFileWebpublic.findOne({
'metadata.originalId': fileId
});
if (web != null) {
ctx.set('Content-Type', file.contentType);
const bucket = await getDriveFileWebpublicBucket();
ctx.body = bucket.openDownloadStream(web._id);
} else {
await sendRaw();
}
} else {
if ('download' in ctx.query) {
ctx.set('Content-Disposition', 'attachment');

159
src/services/drive/add-file.ts
View file

@ -16,6 +16,7 @@ import { publishMainStream, publishDriveStream } from '../../stream';
import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import delFile from './delete-file';
import config from '../../config';
import { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import driveChart from '../../chart/drive';
import perUserDriveChart from '../../chart/per-user-drive';
@ -23,7 +24,71 @@ import fetchMeta from '../../misc/fetch-meta';
const log = debug('misskey:drive:add-file');
async function save(path: string, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> {
/***
* Save file
* @param path Path for original
* @param name Name for original
* @param type Content-Type for original
* @param hash Hash for original
* @param size Size for original
* @param metadata
*/
async function save(path: string, name: string, type: string, hash: string, size: number, metadata: IMetadata): Promise<IDriveFile> {
// #region webpublic
let webpublic: Buffer;
let webpublicExt = 'jpg';
let webpublicType = 'image/jpeg';
if (!metadata.uri) { // from local instance
log(`creating web image`);
if (['image/jpeg'].includes(type)) {
webpublic = await sharp(path)
.resize(2048, 2048, {
fit: 'inside',
withoutEnlargement: true
})
.rotate()
.jpeg({
quality: 85,
progressive: true
})
.toBuffer();
} else if (['image/webp'].includes(type)) {
webpublic = await sharp(path)
.resize(2048, 2048, {
fit: 'inside',
withoutEnlargement: true
})
.rotate()
.webp({
quality: 85
})
.toBuffer();
webpublicExt = 'webp';
webpublicType = 'image/webp';
} else if (['image/png'].includes(type)) {
webpublic = await sharp(path)
.resize(2048, 2048, {
fit: 'inside',
withoutEnlargement: true
})
.rotate()
.png()
.toBuffer();
webpublicExt = 'png';
webpublicType = 'image/png';
} else {
log(`web image not created (not an image)`);
}
} else {
log(`web image not created (from remote)`);
}
// #endregion webpublic
// #region thumbnail
let thumbnail: Buffer;
let thumbnailExt = 'jpg';
let thumbnailType = 'image/jpeg';
@ -53,10 +118,9 @@ async function save(path: string, name: string, type: string, hash: string, size
thumbnailExt = 'png';
thumbnailType = 'image/png';
}
// #endregion thumbnail
if (config.drive && config.drive.storage == 'minio') {
const minio = new Minio.Client(config.drive.config);
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
if (ext === '') {
@ -66,33 +130,41 @@ async function save(path: string, name: string, type: string, hash: string, size
}
const key = `${config.drive.prefix}/${uuid.v4()}${ext}`;
const webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${webpublicExt}`;
const thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${thumbnailExt}`;
log(`uploading original: ${key}`);
const uploads = [
upload(key, fs.createReadStream(path), type)
];
if (webpublic) {
log(`uploading webpublic: ${webpublicKey}`);
uploads.push(upload(webpublicKey, webpublic, webpublicType));
}
if (thumbnail) {
log(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(upload(thumbnailKey, thumbnail, thumbnailType));
}
await Promise.all(uploads);
const baseUrl = config.drive.baseUrl
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
await minio.putObject(config.drive.bucket, key, fs.createReadStream(path), size, {
'Content-Type': type,
'Cache-Control': 'max-age=31536000, immutable'
});
if (thumbnail) {
await minio.putObject(config.drive.bucket, thumbnailKey, thumbnail, size, {
'Content-Type': thumbnailType,
'Cache-Control': 'max-age=31536000, immutable'
});
}
Object.assign(metadata, {
withoutChunks: true,
storage: 'minio',
storageProps: {
key: key,
thumbnailKey: thumbnailKey
webpublicKey: webpublic ? webpublicKey : null,
thumbnailKey: thumbnail ? thumbnailKey : null,
},
url: `${ baseUrl }/${ key }`,
webpublicUrl: webpublic ? `${ baseUrl }/${ webpublicKey }` : null,
thumbnailUrl: thumbnail ? `${ baseUrl }/${ thumbnailKey }` : null
});
} as IMetadata);
const file = await DriveFile.insert({
length: size,
@ -105,29 +177,55 @@ async function save(path: string, name: string, type: string, hash: string, size
return file;
} else {
// Get MongoDB GridFS bucket
const bucket = await getDriveFileBucket();
// #region store original
const originalDst = await getDriveFileBucket();
const file = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, {
// web用(Exif削除済み)がある場合はオリジナルにアクセス制限
if (webpublic) metadata.accessKey = uuid.v4();
const originalFile = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = originalDst.openUploadStream(name, {
contentType: type,
metadata
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
fs.createReadStream(path).pipe(writeStream);
});
log(`original stored to ${originalFile._id}`);
// #endregion store original
// #region store webpublic
if (webpublic) {
const webDst = await getDriveFileWebpublicBucket();
const webFile = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = webDst.openUploadStream(name, {
contentType: webpublicType,
metadata: {
originalId: originalFile._id
}
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
writeStream.end(webpublic);
});
log(`web stored ${webFile._id}`);
}
// #endregion store webpublic
if (thumbnail) {
const thumbnailBucket = await getDriveFileThumbnailBucket();
await new Promise<IDriveFile>((resolve, reject) => {
const tuhmFile = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = thumbnailBucket.openUploadStream(name, {
contentType: thumbnailType,
metadata: {
originalId: file._id
originalId: originalFile._id
}
});
@ -135,12 +233,23 @@ async function save(path: string, name: string, type: string, hash: string, size
writeStream.on('error', reject);
writeStream.end(thumbnail);
});
log(`thumbnail stored ${tuhmFile._id}`);
}
return file;
return originalFile;
}
}
async function upload(key: string, stream: fs.ReadStream | Buffer, type: string) {
const minio = new Minio.Client(config.drive.config);
await minio.putObject(config.drive.bucket, key, stream, null, {
'Content-Type': type,
'Cache-Control': 'max-age=31536000, immutable'
});
}
async function deleteOldFile(user: IRemoteUser) {
const oldFile = await DriveFile.findOne({
_id: {

20
src/services/drive/delete-file.ts
View file

@ -4,6 +4,7 @@ import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-
import config from '../../config';
import driveChart from '../../chart/drive';
import perUserDriveChart from '../../chart/per-user-drive';
import DriveFileWebpublic, { DriveFileWebpublicChunk } from '../../models/drive-file-webpublic';
export default async function(file: IDriveFile, isExpired = false) {
if (file.metadata.storage == 'minio') {
@ -20,6 +21,11 @@ export default async function(file: IDriveFile, isExpired = false) {
const thumbnailObj = file.metadata.storageProps.thumbnailKey ? file.metadata.storageProps.thumbnailKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-thumbnail`;
await minio.removeObject(config.drive.bucket, thumbnailObj);
}
if (file.metadata.webpublicUrl) {
const webpublicObj = file.metadata.storageProps.webpublicKey ? file.metadata.storageProps.webpublicKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-original`;
await minio.removeObject(config.drive.bucket, webpublicObj);
}
}
// チャンクをすべて削除
@ -48,6 +54,20 @@ export default async function(file: IDriveFile, isExpired = false) {
}
//#endregion
//#region Web公開用もあれば削除
const webpublic = await DriveFileWebpublic.findOne({
'metadata.originalId': file._id
});
if (webpublic) {
await DriveFileWebpublicChunk.remove({
files_id: webpublic._id
});
await DriveFileWebpublic.remove({ _id: webpublic._id });
}
//#endregion
// 統計を更新
driveChart.update(file, false);
perUserDriveChart.update(file, false);