heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Fix #4546 (#4548)

Browse source 
* Refactor download

* emoji type
This commit is contained in:
MeiMei 2019-03-21 04:50:44 +09:00 committed by syuilo
parent 1c79e30436
commit 7bebea087c
12 changed files with 151 additions and 237 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/misc/create-temp.ts Normal file
View file

@ -0,0 +1,10 @@
import * as tmp from 'tmp';
export function createTemp(): Promise<[string, any]> {
return new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
}

31
src/misc/detect-mine.ts Normal file
View file

@ -0,0 +1,31 @@
import * as fs from 'fs';
import fileType from 'file-type';
import checkSvg from '../misc/check-svg';
export async function detectMine(path: string) {
return new Promise<[string, string]>((res, rej) => {
const readable = fs.createReadStream(path);
readable
.on('error', rej)
.once('data', (buffer: Buffer) => {
readable.destroy();
const type = fileType(buffer);
if (type) {
if (type.mime == 'application/xml' && checkSvg(path)) {
res(['image/svg+xml', 'svg']);
} else {
res([type.mime, type.ext]);
}
} else if (checkSvg(path)) {
res(['image/svg+xml', 'svg']);
} else {
// 種類が同定できなかったら application/octet-stream にする
res(['application/octet-stream', null]);
}
})
.on('end', () => {
// maybe 0 bytes
res(['application/octet-stream', null]);
});
});
}

15
src/misc/detect-url-mine.ts Normal file
View file

@ -0,0 +1,15 @@
import { createTemp } from './create-temp';
import { downloadUrl } from './donwload-url';
import { detectMine } from './detect-mine';
export async function detectUrlMine(url: string) {
const [path, cleanup] = await createTemp();
try {
await downloadUrl(url, path);
const [type] = await detectMine(path);
return type;
} finally {
cleanup();
}
}

61
src/misc/donwload-url.ts Normal file
View file

@ -0,0 +1,61 @@
import * as fs from 'fs';
import * as URL from 'url';
import * as request from 'request';
import config from '../config';
import chalk from 'chalk';
import Logger from '../services/logger';
export async function downloadUrl(url: string, path: string) {
const logger = new Logger('download-url');
await new Promise((res, rej) => {
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const writable = fs.createWriteStream(path);
writable.on('finish', () => {
logger.succ(`Download finished: ${chalk.cyan(url)}`);
res();
});
writable.on('error', error => {
logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
rej(error);
});
const requestUrl = URL.parse(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url;
const req = request({
url: requestUrl,
proxy: config.proxy,
timeout: 10 * 1000,
headers: {
'User-Agent': config.userAgent
}
});
req.pipe(writable);
req.on('response', response => {
if (response.statusCode !== 200) {
logger.error(`Got ${response.statusCode} (${url})`);
writable.close();
rej(response.statusCode);
}
});
req.on('error', error => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
logger.succ(`Downloaded to: ${path}`);
});
}

76
src/misc/download-text-file.ts
View file

@ -1,79 +1,25 @@
import * as tmp from 'tmp';
import * as fs from 'fs';
import * as util from 'util';
import chalk from 'chalk';
import * as request from 'request';
import Logger from '../services/logger';
import config from '../config';
import { createTemp } from './create-temp';
import { downloadUrl } from './donwload-url';
const logger = new Logger('download-text-file');
export async function downloadTextFile(url: string): Promise<string> {
// Create temp file
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
const [path, cleanup] = await createTemp();
logger.info(`Temp file is ${path}`);
// write content at URL to temp file
await new Promise((res, rej) => {
logger.info(`Downloading ${chalk.cyan(url)} ...`);
try {
// write content at URL to temp file
await downloadUrl(url, path);
const writable = fs.createWriteStream(path);
const text = await util.promisify(fs.readFile)(path, 'utf8');
writable.on('finish', () => {
logger.succ(`Download finished: ${chalk.cyan(url)}`);
res();
});
writable.on('error', error => {
logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
rej(error);
});
const requestUrl = new URL(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url;
const req = request({
url: requestUrl,
proxy: config.proxy,
timeout: 10 * 1000,
headers: {
'User-Agent': config.userAgent
}
});
req.pipe(writable);
req.on('response', response => {
if (response.statusCode !== 200) {
logger.error(`Got ${response.statusCode} (${url})`);
writable.close();
rej(response.statusCode);
}
});
req.on('error', error => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
});
logger.succ(`Downloaded to: ${path}`);
const text = await util.promisify(fs.readFile)(path, 'utf8');
cleanup();
return text;
return text;
} finally {
cleanup();
}
}

1
src/models/emoji.ts
View file

@ -17,4 +17,5 @@ export type IEmoji = {
updatedAt?: Date;
/** AP object id */
uri?: string;
type?: string;
};

2
src/remote/activitypub/renderer/emoji.ts
View file

@ -8,7 +8,7 @@ export default (emoji: IEmoji) => ({
updated: emoji.updatedAt != null ? emoji.updatedAt.toISOString() : new Date().toISOString,
icon: {
type: 'Image',
mediaType: 'image/png', //Mei-TODO
mediaType: emoji.type || 'image/png',
url: emoji.url
}
});

6
src/server/api/endpoints/admin/emoji/add.ts
View file

@ -1,6 +1,7 @@
import $ from 'cafy';
import Emoji from '../../../../../models/emoji';
import define from '../../../define';
import { detectUrlMine } from '../../../../../misc/detect-url-mine';
export const meta = {
desc: {
@ -29,12 +30,15 @@ export const meta = {
};
export default define(meta, async (ps) => {
const type = await detectUrlMine(ps.url);
const emoji = await Emoji.insert({
updatedAt: new Date(),
name: ps.name,
host: null,
aliases: ps.aliases,
url: ps.url
url: ps.url,
type,
});
return {

6
src/server/api/endpoints/admin/emoji/update.ts
View file

@ -2,6 +2,7 @@ import $ from 'cafy';
import Emoji from '../../../../../models/emoji';
import define from '../../../define';
import ID from '../../../../../misc/cafy-id';
import { detectUrlMine } from '../../../../../misc/detect-url-mine';
export const meta = {
desc: {
@ -39,12 +40,15 @@ export default define(meta, async (ps) => {
if (emoji == null) throw new Error('emoji not found');
const type = await detectUrlMine(ps.url);
await Emoji.update({ _id: emoji._id }, {
$set: {
updatedAt: new Date(),
name: ps.name,
aliases: ps.aliases,
url: ps.url
url: ps.url,
type,
}
});

85
src/server/proxy/proxy-media.ts
View file

@ -1,27 +1,19 @@
import * as fs from 'fs';
import * as URL from 'url';
import * as tmp from 'tmp';
import * as Koa from 'koa';
import * as request from 'request';
import fileType from 'file-type';
import { serverLogger } from '..';
import config from '../../config';
import { IImage, ConvertToPng, ConvertToJpeg } from '../../services/drive/image-processor';
import checkSvg from '../../misc/check-svg';
import { createTemp } from '../../misc/create-temp';
import { downloadUrl } from '../../misc/donwload-url';
import { detectMine } from '../../misc/detect-mine';
export async function proxyMedia(ctx: Koa.BaseContext) {
const url = 'url' in ctx.query ? ctx.query.url : 'https://' + ctx.params.url;
// Create temp file
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
const [path, cleanup] = await createTemp();
try {
await fetch(url, path);
await downloadUrl(url, path);
const [type, ext] = await detectMine(path);
@ -54,70 +46,3 @@ export async function proxyMedia(ctx: Koa.BaseContext) {
cleanup();
}
}
async function fetch(url: string, path: string) {
await new Promise((res, rej) => {
const writable = fs.createWriteStream(path);
writable.on('finish', () => {
res();
});
writable.on('error', error => {
rej(error);
});
const requestUrl = URL.parse(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url;
const req = request({
url: requestUrl,
proxy: config.proxy,
timeout: 10 * 1000,
headers: {
'User-Agent': config.userAgent
}
});
req.pipe(writable);
req.on('response', response => {
if (response.statusCode !== 200) {
writable.close();
rej(response.statusCode);
}
});
req.on('error', error => {
writable.close();
rej(error);
});
});
}
async function detectMine(path: string) {
return new Promise<[string, string]>((res, rej) => {
const readable = fs.createReadStream(path);
readable
.on('error', rej)
.once('data', (buffer: Buffer) => {
readable.destroy();
const type = fileType(buffer);
if (type) {
if (type.mime == 'application/xml' && checkSvg(path)) {
res(['image/svg+xml', 'svg']);
} else {
res([type.mime, type.ext]);
}
} else if (checkSvg(path)) {
res(['image/svg+xml', 'svg']);
} else {
// 種類が同定できなかったら application/octet-stream にする
res(['application/octet-stream', null]);
}
})
.on('end', () => {
// maybe 0 bytes
res(['application/octet-stream', null]);
});
});
}

32
src/services/drive/add-file.ts
View file

@ -6,7 +6,6 @@ import * as crypto from 'crypto';
import * as Minio from 'minio';
import * as uuid from 'uuid';
import * as sharp from 'sharp';
import fileType from 'file-type';
import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file';
import DriveFolder from '../../models/drive-folder';
@ -25,8 +24,8 @@ import { GenerateVideoThumbnail } from './generate-video-thumbnail';
import { driveLogger } from './logger';
import { IImage, ConvertToJpeg, ConvertToWebp, ConvertToPng } from './image-processor';
import Instance from '../../models/instance';
import checkSvg from '../../misc/check-svg';
import { contentDisposition } from '../../misc/content-disposition';
import { detectMine } from '../../misc/detect-mine';
const logger = driveLogger.createSubLogger('register', 'yellow');
@ -306,33 +305,6 @@ export default async function(
});
});
// Detect content type
const detectMime = new Promise<[string, string]>((res, rej) => {
const readable = fs.createReadStream(path);
readable
.on('error', rej)
.once('data', (buffer: Buffer) => {
readable.destroy();
const type = fileType(buffer);
if (type) {
if (type.mime == 'application/xml' && checkSvg(path)) {
res(['image/svg+xml', 'svg']);
} else {
res([type.mime, type.ext]);
}
} else if (checkSvg(path)) {
res(['image/svg+xml', 'svg']);
} else {
// 種類が同定できなかったら application/octet-stream にする
res(['application/octet-stream', null]);
}
})
.on('end', () => {
// maybe 0 bytes
res(['application/octet-stream', null]);
});
});
// Get file size
const getFileSize = new Promise<number>((res, rej) => {
fs.stat(path, (err, stats) => {
@ -341,7 +313,7 @@ export default async function(
});
});
const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMime, getFileSize]);
const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMine(path), getFileSize]);
logger.info(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`);

63
src/services/drive/upload-from-url.ts
View file

@ -1,15 +1,12 @@
import * as fs from 'fs';
import * as URL from 'url';
import * as tmp from 'tmp';
import * as request from 'request';
import { IDriveFile, validateFileName } from '../../models/drive-file';
import create from './add-file';
import config from '../../config';
import { IUser } from '../../models/user';
import * as mongodb from 'mongodb';
import { driveLogger } from './logger';
import chalk from 'chalk';
import { createTemp } from '../../misc/create-temp';
import { downloadUrl } from '../../misc/donwload-url';
const logger = driveLogger.createSubLogger('downloader');
@ -28,62 +25,10 @@ export default async (
}
// Create temp file
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
const [path, cleanup] = await createTemp();
// write content at URL to temp file
await new Promise((res, rej) => {
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const writable = fs.createWriteStream(path);
writable.on('finish', () => {
logger.succ(`Download finished: ${chalk.cyan(url)}`);
res();
});
writable.on('error', error => {
logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
rej(error);
});
const requestUrl = URL.parse(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url;
const req = request({
url: requestUrl,
proxy: config.proxy,
timeout: 10 * 1000,
headers: {
'User-Agent': config.userAgent
}
});
req.pipe(writable);
req.on('response', response => {
if (response.statusCode !== 200) {
logger.error(`Got ${response.statusCode} (${url})`);
writable.close();
rej(response.statusCode);
}
});
req.on('error', error => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
});
await downloadUrl(url, path);
let driveFile: IDriveFile;
let error;