Use node-fetch instead of request (#6228)

* requestをnode-fetchになど

* format

* fix error

* t

* Fix test
This commit is contained in:
MeiMei 2020-04-09 23:42:23 +09:00 committed by GitHub
parent bb7edfee04
commit d3c0f3c251
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 140 additions and 218 deletions

View File

@ -72,6 +72,7 @@
"@types/markdown-it": "0.0.9",
"@types/mocha": "7.0.2",
"@types/node": "13.11.0",
"@types/node-fetch": "2.5.5",
"@types/nodemailer": "6.4.0",
"@types/nprogress": "0.2.0",
"@types/oauth": "0.9.1",
@ -84,8 +85,6 @@
"@types/ratelimiter": "2.1.28",
"@types/redis": "2.8.17",
"@types/rename": "1.0.1",
"@types/request": "2.48.4",
"@types/request-promise-native": "1.0.17",
"@types/request-stats": "3.0.0",
"@types/rimraf": "2.0.3",
"@types/seedrandom": "2.4.28",
@ -102,7 +101,6 @@
"@types/websocket": "1.0.0",
"@types/ws": "7.2.3",
"@typescript-eslint/parser": "2.26.0",
"agentkeepalive": "4.1.0",
"animejs": "3.1.0",
"apexcharts": "3.17.1",
"autobind-decorator": "2.4.0",
@ -145,6 +143,7 @@
"gulp-typescript": "5.0.1",
"hard-source-webpack-plugin": "0.13.1",
"html-minifier": "4.0.0",
"http-proxy-agent": "4.0.1",
"http-signature": "1.3.4",
"https-proxy-agent": "5.0.0",
"insert-text-at-cursor": "0.3.0",
@ -205,8 +204,6 @@
"redis-lock": "0.1.4",
"reflect-metadata": "0.1.13",
"rename": "1.0.4",
"request": "2.88.2",
"request-promise-native": "1.0.8",
"request-stats": "3.0.0",
"require-all": "3.0.0",
"rimraf": "3.0.2",

View File

@ -1,5 +1,6 @@
import * as fs from 'fs';
import * as request from 'request';
import fetch from 'node-fetch';
import { httpAgent, httpsAgent } from './fetch';
import config from '../config';
import * as chalk from 'chalk';
import Logger from '../services/logger';
@ -7,11 +8,35 @@ import Logger from '../services/logger';
export async function downloadUrl(url: string, path: string) {
const logger = new Logger('download');
await new Promise((res, rej) => {
logger.info(`Downloading ${chalk.cyan(url)} ...`);
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const response = await fetch(new URL(url).href, {
headers: {
'User-Agent': config.userAgent
},
timeout: 10 * 1000,
agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
}).then(response => {
if (!response.ok) {
logger.error(`Got ${response.status} (${url})`);
throw response.status;
} else {
return response;
}
});
await new Promise((res, rej) => {
const writable = fs.createWriteStream(path);
response.body.on('error', (error: any) => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
writable.on('finish', () => {
logger.succ(`Download finished: ${chalk.cyan(url)}`);
res();
@ -25,35 +50,6 @@ export async function downloadUrl(url: string, path: string) {
rej(error);
});
const req = request({
url: new URL(url).href, // https://github.com/syuilo/misskey/issues/2637
proxy: config.proxy,
timeout: 10 * 1000,
forever: true,
headers: {
'User-Agent': config.userAgent
}
});
req.pipe(writable);
req.on('response', response => {
if (response.statusCode !== 200) {
logger.error(`Got ${response.statusCode} (${url})`);
writable.close();
rej(response.statusCode);
}
});
req.on('error', error => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
logger.succ(`Downloaded to: ${path}`);
response.body.pipe(writable);
});
}

43
src/misc/fetch.ts Normal file
View File

@ -0,0 +1,43 @@
import * as http from 'http';
import * as https from 'https';
import * as cache from 'lookup-dns-cache';
import fetch, { HeadersInit } from 'node-fetch';
import { HttpProxyAgent } from 'http-proxy-agent';
import { HttpsProxyAgent } from 'https-proxy-agent';
import config from '../config';
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) {
const res = await fetch(url, {
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: accept
}, headers || {}),
timeout,
agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
});
if (!res.ok) {
throw {
name: `StatusError`,
statusCode: res.status,
message: `${res.status} ${res.statusText}`,
};
}
return await res.json();
}
export const httpAgent = config.proxy
? new HttpProxyAgent(config.proxy)
: new http.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
});
export const httpsAgent = config.proxy
? new HttpsProxyAgent(config.proxy)
: new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
});

View File

@ -1,19 +1,12 @@
import * as https from 'https';
import { sign } from 'http-signature';
import * as crypto from 'crypto';
import * as cache from 'lookup-dns-cache';
import config from '../../config';
import { ILocalUser } from '../../models/entities/user';
import { UserKeypairs } from '../../models';
import { ensure } from '../../prelude/ensure';
import { HttpsProxyAgent } from 'https-proxy-agent';
const agent = config.proxy
? new HttpsProxyAgent(config.proxy)
: new https.Agent({
lookup: cache.lookup,
});
import { httpsAgent } from '../../misc/fetch';
export default async (user: ILocalUser, url: string, object: any) => {
const timeout = 10 * 1000;
@ -32,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => {
await new Promise((resolve, reject) => {
const req = https.request({
agent,
agent: httpsAgent,
protocol,
hostname,
port,

View File

@ -1,10 +1,8 @@
import * as request from 'request-promise-native';
import { getJson } from '../../misc/fetch';
import { IObject, isCollectionOrOrderedCollection, ICollection, IOrderedCollection } from './type';
import config from '../../config';
export default class Resolver {
private history: Set<string>;
private timeout = 10 * 1000;
constructor() {
this.history = new Set();
@ -41,24 +39,7 @@ export default class Resolver {
this.history.add(value);
const object = await request({
url: value,
proxy: config.proxy,
timeout: this.timeout,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/activity+json, application/ld+json'
},
json: true
}).catch(e => {
const message = `${e.name}: ${e.message ? e.message.substr(0, 200) : undefined}, url=${value}`;
throw {
name: e.name,
statusCode: e.statusCode,
message,
};
});
const object = await getJson(value, 'application/activity+json, application/ld+json');
if (object == null || (
Array.isArray(object['@context']) ?

View File

@ -1,5 +1,4 @@
import config from '../config';
import * as request from 'request-promise-native';
import { getJson } from '../misc/fetch';
import { query as urlQuery } from '../prelude/url';
type ILink = {
@ -15,17 +14,7 @@ type IWebFinger = {
export default async function(query: string): Promise<IWebFinger> {
const url = genUrl(query);
return await request({
url,
proxy: config.proxy,
timeout: 10 * 1000,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/jrd+json, application/json'
},
json: true
});
return await getJson(url, 'application/jrd+json, application/json');
}
function genUrl(query: string) {

View File

@ -1,6 +1,6 @@
import * as Koa from 'koa';
import * as Router from '@koa/router';
import * as request from 'request';
import { getJson } from '../../../misc/fetch';
import { OAuth2 } from 'oauth';
import config from '../../../config';
import { publishMainStream } from '../../../services/stream';
@ -174,20 +174,9 @@ router.get('/dc/cb', async ctx => {
}
}));
const { id, username, discriminator } = await new Promise<any>((res, rej) =>
request({
url: 'https://discordapp.com/api/users/@me',
headers: {
'Authorization': `Bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err) {
rej(err);
} else {
res(JSON.parse(body));
}
}));
const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
'Authorization': `Bearer ${accessToken}`,
});
if (!id || !username || !discriminator) {
ctx.throw(400, 'invalid session');
@ -256,21 +245,9 @@ router.get('/dc/cb', async ctx => {
}
}));
const { id, username, discriminator } = await new Promise<any>((res, rej) =>
request({
url: 'https://discordapp.com/api/users/@me',
headers: {
'Authorization': `Bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err) {
rej(err);
} else {
res(JSON.parse(body));
}
}));
const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
'Authorization': `Bearer ${accessToken}`,
});
if (!id || !username || !discriminator) {
ctx.throw(400, 'invalid session');
return;

View File

@ -1,6 +1,6 @@
import * as Koa from 'koa';
import * as Router from '@koa/router';
import * as request from 'request';
import { getJson } from '../../../misc/fetch';
import { OAuth2 } from 'oauth';
import config from '../../../config';
import { publishMainStream } from '../../../services/stream';
@ -167,21 +167,9 @@ router.get('/gh/cb', async ctx => {
}
}));
const { login, id } = await new Promise<any>((res, rej) =>
request({
url: 'https://api.github.com/user',
headers: {
'Accept': 'application/vnd.github.v3+json',
'Authorization': `bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err)
rej(err);
else
res(JSON.parse(body));
}));
const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
'Authorization': `bearer ${accessToken}`
});
if (!login || !id) {
ctx.throw(400, 'invalid session');
return;
@ -230,20 +218,9 @@ router.get('/gh/cb', async ctx => {
res({ accessToken });
}));
const { login, id } = await new Promise<any>((res, rej) =>
request({
url: 'https://api.github.com/user',
headers: {
'Accept': 'application/vnd.github.v3+json',
'Authorization': `bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err)
rej(err);
else
res(JSON.parse(body));
}));
const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
'Authorization': `bearer ${accessToken}`
});
if (!login || !id) {
ctx.throw(400, 'invalid session');

View File

@ -1,10 +1,10 @@
import * as Koa from 'koa';
import * as request from 'request-promise-native';
import summaly from 'summaly';
import { fetchMeta } from '../../misc/fetch-meta';
import Logger from '../../services/logger';
import config from '../../config';
import { query } from '../../prelude/url';
import { getJson } from '../../misc/fetch';
const logger = new Logger('url-preview');
@ -16,15 +16,10 @@ module.exports = async (ctx: Koa.Context) => {
: `Getting preview of ${ctx.query.url}@${ctx.query.lang} ...`);
try {
const summary = meta.summalyProxy ? await request.get({
url: meta.summalyProxy,
qs: {
url: ctx.query.url,
lang: ctx.query.lang || 'ja-JP'
},
forever: true,
json: true
}) : await summaly(ctx.query.url, {
const summary = meta.summalyProxy ? await getJson(`${meta.summalyProxy}?${query({
url: ctx.query.url,
lang: ctx.query.lang || 'ja-JP'
})}`) : await summaly(ctx.query.url, {
followRedirects: false,
lang: ctx.query.lang || 'ja-JP'
});

View File

@ -1,32 +1,17 @@
import * as S3 from 'aws-sdk/clients/s3';
import config from '../../config';
import { Meta } from '../../models/entities/meta';
import { HttpsProxyAgent } from 'https-proxy-agent';
import * as agentkeepalive from 'agentkeepalive';
const httpsAgent = config.proxy
? new HttpsProxyAgent(config.proxy)
: new agentkeepalive.HttpsAgent({
freeSocketTimeout: 30 * 1000
});
import { httpsAgent, httpAgent } from '../../misc/fetch';
export function getS3(meta: Meta) {
const conf = {
return new S3({
endpoint: meta.objectStorageEndpoint || undefined,
accessKeyId: meta.objectStorageAccessKey,
secretAccessKey: meta.objectStorageSecretKey,
accessKeyId: meta.objectStorageAccessKey!,
secretAccessKey: meta.objectStorageSecretKey!,
region: meta.objectStorageRegion || undefined,
sslEnabled: meta.objectStorageUseSSL,
s3ForcePathStyle: !!meta.objectStorageEndpoint,
httpOptions: {
agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent
}
} as S3.ClientConfiguration;
if (meta.objectStorageUseSSL) {
conf.httpOptions!.agent = httpsAgent;
}
const s3 = new S3(conf);
return s3;
});
}

View File

@ -1,7 +1,6 @@
import * as request from 'request-promise-native';
import { getJson } from '../misc/fetch';
import { Instance } from '../models/entities/instance';
import { Instances } from '../models';
import config from '../config';
import { getNodeinfoLock } from '../misc/app-lock';
import Logger from '../services/logger';
@ -20,23 +19,14 @@ export async function fetchNodeinfo(instance: Instance) {
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await request({
url: 'https://' + instance.host + '/.well-known/nodeinfo',
proxy: config.proxy,
timeout: 1000 * 10,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/json, */*'
},
json: true
}).catch(e => {
if (e.statusCode === 404) {
throw 'No nodeinfo provided';
} else {
throw e.statusCode || e.message;
}
});
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
.catch(e => {
if (e.statusCode === 404) {
throw 'No nodeinfo provided';
} else {
throw e.statusCode || e.message;
}
});
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links';
@ -53,19 +43,10 @@ export async function fetchNodeinfo(instance: Instance) {
throw 'No nodeinfo link provided';
}
const info = await request({
url: link.href,
proxy: config.proxy,
timeout: 1000 * 10,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/json, */*'
},
json: true
}).catch(e => {
throw e.statusCode || e.message;
});
const info = await getJson(link.href)
.catch(e => {
throw e.statusCode || e.message;
});
await Instances.update(instance.id, {
infoUpdatedAt: new Date(),

View File

@ -1,7 +1,7 @@
import * as fs from 'fs';
import * as WebSocket from 'ws';
const fetch = require('node-fetch');
import * as req from 'request';
import fetch from 'node-fetch';
const FormData = require('form-data');
import * as childProcess from 'child_process';
export const async = (fn: Function) => (done: Function) => {
@ -20,6 +20,9 @@ export const request = async (endpoint: string, params: any, me?: any): Promise<
try {
const res = await fetch('http://localhost:8080/api' + endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(Object.assign(auth, params))
});
@ -64,18 +67,23 @@ export const react = async (user: any, note: any, reaction: string): Promise<any
}, user);
};
export const uploadFile = (user: any, path?: string): Promise<any> => new Promise((ok, rej) => {
req.post({
url: 'http://localhost:8080/api/drive/files/create',
formData: {
i: user.token,
file: fs.createReadStream(path || __dirname + '/resources/Lenna.png')
},
json: true
}, (err, httpResponse, body) => {
ok(body);
});
});
export const uploadFile = (user: any, path?: string): Promise<any> => {
const formData = new FormData();
formData.append('i', user.token);
formData.append('file', fs.createReadStream(path || __dirname + '/resources/Lenna.png'));
return fetch('http://localhost:8080/api/drive/files/create', {
method: 'post',
body: formData,
timeout: 30 * 1000,
}).then(res => {
if (!res.ok) {
throw `${res.status} ${res.statusText}`;
} else {
return res.json();
}
});
};
export function connectStream(user: any, channel: string, listener: (message: Record<string, any>) => any, params?: any): Promise<WebSocket> {
return new Promise((res, rej) => {