profxadke/misskey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

perf: Improve network request performance (#7636)

Browse source 
* perf: Improve fetch

* CHANGELOG

* lifo
This commit is contained in:
MeiMei 2021-08-16 17:44:43 +09:00 • committed by GitHub
parent 1b84ae9f3f
commit def32107af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 44 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/@types/lookup-dns-cache.d.ts vendored
View file

@ -1,9 +0,0 @@
declare module 'lookup-dns-cache' {
import { LookupOneOptions, LookupAllOptions, LookupOptions, LookupAddress } from 'dns';
function lookup(hostname: string, family: number, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void): void;
function lookup(hostname: string, options: LookupOneOptions, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void): void;
function lookup(hostname: string, options: LookupAllOptions, callback: (err: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void): void;
function lookup(hostname: string, options: LookupOptions, callback: (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family: number) => void): void;
function lookup(hostname: string, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void): void;
}

33
src/misc/fetch.ts
View file

@ -1,9 +1,8 @@
import * as http from 'http';
import * as https from 'https';
import * as cache from 'lookup-dns-cache';
import CacheableLookup from 'cacheable-lookup';
import fetch, { HeadersInit } from 'node-fetch';
import { HttpProxyAgent } from 'http-proxy-agent';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import config from '@/config';
import { URL } from 'url';
@ -49,6 +48,12 @@ export async function getHtml(url: string, accept = 'text/html, */*', timeout =
return await res.text();
}
const cache = new CacheableLookup({
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});
/**
* Get http non-proxy agent
*/
@ -65,20 +70,36 @@ const _https = new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
});
} as https.AgentOptions);
const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
/**
* Get http proxy or non-proxy agent
*/
export const httpAgent = config.proxy
? new HttpProxyAgent(config.proxy)
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy
})
: _http;
/**
* Get https proxy or non-proxy agent
*/
export const httpsAgent = config.proxy
? new HttpsProxyAgent(config.proxy)
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy
})
: _https;
/**