format
This commit is contained in:
taskylizard
parent
af1d873de0
commit
a451755d2b
|
|
@ -0,0 +1,2 @@
|
|||
**/*.md
|
||||
pnpm-lock.yaml
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
proseWrap: always
|
||||
semi: false
|
||||
singleQuote: true
|
||||
printWidth: 80
|
||||
trailingComma: none
|
||||
htmlWhitespaceSensitivity: ignore
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
"scripts": {
|
||||
"clean": "rm -rf build",
|
||||
"prebuild": "npm run clean",
|
||||
"format": "prettier -w --cache --check .",
|
||||
"build": "tsc --build"
|
||||
},
|
||||
"author": "",
|
||||
|
|
@ -22,6 +23,7 @@
|
|||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.12.12",
|
||||
"dotenv": "^16.4.4"
|
||||
"dotenv": "^16.4.4",
|
||||
"prettier": "^3.2.5"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,6 +45,9 @@ importers:
|
|||
dotenv:
|
||||
specifier: ^16.4.4
|
||||
version: 16.4.4
|
||||
prettier:
|
||||
specifier: ^3.2.5
|
||||
version: 3.2.5
|
||||
|
||||
packages:
|
||||
|
||||
|
|
@ -292,6 +295,11 @@ packages:
|
|||
resolution: {integrity: sha512-Mz/gKiRyuXu4HnpHgi1YWdHQCoWMufapzooisvFn78zl4dZciAxS+YeRkUxXl1ee/SzU80YCz1zpECCh4oC6Aw==}
|
||||
hasBin: true
|
||||
|
||||
prettier@3.2.5:
|
||||
resolution: {integrity: sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==}
|
||||
engines: {node: '>=14'}
|
||||
hasBin: true
|
||||
|
||||
process-warning@2.3.2:
|
||||
resolution: {integrity: sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==}
|
||||
|
||||
|
|
@ -668,6 +676,8 @@ snapshots:
|
|||
sonic-boom: 3.8.0
|
||||
thread-stream: 2.4.1
|
||||
|
||||
prettier@3.2.5: {}
|
||||
|
||||
process-warning@2.3.2: {}
|
||||
|
||||
process-warning@3.0.0: {}
|
||||
|
|
|
|||
|
|
@ -1,89 +1,116 @@
|
|||
|
||||
import fastify, {
|
||||
FastifyInstance,
|
||||
FastifyListenOptions,
|
||||
FastifyReply,
|
||||
FastifyRequest,
|
||||
} from "fastify"
|
||||
import { PinoLoggerOptions } from "fastify/types/logger"
|
||||
import { Proxy } from "./proxy"
|
||||
import { Logger } from "pino"
|
||||
FastifyInstance,
|
||||
FastifyListenOptions,
|
||||
FastifyReply,
|
||||
FastifyRequest
|
||||
} from 'fastify'
|
||||
import { PinoLoggerOptions } from 'fastify/types/logger'
|
||||
import { Proxy } from './proxy'
|
||||
import { Logger } from 'pino'
|
||||
import 'dotenv/config'
|
||||
|
||||
const host = process.env.HOST
|
||||
const port = parseInt(process.env.PORT ?? "8080", 10)
|
||||
const port = parseInt(process.env.PORT ?? '8080', 10)
|
||||
const baseUrl = process.env.NITTER_BASE_URL
|
||||
const concurrency = parseInt(process.env.CONCURRENCY ?? "1", 10)
|
||||
const retryAfterMillis = process.env.RETRY_AFTER_MILLIS ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) : null
|
||||
const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? "100000", 10)
|
||||
const logLevel = process.env.LOG_LEVEL ?? "debug"
|
||||
const concurrency = parseInt(process.env.CONCURRENCY ?? '1', 10)
|
||||
const retryAfterMillis = process.env.RETRY_AFTER_MILLIS
|
||||
? parseInt(process.env.RETRY_AFTER_MILLIS, 10)
|
||||
: null
|
||||
const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? '100000', 10)
|
||||
const logLevel = process.env.LOG_LEVEL ?? 'debug'
|
||||
|
||||
const server = fastify({
|
||||
logger: {
|
||||
name: "app",
|
||||
level: logLevel,
|
||||
...( logLevel == "trace" ? { transport: { target: 'pino-pretty' } } : {})
|
||||
} as PinoLoggerOptions
|
||||
logger: {
|
||||
name: 'app',
|
||||
level: logLevel,
|
||||
...(logLevel == 'trace' ? { transport: { target: 'pino-pretty' } } : {})
|
||||
} as PinoLoggerOptions
|
||||
})
|
||||
|
||||
const log = server.log as Logger
|
||||
const proxy = new Proxy(log, baseUrl, concurrency, retryAfterMillis, maxCacheSize)
|
||||
const proxy = new Proxy(
|
||||
log,
|
||||
baseUrl,
|
||||
concurrency,
|
||||
retryAfterMillis,
|
||||
maxCacheSize
|
||||
)
|
||||
|
||||
async function main() {
|
||||
server.register(
|
||||
(fastify: FastifyInstance, opts, done) => {
|
||||
fastify.get(
|
||||
`/user/:username`,
|
||||
{},
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
log.debug(
|
||||
{
|
||||
headers: request.headers,
|
||||
reqId: request.id,
|
||||
params: request.params
|
||||
},
|
||||
'incoming request /user/:username'
|
||||
)
|
||||
const { username } = request.params as any
|
||||
const { status, data } = await proxy.getUser(username, {
|
||||
reqId: request.id
|
||||
})
|
||||
reply.status(status).send(data)
|
||||
}
|
||||
)
|
||||
|
||||
server.register((fastify: FastifyInstance, opts, done) => {
|
||||
fastify.get(
|
||||
`/user/:userId/tweets`,
|
||||
{},
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { userId } = request.params as any
|
||||
const { cursor } = request.query as any
|
||||
const { status, data } = await proxy.getUserTweets(userId, cursor, {
|
||||
reqId: request.id
|
||||
})
|
||||
reply.status(status).send(data)
|
||||
}
|
||||
)
|
||||
|
||||
fastify.get(`/user/:username`, {},
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
log.debug({
|
||||
headers: request.headers,
|
||||
reqId: request.id,
|
||||
params: request.params }, 'incoming request /user/:username')
|
||||
const { username } = request.params as any
|
||||
const { status, data } = await proxy.getUser(username, { reqId: request.id })
|
||||
reply.status(status).send(data)
|
||||
});
|
||||
fastify.get(
|
||||
`/tweet/:id`,
|
||||
{},
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { id } = request.params as any
|
||||
const { status, data } = await proxy.getTweetById(id, {
|
||||
reqId: request.id
|
||||
})
|
||||
reply.status(status).send(data)
|
||||
}
|
||||
)
|
||||
|
||||
fastify.get(`/user/:userId/tweets`, {},
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { userId } = request.params as any
|
||||
const { cursor } = request.query as any
|
||||
const { status, data } = await proxy.getUserTweets(userId, cursor, { reqId: request.id })
|
||||
reply.status(status).send(data)
|
||||
});
|
||||
done()
|
||||
},
|
||||
{ prefix: '/api' }
|
||||
)
|
||||
|
||||
fastify.get(`/tweet/:id`, {},
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { id } = request.params as any
|
||||
const { status, data } = await proxy.getTweetById(id, { reqId: request.id })
|
||||
reply.status(status).send(data)
|
||||
});
|
||||
server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => {
|
||||
reply.status(404).send({ message: `Method not found` })
|
||||
})
|
||||
|
||||
done()
|
||||
server.setErrorHandler(
|
||||
(err: Error, request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { log } = request
|
||||
log.error(err)
|
||||
// Send error response
|
||||
reply.status(500).send({ message: `Internal server error` })
|
||||
}
|
||||
)
|
||||
|
||||
}, { prefix: '/api' })
|
||||
// await server.register(import('@fastify/rate-limit'), {
|
||||
// max: 100,
|
||||
// timeWindow: '1 minute'
|
||||
// })
|
||||
|
||||
server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => {
|
||||
reply.status(404)
|
||||
.send({ message: `Method not found` })
|
||||
})
|
||||
|
||||
server.setErrorHandler((err: Error, request: FastifyRequest, reply: FastifyReply) => {
|
||||
const { log } = request
|
||||
log.error(err)
|
||||
// Send error response
|
||||
reply.status(500).send({ message: `Internal server error` })
|
||||
})
|
||||
|
||||
// await server.register(import('@fastify/rate-limit'), {
|
||||
// max: 100,
|
||||
// timeWindow: '1 minute'
|
||||
// })
|
||||
|
||||
await server.listen({ port, host } as FastifyListenOptions);
|
||||
await server.listen({ port, host } as FastifyListenOptions)
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
log.fatal(err)
|
||||
process.exit(1)
|
||||
main().catch((err) => {
|
||||
log.fatal(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,209 +1,226 @@
|
|||
// noinspection TypeScriptUnresolvedReference
|
||||
|
||||
import axios from "axios"
|
||||
import { AxiosInstance, AxiosRequestConfig } from "axios"
|
||||
import fastq from "fastq"
|
||||
import { Logger } from "pino"
|
||||
import retry from "axios-retry-after"
|
||||
import axios from 'axios'
|
||||
import { AxiosInstance, AxiosRequestConfig } from 'axios'
|
||||
import fastq from 'fastq'
|
||||
import { Logger } from 'pino'
|
||||
import retry from 'axios-retry-after'
|
||||
import { LRUCache } from 'lru-cache'
|
||||
|
||||
const GET_USER_POSITIVE_TTL_MS = process.env.GET_USER_POSITIVE_TTL
|
||||
? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000
|
||||
: 30 * 24 * 3600 * 1000
|
||||
? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000
|
||||
: 30 * 24 * 3600 * 1000
|
||||
const GET_USER_NEGATIVE_TTL_MS = process.env.GET_USER_NEGATIVE_TTL
|
||||
? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000
|
||||
: 3600 * 1000
|
||||
? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000
|
||||
: 3600 * 1000
|
||||
const GET_TWEETS_POSITIVE_TTL_MS = process.env.GET_TWEETS_POSITIVE_TTL
|
||||
? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
const GET_TWEETS_NEGATIVE_TTL_MS = process.env.GET_TWEETS_NEGATIVE_TTL
|
||||
? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
const GET_TWEET_POSITIVE_TTL_MS = process.env.GET_TWEET_POSITIVE_TTL
|
||||
? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
const GET_TWEET_NEGATIVE_TTL_MS = process.env.GET_TWEET_NEGATIVE_TTL
|
||||
? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000
|
||||
: 60 * 1000
|
||||
|
||||
export interface Job {
|
||||
reqId: string
|
||||
url: string
|
||||
params?: Record<string, any>
|
||||
reqId: string
|
||||
url: string
|
||||
params?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface JobResponse {
|
||||
status: number,
|
||||
data: any
|
||||
status: number
|
||||
data: any
|
||||
}
|
||||
|
||||
export class Proxy {
|
||||
private readonly cache: LRUCache<string, JobResponse>
|
||||
private readonly client: AxiosInstance
|
||||
private readonly queue: fastq.queueAsPromised<Job, JobResponse>
|
||||
private counter: { requests: number }
|
||||
private timeWindowMillis = 15 * 60 * 1000
|
||||
private maxRequestsPerAccount = 15 * 60
|
||||
|
||||
private readonly cache: LRUCache<string, JobResponse>
|
||||
private readonly client: AxiosInstance
|
||||
private readonly queue: fastq.queueAsPromised<Job, JobResponse>
|
||||
private counter: { requests: number }
|
||||
private timeWindowMillis = 15 * 60 * 1000
|
||||
private maxRequestsPerAccount = 15 * 60
|
||||
|
||||
constructor(
|
||||
private log: Logger,
|
||||
private baseUrl: string,
|
||||
private concurrency: number,
|
||||
retryAfterMillis: number,
|
||||
maxCacheSize: number
|
||||
) {
|
||||
this.cache = new LRUCache({ max: maxCacheSize })
|
||||
this.queue = fastq.promise(this, this.sendRequest, concurrency)
|
||||
this.client = axios.create()
|
||||
this.counter = {
|
||||
requests: 0
|
||||
}
|
||||
|
||||
setInterval(() => {
|
||||
this.counter.requests = 0
|
||||
}, this.timeWindowMillis)
|
||||
|
||||
if ( retryAfterMillis ) {
|
||||
this.client.interceptors.response.use(null, retry(this.client, {
|
||||
// Determine when we should attempt to retry
|
||||
isRetryable (error) {
|
||||
log.debug({ status: error.response?.status, headers: error.response?.headers }, 'checking retryable')
|
||||
return (
|
||||
error.response && error.response.status === 429
|
||||
// Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds
|
||||
// && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60
|
||||
)
|
||||
},
|
||||
// Customize the wait behavior
|
||||
wait (error) {
|
||||
log.debug({ status: error.response?.status, headers: error.response?.headers }, 'waiting for retry')
|
||||
return new Promise(
|
||||
// Use X-Retry-After rather than Retry-After
|
||||
// resolve => setTimeout(resolve, error.response.headers['x-retry-after'])
|
||||
resolve => setTimeout(resolve, retryAfterMillis)
|
||||
)
|
||||
}
|
||||
}))
|
||||
}
|
||||
constructor(
|
||||
private log: Logger,
|
||||
private baseUrl: string,
|
||||
private concurrency: number,
|
||||
retryAfterMillis: number,
|
||||
maxCacheSize: number
|
||||
) {
|
||||
this.cache = new LRUCache({ max: maxCacheSize })
|
||||
this.queue = fastq.promise(this, this.sendRequest, concurrency)
|
||||
this.client = axios.create()
|
||||
this.counter = {
|
||||
requests: 0
|
||||
}
|
||||
|
||||
async getUser(username: string, options?: { reqId?: string }) {
|
||||
const key = `usernames:${username}`
|
||||
setInterval(() => {
|
||||
this.counter.requests = 0
|
||||
}, this.timeWindowMillis)
|
||||
|
||||
if ( this.cache.has(key)) {
|
||||
return this.cache.get(key)
|
||||
}
|
||||
|
||||
const result = await this.queue.push({
|
||||
url: `/api/user/${ username }`,
|
||||
reqId: options?.reqId
|
||||
if (retryAfterMillis) {
|
||||
this.client.interceptors.response.use(
|
||||
null,
|
||||
retry(this.client, {
|
||||
// Determine when we should attempt to retry
|
||||
isRetryable(error) {
|
||||
log.debug(
|
||||
{
|
||||
status: error.response?.status,
|
||||
headers: error.response?.headers
|
||||
},
|
||||
'checking retryable'
|
||||
)
|
||||
return (
|
||||
error.response && error.response.status === 429
|
||||
// Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds
|
||||
// && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60
|
||||
)
|
||||
},
|
||||
// Customize the wait behavior
|
||||
wait(error) {
|
||||
log.debug(
|
||||
{
|
||||
status: error.response?.status,
|
||||
headers: error.response?.headers
|
||||
},
|
||||
'waiting for retry'
|
||||
)
|
||||
return new Promise(
|
||||
// Use X-Retry-After rather than Retry-After
|
||||
// resolve => setTimeout(resolve, error.response.headers['x-retry-after'])
|
||||
(resolve) => setTimeout(resolve, retryAfterMillis)
|
||||
)
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if ( result.status === 200 ) {
|
||||
this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS })
|
||||
}
|
||||
if ( result.status === 404 ) {
|
||||
this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS })
|
||||
}
|
||||
async getUser(username: string, options?: { reqId?: string }) {
|
||||
const key = `usernames:${username}`
|
||||
|
||||
return result
|
||||
if (this.cache.has(key)) {
|
||||
return this.cache.get(key)
|
||||
}
|
||||
|
||||
async getUserTweets(userId: string, cursor?: string, options?: { reqId?: string }) {
|
||||
const key = `users:${userId}:tweets:${cursor ?? 'last'}`
|
||||
const result = await this.queue.push({
|
||||
url: `/api/user/${username}`,
|
||||
reqId: options?.reqId
|
||||
})
|
||||
|
||||
if ( this.cache.has(key) ) {
|
||||
return this.cache.get(key)
|
||||
}
|
||||
|
||||
const result = await this.queue.push({
|
||||
url: `/api/user/${ userId }/tweets`,
|
||||
params: { cursor },
|
||||
reqId: options?.reqId
|
||||
})
|
||||
|
||||
if ( result.status === 200 ) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS })
|
||||
}
|
||||
if ( result.status === 404 ) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS })
|
||||
}
|
||||
|
||||
return result
|
||||
if (result.status === 200) {
|
||||
this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS })
|
||||
}
|
||||
if (result.status === 404) {
|
||||
this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS })
|
||||
}
|
||||
|
||||
async getTweetById(tweetId: string, options?: { reqId?: string }) {
|
||||
const key = `tweets:${tweetId}`
|
||||
return result
|
||||
}
|
||||
|
||||
if ( this.cache.has(key) ) {
|
||||
return this.cache.get(key)
|
||||
}
|
||||
async getUserTweets(
|
||||
userId: string,
|
||||
cursor?: string,
|
||||
options?: { reqId?: string }
|
||||
) {
|
||||
const key = `users:${userId}:tweets:${cursor ?? 'last'}`
|
||||
|
||||
const result = await this.queue.push({
|
||||
url: `/api/tweet/${ tweetId }`,
|
||||
reqId: options?.reqId
|
||||
})
|
||||
|
||||
if ( result.status === 200 ) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS })
|
||||
}
|
||||
if ( result.status === 404 ) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS })
|
||||
}
|
||||
|
||||
return result
|
||||
if (this.cache.has(key)) {
|
||||
return this.cache.get(key)
|
||||
}
|
||||
|
||||
private async sendRequest(job: Job): Promise<any> {
|
||||
const result = await this.queue.push({
|
||||
url: `/api/user/${userId}/tweets`,
|
||||
params: { cursor },
|
||||
reqId: options?.reqId
|
||||
})
|
||||
|
||||
const { reqId, url, params } = job
|
||||
|
||||
if ( this.counter.requests > this.concurrency * this.maxRequestsPerAccount ) {
|
||||
return {
|
||||
status: 429
|
||||
}
|
||||
}
|
||||
|
||||
let config = {
|
||||
url,
|
||||
method: "get",
|
||||
baseURL: this.baseUrl,
|
||||
params,
|
||||
} as AxiosRequestConfig
|
||||
|
||||
this.log.trace({ config, reqId: reqId }, 'sending request to nitter')
|
||||
|
||||
try {
|
||||
const response = await this.client.request(config)
|
||||
|
||||
this.log.trace({
|
||||
status: response.status,
|
||||
data: response.data,
|
||||
reqId: reqId
|
||||
}, 'nitter response')
|
||||
|
||||
return {
|
||||
status: response.status,
|
||||
data: response.data,
|
||||
} as JobResponse
|
||||
|
||||
} catch(err) {
|
||||
|
||||
this.log.warn({ err, reqId }, 'nitter error')
|
||||
|
||||
if ( err.name === "AxiosError" ) {
|
||||
|
||||
this.counter.requests = Number.MAX_SAFE_INTEGER
|
||||
|
||||
return {
|
||||
status: 429
|
||||
} as JobResponse
|
||||
}
|
||||
|
||||
return {
|
||||
status: 500
|
||||
}
|
||||
}
|
||||
if (result.status === 200) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS })
|
||||
}
|
||||
if (result.status === 404) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS })
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
async getTweetById(tweetId: string, options?: { reqId?: string }) {
|
||||
const key = `tweets:${tweetId}`
|
||||
|
||||
if (this.cache.has(key)) {
|
||||
return this.cache.get(key)
|
||||
}
|
||||
|
||||
const result = await this.queue.push({
|
||||
url: `/api/tweet/${tweetId}`,
|
||||
reqId: options?.reqId
|
||||
})
|
||||
|
||||
if (result.status === 200) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS })
|
||||
}
|
||||
if (result.status === 404) {
|
||||
this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS })
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private async sendRequest(job: Job): Promise<any> {
|
||||
const { reqId, url, params } = job
|
||||
|
||||
if (this.counter.requests > this.concurrency * this.maxRequestsPerAccount) {
|
||||
return {
|
||||
status: 429
|
||||
}
|
||||
}
|
||||
|
||||
let config = {
|
||||
url,
|
||||
method: 'get',
|
||||
baseURL: this.baseUrl,
|
||||
params
|
||||
} as AxiosRequestConfig
|
||||
|
||||
this.log.trace({ config, reqId: reqId }, 'sending request to nitter')
|
||||
|
||||
try {
|
||||
const response = await this.client.request(config)
|
||||
|
||||
this.log.trace(
|
||||
{
|
||||
status: response.status,
|
||||
data: response.data,
|
||||
reqId: reqId
|
||||
},
|
||||
'nitter response'
|
||||
)
|
||||
|
||||
return {
|
||||
status: response.status,
|
||||
data: response.data
|
||||
} as JobResponse
|
||||
} catch (err) {
|
||||
this.log.warn({ err, reqId }, 'nitter error')
|
||||
|
||||
if (err.name === 'AxiosError') {
|
||||
this.counter.requests = Number.MAX_SAFE_INTEGER
|
||||
|
||||
return {
|
||||
status: 429
|
||||
} as JobResponse
|
||||
}
|
||||
|
||||
return {
|
||||
status: 500
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,39 +1,37 @@
|
|||
declare module 'axios-retry-after' {
|
||||
import { AxiosError, AxiosInstance } from 'axios'
|
||||
|
||||
import { AxiosError, AxiosInstance } from "axios";
|
||||
/**
|
||||
* Function to enhance Axios instance with retry-after functionality.
|
||||
* @param axios Axios instance to be enhanced.
|
||||
* @param options Configuration options for retry behavior.
|
||||
*/
|
||||
export default function (
|
||||
axios: AxiosInstance,
|
||||
options?: AxiosRetryAfterOptions
|
||||
): (error: AxiosError) => Promise<void>
|
||||
|
||||
/**
|
||||
* Configuration options for axios-retry-after.
|
||||
*/
|
||||
export interface AxiosRetryAfterOptions {
|
||||
/**
|
||||
* Function to determine if an error response is retryable.
|
||||
* @param error The Axios error to evaluate.
|
||||
*/
|
||||
isRetryable?: (error: AxiosError) => boolean
|
||||
|
||||
/**
|
||||
* Function to enhance Axios instance with retry-after functionality.
|
||||
* @param axios Axios instance to be enhanced.
|
||||
* @param options Configuration options for retry behavior.
|
||||
* Function to wait for a specified amount of time.
|
||||
* @param error The Axios error that contains retry-after header.
|
||||
*/
|
||||
export default function(
|
||||
axios: AxiosInstance,
|
||||
options?: AxiosRetryAfterOptions
|
||||
): (error: AxiosError) => Promise<void>;
|
||||
wait?: (error: AxiosError) => Promise<void>
|
||||
|
||||
/**
|
||||
* Configuration options for axios-retry-after.
|
||||
* Function to retry the original request.
|
||||
* @param axios The Axios instance used for retrying the request.
|
||||
* @param error The Axios error to retry.
|
||||
*/
|
||||
export interface AxiosRetryAfterOptions {
|
||||
/**
|
||||
* Function to determine if an error response is retryable.
|
||||
* @param error The Axios error to evaluate.
|
||||
*/
|
||||
isRetryable?: (error: AxiosError) => boolean;
|
||||
|
||||
/**
|
||||
* Function to wait for a specified amount of time.
|
||||
* @param error The Axios error that contains retry-after header.
|
||||
*/
|
||||
wait?: (error: AxiosError) => Promise<void>;
|
||||
|
||||
/**
|
||||
* Function to retry the original request.
|
||||
* @param axios The Axios instance used for retrying the request.
|
||||
* @param error The Axios error to retry.
|
||||
*/
|
||||
retry?: (axios: AxiosInstance, error: AxiosError) => Promise<any>;
|
||||
}
|
||||
retry?: (axios: AxiosInstance, error: AxiosError) => Promise<any>
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,5 @@
|
|||
"outDir": "./build"
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": [
|
||||
"node_modules"
|
||||
]
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue