From a451755d2bf36b6923b1f3a9ce6fb9fa7b265931 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sun, 19 May 2024 07:05:01 +0000 Subject: [PATCH] format --- proxy/.prettierignore | 2 + proxy/.prettierrc.yaml | 6 + proxy/package.json | 4 +- proxy/pnpm-lock.yaml | 10 ++ proxy/src/index.ts | 161 ++++++++++-------- proxy/src/proxy.ts | 361 +++++++++++++++++++++-------------------- proxy/src/types.d.ts | 58 ++++--- proxy/tsconfig.json | 4 +- 8 files changed, 333 insertions(+), 273 deletions(-) create mode 100644 proxy/.prettierignore create mode 100644 proxy/.prettierrc.yaml diff --git a/proxy/.prettierignore b/proxy/.prettierignore new file mode 100644 index 0000000..18f1a04 --- /dev/null +++ b/proxy/.prettierignore @@ -0,0 +1,2 @@ +**/*.md +pnpm-lock.yaml diff --git a/proxy/.prettierrc.yaml b/proxy/.prettierrc.yaml new file mode 100644 index 0000000..88e0a15 --- /dev/null +++ b/proxy/.prettierrc.yaml @@ -0,0 +1,6 @@ +proseWrap: always +semi: false +singleQuote: true +printWidth: 80 +trailingComma: none +htmlWhitespaceSensitivity: ignore diff --git a/proxy/package.json b/proxy/package.json index 0092699..0aa594b 100644 --- a/proxy/package.json +++ b/proxy/package.json @@ -4,6 +4,7 @@ "scripts": { "clean": "rm -rf build", "prebuild": "npm run clean", + "format": "prettier -w --cache --check .", "build": "tsc --build" }, "author": "", @@ -22,6 +23,7 @@ }, "devDependencies": { "@types/node": "^20.12.12", - "dotenv": "^16.4.4" + "dotenv": "^16.4.4", + "prettier": "^3.2.5" } } diff --git a/proxy/pnpm-lock.yaml b/proxy/pnpm-lock.yaml index 2d53adc..1876821 100644 --- a/proxy/pnpm-lock.yaml +++ b/proxy/pnpm-lock.yaml @@ -45,6 +45,9 @@ importers: dotenv: specifier: ^16.4.4 version: 16.4.4 + prettier: + specifier: ^3.2.5 + version: 3.2.5 packages: @@ -292,6 +295,11 @@ packages: resolution: {integrity: sha512-Mz/gKiRyuXu4HnpHgi1YWdHQCoWMufapzooisvFn78zl4dZciAxS+YeRkUxXl1ee/SzU80YCz1zpECCh4oC6Aw==} hasBin: true + prettier@3.2.5: + resolution: {integrity: sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==} + engines: {node: '>=14'} + hasBin: true + process-warning@2.3.2: resolution: {integrity: sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==} @@ -668,6 +676,8 @@ snapshots: sonic-boom: 3.8.0 thread-stream: 2.4.1 + prettier@3.2.5: {} + process-warning@2.3.2: {} process-warning@3.0.0: {} diff --git a/proxy/src/index.ts b/proxy/src/index.ts index 020c5ef..b7c121d 100644 --- a/proxy/src/index.ts +++ b/proxy/src/index.ts @@ -1,89 +1,116 @@ - import fastify, { - FastifyInstance, - FastifyListenOptions, - FastifyReply, - FastifyRequest, -} from "fastify" -import { PinoLoggerOptions } from "fastify/types/logger" -import { Proxy } from "./proxy" -import { Logger } from "pino" + FastifyInstance, + FastifyListenOptions, + FastifyReply, + FastifyRequest +} from 'fastify' +import { PinoLoggerOptions } from 'fastify/types/logger' +import { Proxy } from './proxy' +import { Logger } from 'pino' import 'dotenv/config' const host = process.env.HOST -const port = parseInt(process.env.PORT ?? "8080", 10) +const port = parseInt(process.env.PORT ?? '8080', 10) const baseUrl = process.env.NITTER_BASE_URL -const concurrency = parseInt(process.env.CONCURRENCY ?? "1", 10) -const retryAfterMillis = process.env.RETRY_AFTER_MILLIS ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) : null -const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? "100000", 10) -const logLevel = process.env.LOG_LEVEL ?? "debug" +const concurrency = parseInt(process.env.CONCURRENCY ?? '1', 10) +const retryAfterMillis = process.env.RETRY_AFTER_MILLIS + ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) + : null +const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? '100000', 10) +const logLevel = process.env.LOG_LEVEL ?? 'debug' const server = fastify({ - logger: { - name: "app", - level: logLevel, - ...( logLevel == "trace" ? { transport: { target: 'pino-pretty' } } : {}) - } as PinoLoggerOptions + logger: { + name: 'app', + level: logLevel, + ...(logLevel == 'trace' ? { transport: { target: 'pino-pretty' } } : {}) + } as PinoLoggerOptions }) const log = server.log as Logger -const proxy = new Proxy(log, baseUrl, concurrency, retryAfterMillis, maxCacheSize) +const proxy = new Proxy( + log, + baseUrl, + concurrency, + retryAfterMillis, + maxCacheSize +) async function main() { + server.register( + (fastify: FastifyInstance, opts, done) => { + fastify.get( + `/user/:username`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + log.debug( + { + headers: request.headers, + reqId: request.id, + params: request.params + }, + 'incoming request /user/:username' + ) + const { username } = request.params as any + const { status, data } = await proxy.getUser(username, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) - server.register((fastify: FastifyInstance, opts, done) => { + fastify.get( + `/user/:userId/tweets`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { userId } = request.params as any + const { cursor } = request.query as any + const { status, data } = await proxy.getUserTweets(userId, cursor, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) - fastify.get(`/user/:username`, {}, - async (request: FastifyRequest, reply: FastifyReply) => { - log.debug({ - headers: request.headers, - reqId: request.id, - params: request.params }, 'incoming request /user/:username') - const { username } = request.params as any - const { status, data } = await proxy.getUser(username, { reqId: request.id }) - reply.status(status).send(data) - }); + fastify.get( + `/tweet/:id`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as any + const { status, data } = await proxy.getTweetById(id, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) - fastify.get(`/user/:userId/tweets`, {}, - async (request: FastifyRequest, reply: FastifyReply) => { - const { userId } = request.params as any - const { cursor } = request.query as any - const { status, data } = await proxy.getUserTweets(userId, cursor, { reqId: request.id }) - reply.status(status).send(data) - }); + done() + }, + { prefix: '/api' } + ) - fastify.get(`/tweet/:id`, {}, - async (request: FastifyRequest, reply: FastifyReply) => { - const { id } = request.params as any - const { status, data } = await proxy.getTweetById(id, { reqId: request.id }) - reply.status(status).send(data) - }); + server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => { + reply.status(404).send({ message: `Method not found` }) + }) - done() + server.setErrorHandler( + (err: Error, request: FastifyRequest, reply: FastifyReply) => { + const { log } = request + log.error(err) + // Send error response + reply.status(500).send({ message: `Internal server error` }) + } + ) - }, { prefix: '/api' }) + // await server.register(import('@fastify/rate-limit'), { + // max: 100, + // timeWindow: '1 minute' + // }) - server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => { - reply.status(404) - .send({ message: `Method not found` }) - }) - - server.setErrorHandler((err: Error, request: FastifyRequest, reply: FastifyReply) => { - const { log } = request - log.error(err) - // Send error response - reply.status(500).send({ message: `Internal server error` }) - }) - - // await server.register(import('@fastify/rate-limit'), { - // max: 100, - // timeWindow: '1 minute' - // }) - - await server.listen({ port, host } as FastifyListenOptions); + await server.listen({ port, host } as FastifyListenOptions) } -main().catch(err => { - log.fatal(err) - process.exit(1) +main().catch((err) => { + log.fatal(err) + process.exit(1) }) diff --git a/proxy/src/proxy.ts b/proxy/src/proxy.ts index 872f481..a9fb9e5 100644 --- a/proxy/src/proxy.ts +++ b/proxy/src/proxy.ts @@ -1,209 +1,226 @@ // noinspection TypeScriptUnresolvedReference -import axios from "axios" -import { AxiosInstance, AxiosRequestConfig } from "axios" -import fastq from "fastq" -import { Logger } from "pino" -import retry from "axios-retry-after" +import axios from 'axios' +import { AxiosInstance, AxiosRequestConfig } from 'axios' +import fastq from 'fastq' +import { Logger } from 'pino' +import retry from 'axios-retry-after' import { LRUCache } from 'lru-cache' const GET_USER_POSITIVE_TTL_MS = process.env.GET_USER_POSITIVE_TTL - ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000 - : 30 * 24 * 3600 * 1000 + ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000 + : 30 * 24 * 3600 * 1000 const GET_USER_NEGATIVE_TTL_MS = process.env.GET_USER_NEGATIVE_TTL - ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000 - : 3600 * 1000 + ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000 + : 3600 * 1000 const GET_TWEETS_POSITIVE_TTL_MS = process.env.GET_TWEETS_POSITIVE_TTL - ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 const GET_TWEETS_NEGATIVE_TTL_MS = process.env.GET_TWEETS_NEGATIVE_TTL - ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 const GET_TWEET_POSITIVE_TTL_MS = process.env.GET_TWEET_POSITIVE_TTL - ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 const GET_TWEET_NEGATIVE_TTL_MS = process.env.GET_TWEET_NEGATIVE_TTL - ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 export interface Job { - reqId: string - url: string - params?: Record + reqId: string + url: string + params?: Record } export interface JobResponse { - status: number, - data: any + status: number + data: any } export class Proxy { + private readonly cache: LRUCache + private readonly client: AxiosInstance + private readonly queue: fastq.queueAsPromised + private counter: { requests: number } + private timeWindowMillis = 15 * 60 * 1000 + private maxRequestsPerAccount = 15 * 60 - private readonly cache: LRUCache - private readonly client: AxiosInstance - private readonly queue: fastq.queueAsPromised - private counter: { requests: number } - private timeWindowMillis = 15 * 60 * 1000 - private maxRequestsPerAccount = 15 * 60 - - constructor( - private log: Logger, - private baseUrl: string, - private concurrency: number, - retryAfterMillis: number, - maxCacheSize: number - ) { - this.cache = new LRUCache({ max: maxCacheSize }) - this.queue = fastq.promise(this, this.sendRequest, concurrency) - this.client = axios.create() - this.counter = { - requests: 0 - } - - setInterval(() => { - this.counter.requests = 0 - }, this.timeWindowMillis) - - if ( retryAfterMillis ) { - this.client.interceptors.response.use(null, retry(this.client, { - // Determine when we should attempt to retry - isRetryable (error) { - log.debug({ status: error.response?.status, headers: error.response?.headers }, 'checking retryable') - return ( - error.response && error.response.status === 429 - // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds - // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60 - ) - }, - // Customize the wait behavior - wait (error) { - log.debug({ status: error.response?.status, headers: error.response?.headers }, 'waiting for retry') - return new Promise( - // Use X-Retry-After rather than Retry-After - // resolve => setTimeout(resolve, error.response.headers['x-retry-after']) - resolve => setTimeout(resolve, retryAfterMillis) - ) - } - })) - } + constructor( + private log: Logger, + private baseUrl: string, + private concurrency: number, + retryAfterMillis: number, + maxCacheSize: number + ) { + this.cache = new LRUCache({ max: maxCacheSize }) + this.queue = fastq.promise(this, this.sendRequest, concurrency) + this.client = axios.create() + this.counter = { + requests: 0 } - async getUser(username: string, options?: { reqId?: string }) { - const key = `usernames:${username}` + setInterval(() => { + this.counter.requests = 0 + }, this.timeWindowMillis) - if ( this.cache.has(key)) { - return this.cache.get(key) - } - - const result = await this.queue.push({ - url: `/api/user/${ username }`, - reqId: options?.reqId + if (retryAfterMillis) { + this.client.interceptors.response.use( + null, + retry(this.client, { + // Determine when we should attempt to retry + isRetryable(error) { + log.debug( + { + status: error.response?.status, + headers: error.response?.headers + }, + 'checking retryable' + ) + return ( + error.response && error.response.status === 429 + // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds + // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60 + ) + }, + // Customize the wait behavior + wait(error) { + log.debug( + { + status: error.response?.status, + headers: error.response?.headers + }, + 'waiting for retry' + ) + return new Promise( + // Use X-Retry-After rather than Retry-After + // resolve => setTimeout(resolve, error.response.headers['x-retry-after']) + (resolve) => setTimeout(resolve, retryAfterMillis) + ) + } }) + ) + } + } - if ( result.status === 200 ) { - this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS }) - } - if ( result.status === 404 ) { - this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS }) - } + async getUser(username: string, options?: { reqId?: string }) { + const key = `usernames:${username}` - return result + if (this.cache.has(key)) { + return this.cache.get(key) } - async getUserTweets(userId: string, cursor?: string, options?: { reqId?: string }) { - const key = `users:${userId}:tweets:${cursor ?? 'last'}` + const result = await this.queue.push({ + url: `/api/user/${username}`, + reqId: options?.reqId + }) - if ( this.cache.has(key) ) { - return this.cache.get(key) - } - - const result = await this.queue.push({ - url: `/api/user/${ userId }/tweets`, - params: { cursor }, - reqId: options?.reqId - }) - - if ( result.status === 200 ) { - this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS }) - } - if ( result.status === 404 ) { - this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS }) - } - - return result + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS }) + } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS }) } - async getTweetById(tweetId: string, options?: { reqId?: string }) { - const key = `tweets:${tweetId}` + return result + } - if ( this.cache.has(key) ) { - return this.cache.get(key) - } + async getUserTweets( + userId: string, + cursor?: string, + options?: { reqId?: string } + ) { + const key = `users:${userId}:tweets:${cursor ?? 'last'}` - const result = await this.queue.push({ - url: `/api/tweet/${ tweetId }`, - reqId: options?.reqId - }) - - if ( result.status === 200 ) { - this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS }) - } - if ( result.status === 404 ) { - this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS }) - } - - return result + if (this.cache.has(key)) { + return this.cache.get(key) } - private async sendRequest(job: Job): Promise { + const result = await this.queue.push({ + url: `/api/user/${userId}/tweets`, + params: { cursor }, + reqId: options?.reqId + }) - const { reqId, url, params } = job - - if ( this.counter.requests > this.concurrency * this.maxRequestsPerAccount ) { - return { - status: 429 - } - } - - let config = { - url, - method: "get", - baseURL: this.baseUrl, - params, - } as AxiosRequestConfig - - this.log.trace({ config, reqId: reqId }, 'sending request to nitter') - - try { - const response = await this.client.request(config) - - this.log.trace({ - status: response.status, - data: response.data, - reqId: reqId - }, 'nitter response') - - return { - status: response.status, - data: response.data, - } as JobResponse - - } catch(err) { - - this.log.warn({ err, reqId }, 'nitter error') - - if ( err.name === "AxiosError" ) { - - this.counter.requests = Number.MAX_SAFE_INTEGER - - return { - status: 429 - } as JobResponse - } - - return { - status: 500 - } - } + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS }) } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS }) + } + + return result + } + + async getTweetById(tweetId: string, options?: { reqId?: string }) { + const key = `tweets:${tweetId}` + + if (this.cache.has(key)) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/tweet/${tweetId}`, + reqId: options?.reqId + }) + + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS }) + } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS }) + } + + return result + } + + private async sendRequest(job: Job): Promise { + const { reqId, url, params } = job + + if (this.counter.requests > this.concurrency * this.maxRequestsPerAccount) { + return { + status: 429 + } + } + + let config = { + url, + method: 'get', + baseURL: this.baseUrl, + params + } as AxiosRequestConfig + + this.log.trace({ config, reqId: reqId }, 'sending request to nitter') + + try { + const response = await this.client.request(config) + + this.log.trace( + { + status: response.status, + data: response.data, + reqId: reqId + }, + 'nitter response' + ) + + return { + status: response.status, + data: response.data + } as JobResponse + } catch (err) { + this.log.warn({ err, reqId }, 'nitter error') + + if (err.name === 'AxiosError') { + this.counter.requests = Number.MAX_SAFE_INTEGER + + return { + status: 429 + } as JobResponse + } + + return { + status: 500 + } + } + } } diff --git a/proxy/src/types.d.ts b/proxy/src/types.d.ts index a0c18fa..cd9cf13 100644 --- a/proxy/src/types.d.ts +++ b/proxy/src/types.d.ts @@ -1,39 +1,37 @@ declare module 'axios-retry-after' { + import { AxiosError, AxiosInstance } from 'axios' - import { AxiosError, AxiosInstance } from "axios"; + /** + * Function to enhance Axios instance with retry-after functionality. + * @param axios Axios instance to be enhanced. + * @param options Configuration options for retry behavior. + */ + export default function ( + axios: AxiosInstance, + options?: AxiosRetryAfterOptions + ): (error: AxiosError) => Promise + + /** + * Configuration options for axios-retry-after. + */ + export interface AxiosRetryAfterOptions { + /** + * Function to determine if an error response is retryable. + * @param error The Axios error to evaluate. + */ + isRetryable?: (error: AxiosError) => boolean /** - * Function to enhance Axios instance with retry-after functionality. - * @param axios Axios instance to be enhanced. - * @param options Configuration options for retry behavior. + * Function to wait for a specified amount of time. + * @param error The Axios error that contains retry-after header. */ - export default function( - axios: AxiosInstance, - options?: AxiosRetryAfterOptions - ): (error: AxiosError) => Promise; + wait?: (error: AxiosError) => Promise /** - * Configuration options for axios-retry-after. + * Function to retry the original request. + * @param axios The Axios instance used for retrying the request. + * @param error The Axios error to retry. */ - export interface AxiosRetryAfterOptions { - /** - * Function to determine if an error response is retryable. - * @param error The Axios error to evaluate. - */ - isRetryable?: (error: AxiosError) => boolean; - - /** - * Function to wait for a specified amount of time. - * @param error The Axios error that contains retry-after header. - */ - wait?: (error: AxiosError) => Promise; - - /** - * Function to retry the original request. - * @param axios The Axios instance used for retrying the request. - * @param error The Axios error to retry. - */ - retry?: (axios: AxiosInstance, error: AxiosError) => Promise; - } + retry?: (axios: AxiosInstance, error: AxiosError) => Promise + } } - diff --git a/proxy/tsconfig.json b/proxy/tsconfig.json index 12c5931..ee6e859 100644 --- a/proxy/tsconfig.json +++ b/proxy/tsconfig.json @@ -12,7 +12,5 @@ "outDir": "./build" }, "include": ["src/**/*"], - "exclude": [ - "node_modules" - ] + "exclude": ["node_modules"] }