format

proxy/.prettierignore

@@ -0,0 +1,2 @@
+**/*.md
+pnpm-lock.yaml

proxy/.prettierrc.yaml

@@ -0,0 +1,6 @@
+proseWrap: always
+semi: false
+singleQuote: true
+printWidth: 80
+trailingComma: none
+htmlWhitespaceSensitivity: ignore

proxy/package.json

@@ -4,6 +4,7 @@
   "scripts": {
     "clean": "rm -rf build",
     "prebuild": "npm run clean",
+    "format": "prettier -w --cache --check .",
     "build": "tsc --build"
   },
   "author": "",
@@ -22,6 +23,7 @@
   },
   "devDependencies": {
     "@types/node": "^20.12.12",
-    "dotenv": "^16.4.4"
+    "dotenv": "^16.4.4",
+    "prettier": "^3.2.5"
   }
 }

proxy/pnpm-lock.yaml

@@ -45,6 +45,9 @@ importers:
       dotenv:
         specifier: ^16.4.4
         version: 16.4.4
+      prettier:
+        specifier: ^3.2.5
+        version: 3.2.5
 
 packages:
 
@@ -292,6 +295,11 @@ packages:
     resolution: {integrity: sha512-Mz/gKiRyuXu4HnpHgi1YWdHQCoWMufapzooisvFn78zl4dZciAxS+YeRkUxXl1ee/SzU80YCz1zpECCh4oC6Aw==}
     hasBin: true
 
+  prettier@3.2.5:
+    resolution: {integrity: sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==}
+    engines: {node: '>=14'}
+    hasBin: true
+
   process-warning@2.3.2:
     resolution: {integrity: sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==}
 
@@ -668,6 +676,8 @@ snapshots:
       sonic-boom: 3.8.0
       thread-stream: 2.4.1
 
+  prettier@3.2.5: {}
+
   process-warning@2.3.2: {}
 
   process-warning@3.0.0: {}

proxy/src/index.ts

@@ -1,89 +1,116 @@
-
 import fastify, {
-    FastifyInstance,
+  FastifyInstance,
-    FastifyListenOptions,
+  FastifyListenOptions,
-    FastifyReply,
+  FastifyReply,
-    FastifyRequest,
+  FastifyRequest
-} from "fastify"
+} from 'fastify'
-import { PinoLoggerOptions } from "fastify/types/logger"
+import { PinoLoggerOptions } from 'fastify/types/logger'
-import { Proxy } from "./proxy"
+import { Proxy } from './proxy'
-import { Logger } from "pino"
+import { Logger } from 'pino'
 import 'dotenv/config'
 
 const host = process.env.HOST
-const port = parseInt(process.env.PORT ?? "8080", 10)
+const port = parseInt(process.env.PORT ?? '8080', 10)
 const baseUrl = process.env.NITTER_BASE_URL
-const concurrency = parseInt(process.env.CONCURRENCY ?? "1", 10)
+const concurrency = parseInt(process.env.CONCURRENCY ?? '1', 10)
-const retryAfterMillis = process.env.RETRY_AFTER_MILLIS ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) : null
+const retryAfterMillis = process.env.RETRY_AFTER_MILLIS
-const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? "100000", 10)
+  ? parseInt(process.env.RETRY_AFTER_MILLIS, 10)
-const logLevel = process.env.LOG_LEVEL ?? "debug"
+  : null
+const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? '100000', 10)
+const logLevel = process.env.LOG_LEVEL ?? 'debug'
 
 const server = fastify({
-    logger: {
+  logger: {
-        name: "app",
+    name: 'app',
-        level: logLevel,
+    level: logLevel,
-        ...( logLevel == "trace" ? { transport: { target: 'pino-pretty' } } : {})
+    ...(logLevel == 'trace' ? { transport: { target: 'pino-pretty' } } : {})
-    } as PinoLoggerOptions
+  } as PinoLoggerOptions
 })
 
 const log = server.log as Logger
-const proxy = new Proxy(log, baseUrl, concurrency, retryAfterMillis, maxCacheSize)
+const proxy = new Proxy(
+  log,
+  baseUrl,
+  concurrency,
+  retryAfterMillis,
+  maxCacheSize
+)
 
 async function main() {
+  server.register(
+    (fastify: FastifyInstance, opts, done) => {
+      fastify.get(
+        `/user/:username`,
+        {},
+        async (request: FastifyRequest, reply: FastifyReply) => {
+          log.debug(
+            {
+              headers: request.headers,
+              reqId: request.id,
+              params: request.params
+            },
+            'incoming request /user/:username'
+          )
+          const { username } = request.params as any
+          const { status, data } = await proxy.getUser(username, {
+            reqId: request.id
+          })
+          reply.status(status).send(data)
+        }
+      )
 
-    server.register((fastify: FastifyInstance, opts, done) => {
+      fastify.get(
+        `/user/:userId/tweets`,
+        {},
+        async (request: FastifyRequest, reply: FastifyReply) => {
+          const { userId } = request.params as any
+          const { cursor } = request.query as any
+          const { status, data } = await proxy.getUserTweets(userId, cursor, {
+            reqId: request.id
+          })
+          reply.status(status).send(data)
+        }
+      )
 
-        fastify.get(`/user/:username`, {},
+      fastify.get(
-            async (request: FastifyRequest, reply: FastifyReply) => {
+        `/tweet/:id`,
-                log.debug({
+        {},
-                    headers: request.headers,
+        async (request: FastifyRequest, reply: FastifyReply) => {
-                    reqId: request.id,
+          const { id } = request.params as any
-                    params: request.params }, 'incoming request /user/:username')
+          const { status, data } = await proxy.getTweetById(id, {
-                const { username } = request.params as any
+            reqId: request.id
-                const { status, data } = await proxy.getUser(username, { reqId: request.id })
+          })
-                reply.status(status).send(data)
+          reply.status(status).send(data)
-            });
+        }
+      )
 
-        fastify.get(`/user/:userId/tweets`, {},
+      done()
-            async (request: FastifyRequest, reply: FastifyReply) => {
+    },
-                const { userId } = request.params as any
+    { prefix: '/api' }
-                const { cursor } = request.query as any
+  )
-                const { status, data } = await proxy.getUserTweets(userId, cursor, { reqId: request.id })
-                reply.status(status).send(data)
-            });
 
-        fastify.get(`/tweet/:id`, {},
+  server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => {
-            async (request: FastifyRequest, reply: FastifyReply) => {
+    reply.status(404).send({ message: `Method not found` })
-                const { id } = request.params as any
+  })
-                const { status, data } = await proxy.getTweetById(id, { reqId: request.id })
-                reply.status(status).send(data)
-            });
 
-        done()
+  server.setErrorHandler(
+    (err: Error, request: FastifyRequest, reply: FastifyReply) => {
+      const { log } = request
+      log.error(err)
+      // Send error response
+      reply.status(500).send({ message: `Internal server error` })
+    }
+  )
 
-    }, { prefix: '/api' })
+  // await server.register(import('@fastify/rate-limit'), {
+  //     max: 100,
+  //     timeWindow: '1 minute'
+  // })
 
-    server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => {
+  await server.listen({ port, host } as FastifyListenOptions)
-        reply.status(404)
-            .send({ message: `Method not found` })
-    })
-
-    server.setErrorHandler((err: Error, request: FastifyRequest, reply: FastifyReply) => {
-        const { log } = request
-        log.error(err)
-        // Send error response
-        reply.status(500).send({ message: `Internal server error` })
-    })
-
-    // await server.register(import('@fastify/rate-limit'), {
-    //     max: 100,
-    //     timeWindow: '1 minute'
-    // })
-
-    await server.listen({ port, host } as FastifyListenOptions);
 }
 
-main().catch(err => {
+main().catch((err) => {
-    log.fatal(err)
+  log.fatal(err)
-    process.exit(1)
+  process.exit(1)
 })

proxy/src/proxy.ts

@@ -1,209 +1,226 @@
 // noinspection TypeScriptUnresolvedReference
 
-import axios from "axios"
+import axios from 'axios'
-import { AxiosInstance, AxiosRequestConfig } from "axios"
+import { AxiosInstance, AxiosRequestConfig } from 'axios'
-import fastq from "fastq"
+import fastq from 'fastq'
-import { Logger } from "pino"
+import { Logger } from 'pino'
-import retry from "axios-retry-after"
+import retry from 'axios-retry-after'
 import { LRUCache } from 'lru-cache'
 
 const GET_USER_POSITIVE_TTL_MS = process.env.GET_USER_POSITIVE_TTL
-    ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000
+  ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000
-    : 30 * 24 * 3600 * 1000
+  : 30 * 24 * 3600 * 1000
 const GET_USER_NEGATIVE_TTL_MS = process.env.GET_USER_NEGATIVE_TTL
-    ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000
+  ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000
-    : 3600 * 1000
+  : 3600 * 1000
 const GET_TWEETS_POSITIVE_TTL_MS = process.env.GET_TWEETS_POSITIVE_TTL
-    ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000
+  ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000
-    : 60 * 1000
+  : 60 * 1000
 const GET_TWEETS_NEGATIVE_TTL_MS = process.env.GET_TWEETS_NEGATIVE_TTL
-    ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000
+  ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000
-    : 60 * 1000
+  : 60 * 1000
 const GET_TWEET_POSITIVE_TTL_MS = process.env.GET_TWEET_POSITIVE_TTL
-    ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000
+  ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000
-    : 60 * 1000
+  : 60 * 1000
 const GET_TWEET_NEGATIVE_TTL_MS = process.env.GET_TWEET_NEGATIVE_TTL
-    ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000
+  ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000
-    : 60 * 1000
+  : 60 * 1000
 
 export interface Job {
-    reqId: string
+  reqId: string
-    url: string
+  url: string
-    params?: Record<string, any>
+  params?: Record<string, any>
 }
 
 export interface JobResponse {
-    status: number,
+  status: number
-    data: any
+  data: any
 }
 
 export class Proxy {
+  private readonly cache: LRUCache<string, JobResponse>
+  private readonly client: AxiosInstance
+  private readonly queue: fastq.queueAsPromised<Job, JobResponse>
+  private counter: { requests: number }
+  private timeWindowMillis = 15 * 60 * 1000
+  private maxRequestsPerAccount = 15 * 60
 
-    private readonly cache: LRUCache<string, JobResponse>
+  constructor(
-    private readonly client: AxiosInstance
+    private log: Logger,
-    private readonly queue: fastq.queueAsPromised<Job, JobResponse>
+    private baseUrl: string,
-    private counter: { requests: number }
+    private concurrency: number,
-    private timeWindowMillis = 15 * 60 * 1000
+    retryAfterMillis: number,
-    private maxRequestsPerAccount = 15 * 60
+    maxCacheSize: number
-
+  ) {
-    constructor(
+    this.cache = new LRUCache({ max: maxCacheSize })
-        private log: Logger,
+    this.queue = fastq.promise(this, this.sendRequest, concurrency)
-        private baseUrl: string,
+    this.client = axios.create()
-        private concurrency: number,
+    this.counter = {
-        retryAfterMillis: number,
+      requests: 0
-        maxCacheSize: number
-    ) {
-        this.cache =  new LRUCache({ max: maxCacheSize })
-        this.queue = fastq.promise(this, this.sendRequest, concurrency)
-        this.client = axios.create()
-        this.counter = {
-            requests: 0
-        }
-
-        setInterval(() => {
-            this.counter.requests = 0
-        }, this.timeWindowMillis)
-
-        if ( retryAfterMillis ) {
-            this.client.interceptors.response.use(null, retry(this.client, {
-                // Determine when we should attempt to retry
-                isRetryable (error) {
-                    log.debug({ status: error.response?.status, headers: error.response?.headers }, 'checking retryable')
-                    return (
-                        error.response && error.response.status === 429
-                        // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds
-                        // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60
-                    )
-                },
-                // Customize the wait behavior
-                wait (error) {
-                    log.debug({ status: error.response?.status, headers: error.response?.headers }, 'waiting for retry')
-                    return new Promise(
-                        // Use X-Retry-After rather than Retry-After
-                        // resolve => setTimeout(resolve, error.response.headers['x-retry-after'])
-                        resolve => setTimeout(resolve, retryAfterMillis)
-                    )
-                }
-            }))
-        }
     }
 
-    async getUser(username: string, options?: { reqId?: string }) {
+    setInterval(() => {
-        const key = `usernames:${username}`
+      this.counter.requests = 0
+    }, this.timeWindowMillis)
 
-        if ( this.cache.has(key)) {
+    if (retryAfterMillis) {
-            return this.cache.get(key)
+      this.client.interceptors.response.use(
-        }
+        null,
-
+        retry(this.client, {
-        const result = await this.queue.push({
+          // Determine when we should attempt to retry
-            url: `/api/user/${ username }`,
+          isRetryable(error) {
-            reqId: options?.reqId
+            log.debug(
+              {
+                status: error.response?.status,
+                headers: error.response?.headers
+              },
+              'checking retryable'
+            )
+            return (
+              error.response && error.response.status === 429
+              // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds
+              // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60
+            )
+          },
+          // Customize the wait behavior
+          wait(error) {
+            log.debug(
+              {
+                status: error.response?.status,
+                headers: error.response?.headers
+              },
+              'waiting for retry'
+            )
+            return new Promise(
+              // Use X-Retry-After rather than Retry-After
+              // resolve => setTimeout(resolve, error.response.headers['x-retry-after'])
+              (resolve) => setTimeout(resolve, retryAfterMillis)
+            )
+          }
         })
+      )
+    }
+  }
 
-        if ( result.status === 200 ) {
+  async getUser(username: string, options?: { reqId?: string }) {
-            this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS })
+    const key = `usernames:${username}`
-        }
-        if ( result.status === 404 ) {
-            this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS })
-        }
 
-        return result
+    if (this.cache.has(key)) {
+      return this.cache.get(key)
     }
 
-    async getUserTweets(userId: string, cursor?: string, options?: { reqId?: string }) {
+    const result = await this.queue.push({
-        const key = `users:${userId}:tweets:${cursor ?? 'last'}`
+      url: `/api/user/${username}`,
+      reqId: options?.reqId
+    })
 
-        if ( this.cache.has(key) ) {
+    if (result.status === 200) {
-            return this.cache.get(key)
+      this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS })
-        }
+    }
-
+    if (result.status === 404) {
-        const result = await this.queue.push({
+      this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS })
-            url: `/api/user/${ userId }/tweets`,
-            params: { cursor },
-            reqId: options?.reqId
-        })
-
-        if ( result.status === 200 ) {
-            this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS })
-        }
-        if ( result.status === 404 ) {
-            this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS })
-        }
-
-        return result
     }
 
-    async getTweetById(tweetId: string, options?: { reqId?: string }) {
+    return result
-        const key = `tweets:${tweetId}`
+  }
 
-        if ( this.cache.has(key) ) {
+  async getUserTweets(
-            return this.cache.get(key)
+    userId: string,
-        }
+    cursor?: string,
+    options?: { reqId?: string }
+  ) {
+    const key = `users:${userId}:tweets:${cursor ?? 'last'}`
 
-        const result = await this.queue.push({
+    if (this.cache.has(key)) {
-            url: `/api/tweet/${ tweetId }`,
+      return this.cache.get(key)
-            reqId: options?.reqId
-        })
-
-        if ( result.status === 200 ) {
-            this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS })
-        }
-        if ( result.status === 404 ) {
-            this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS })
-        }
-
-        return result
     }
 
-    private async sendRequest(job: Job): Promise<any> {
+    const result = await this.queue.push({
+      url: `/api/user/${userId}/tweets`,
+      params: { cursor },
+      reqId: options?.reqId
+    })
 
-        const { reqId, url, params } = job
+    if (result.status === 200) {
-
+      this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS })
-        if ( this.counter.requests > this.concurrency * this.maxRequestsPerAccount ) {
-            return {
-                status: 429
-            }
-        }
-
-        let config = {
-            url,
-            method: "get",
-            baseURL: this.baseUrl,
-            params,
-        } as AxiosRequestConfig
-
-        this.log.trace({ config, reqId: reqId }, 'sending request to nitter')
-
-        try {
-            const response = await this.client.request(config)
-
-            this.log.trace({
-                status: response.status,
-                data: response.data,
-                reqId: reqId
-            }, 'nitter response')
-
-            return {
-                status: response.status,
-                data: response.data,
-            } as JobResponse
-
-        } catch(err) {
-
-            this.log.warn({ err, reqId }, 'nitter error')
-
-            if ( err.name === "AxiosError" ) {
-
-                this.counter.requests = Number.MAX_SAFE_INTEGER
-
-                return {
-                    status: 429
-                } as JobResponse
-            }
-
-            return {
-                status: 500
-            }
-        }
     }
+    if (result.status === 404) {
+      this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS })
+    }
+
+    return result
+  }
+
+  async getTweetById(tweetId: string, options?: { reqId?: string }) {
+    const key = `tweets:${tweetId}`
+
+    if (this.cache.has(key)) {
+      return this.cache.get(key)
+    }
+
+    const result = await this.queue.push({
+      url: `/api/tweet/${tweetId}`,
+      reqId: options?.reqId
+    })
+
+    if (result.status === 200) {
+      this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS })
+    }
+    if (result.status === 404) {
+      this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS })
+    }
+
+    return result
+  }
+
+  private async sendRequest(job: Job): Promise<any> {
+    const { reqId, url, params } = job
+
+    if (this.counter.requests > this.concurrency * this.maxRequestsPerAccount) {
+      return {
+        status: 429
+      }
+    }
+
+    let config = {
+      url,
+      method: 'get',
+      baseURL: this.baseUrl,
+      params
+    } as AxiosRequestConfig
+
+    this.log.trace({ config, reqId: reqId }, 'sending request to nitter')
+
+    try {
+      const response = await this.client.request(config)
+
+      this.log.trace(
+        {
+          status: response.status,
+          data: response.data,
+          reqId: reqId
+        },
+        'nitter response'
+      )
+
+      return {
+        status: response.status,
+        data: response.data
+      } as JobResponse
+    } catch (err) {
+      this.log.warn({ err, reqId }, 'nitter error')
+
+      if (err.name === 'AxiosError') {
+        this.counter.requests = Number.MAX_SAFE_INTEGER
+
+        return {
+          status: 429
+        } as JobResponse
+      }
+
+      return {
+        status: 500
+      }
+    }
+  }
 }

proxy/src/types.d.ts

@@ -1,39 +1,37 @@
 declare module 'axios-retry-after' {
+  import { AxiosError, AxiosInstance } from 'axios'
 
-    import { AxiosError, AxiosInstance } from "axios";
+  /**
+   * Function to enhance Axios instance with retry-after functionality.
+   * @param axios Axios instance to be enhanced.
+   * @param options Configuration options for retry behavior.
+   */
+  export default function (
+    axios: AxiosInstance,
+    options?: AxiosRetryAfterOptions
+  ): (error: AxiosError) => Promise<void>
+
+  /**
+   * Configuration options for axios-retry-after.
+   */
+  export interface AxiosRetryAfterOptions {
+    /**
+     * Function to determine if an error response is retryable.
+     * @param error The Axios error to evaluate.
+     */
+    isRetryable?: (error: AxiosError) => boolean
 
     /**
-     * Function to enhance Axios instance with retry-after functionality.
+     * Function to wait for a specified amount of time.
-     * @param axios Axios instance to be enhanced.
+     * @param error The Axios error that contains retry-after header.
-     * @param options Configuration options for retry behavior.
      */
-    export default function(
+    wait?: (error: AxiosError) => Promise<void>
-        axios: AxiosInstance,
-        options?: AxiosRetryAfterOptions
-    ): (error: AxiosError) => Promise<void>;
 
     /**
-     * Configuration options for axios-retry-after.
+     * Function to retry the original request.
+     * @param axios The Axios instance used for retrying the request.
+     * @param error The Axios error to retry.
      */
-    export interface AxiosRetryAfterOptions {
+    retry?: (axios: AxiosInstance, error: AxiosError) => Promise<any>
-        /**
+  }
-         * Function to determine if an error response is retryable.
-         * @param error The Axios error to evaluate.
-         */
-        isRetryable?: (error: AxiosError) => boolean;
-
-        /**
-         * Function to wait for a specified amount of time.
-         * @param error The Axios error that contains retry-after header.
-         */
-        wait?: (error: AxiosError) => Promise<void>;
-
-        /**
-         * Function to retry the original request.
-         * @param axios The Axios instance used for retrying the request.
-         * @param error The Axios error to retry.
-         */
-        retry?: (axios: AxiosInstance, error: AxiosError) => Promise<any>;
-    }
 }
-

proxy/tsconfig.json

@@ -12,7 +12,5 @@
     "outDir": "./build"
   },
   "include": ["src/**/*"],
-  "exclude": [
+  "exclude": ["node_modules"]
-    "node_modules"
-  ]
 }