From 372b58d1fe912e73b84b75a95dcbec2ef4456b65 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sun, 19 May 2024 06:32:16 +0000 Subject: [PATCH] feat: cache proxy --- .gitignore | 1 + docker-compose.yml | 16 + proxy/.npmrc | 1 + proxy/Dockerfile | 21 ++ proxy/package.json | 27 ++ proxy/pnpm-lock.yaml | 755 +++++++++++++++++++++++++++++++++++++++++++ proxy/src/index.ts | 89 +++++ proxy/src/proxy.ts | 209 ++++++++++++ proxy/src/types.d.ts | 39 +++ proxy/tsconfig.json | 18 ++ 10 files changed, 1176 insertions(+) create mode 100644 proxy/.npmrc create mode 100644 proxy/Dockerfile create mode 100644 proxy/package.json create mode 100644 proxy/pnpm-lock.yaml create mode 100644 proxy/src/index.ts create mode 100644 proxy/src/proxy.ts create mode 100644 proxy/src/types.d.ts create mode 100644 proxy/tsconfig.json diff --git a/.gitignore b/.gitignore index ea520dc..55031d6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ nitter nitter.conf guest_accounts.json* dump.rdb +proxy/node_modules diff --git a/docker-compose.yml b/docker-compose.yml index 207914e..2c4625e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,22 @@ networks: nitter: services: + # proxy: + # hostname: nitter-proxy + # container_name: nitter-proxy + # build: + # context: ./proxy + # dockerfile: Dockerfile + # environment: + # HOST: "0.0.0.0" + # PORT: "8080" + # NITTER_BASE_URL: "http://nitter:8080" + # CONCURRENCY: "1" + # ports: + # - "8002:8080" + # networks: + # - nitter + nitter: build: . container_name: nitter diff --git a/proxy/.npmrc b/proxy/.npmrc new file mode 100644 index 0000000..3bd3b7d --- /dev/null +++ b/proxy/.npmrc @@ -0,0 +1 @@ +shell-emulator=true diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 0000000..000ead8 --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,21 @@ +# this is our first build stage, it will not persist in the final image +FROM node:20.11.0-buster-slim as intermediate + +# installation required packages +RUN apt-get update && apt-get install -y --no-install-recommends ssh git python python3 build-essential +RUN npm install -g pnpm@9.1.1 +RUN mkdir -p /opt +WORKDIR /opt + +COPY tsconfig.json /opt +COPY package.json pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile + +COPY ./src /opt/src + +RUN pnpm run build + +# copy just the package form the previous image +FROM node:20.11.0-buster-slim +COPY --from=intermediate /opt /opt +ENTRYPOINT ["node", "/opt/build/index.js"] diff --git a/proxy/package.json b/proxy/package.json new file mode 100644 index 0000000..0092699 --- /dev/null +++ b/proxy/package.json @@ -0,0 +1,27 @@ +{ + "name": "nitter-proxy", + "version": "1.6.7", + "scripts": { + "clean": "rm -rf build", + "prebuild": "npm run clean", + "build": "tsc --build" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@fastify/rate-limit": "^9.1.0", + "axios": "^1.6.7", + "axios-retry-after": "^2.0.0", + "fastify": "^4.21.0", + "fastq": "^1.17.1", + "lru-cache": "^10.2.0", + "lru-ttl-cache": "^2.4.8", + "pino": "^8.14.2", + "pino-pretty": "^10.2.0", + "typescript": "^5.3.3" + }, + "devDependencies": { + "@types/node": "^20.12.12", + "dotenv": "^16.4.4" + } +} diff --git a/proxy/pnpm-lock.yaml b/proxy/pnpm-lock.yaml new file mode 100644 index 0000000..2d53adc --- /dev/null +++ b/proxy/pnpm-lock.yaml @@ -0,0 +1,755 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@fastify/rate-limit': + specifier: ^9.1.0 + version: 9.1.0 + axios: + specifier: ^1.6.7 + version: 1.6.7 + axios-retry-after: + specifier: ^2.0.0 + version: 2.0.0(axios@1.6.7) + fastify: + specifier: ^4.21.0 + version: 4.26.1 + fastq: + specifier: ^1.17.1 + version: 1.17.1 + lru-cache: + specifier: ^10.2.0 + version: 10.2.0 + lru-ttl-cache: + specifier: ^2.4.8 + version: 2.4.8 + pino: + specifier: ^8.14.2 + version: 8.18.0 + pino-pretty: + specifier: ^10.2.0 + version: 10.3.1 + typescript: + specifier: ^5.3.3 + version: 5.3.3 + devDependencies: + '@types/node': + specifier: ^20.12.12 + version: 20.12.12 + dotenv: + specifier: ^16.4.4 + version: 16.4.4 + +packages: + + '@fastify/ajv-compiler@3.5.0': + resolution: {integrity: sha512-ebbEtlI7dxXF5ziNdr05mOY8NnDiPB1XvAlLHctRt/Rc+C3LCOVW5imUVX+mhvUhnNzmPBHewUkOFgGlCxgdAA==} + + '@fastify/error@3.4.1': + resolution: {integrity: sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ==} + + '@fastify/fast-json-stringify-compiler@4.3.0': + resolution: {integrity: sha512-aZAXGYo6m22Fk1zZzEUKBvut/CIIQe/BapEORnxiD5Qr0kPHqqI69NtEMCme74h+at72sPhbkb4ZrLd1W3KRLA==} + + '@fastify/merge-json-schemas@0.1.1': + resolution: {integrity: sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA==} + + '@fastify/rate-limit@9.1.0': + resolution: {integrity: sha512-h5dZWCkuZXN0PxwqaFQLxeln8/LNwQwH9popywmDCFdKfgpi4b/HoMH1lluy6P+30CG9yzzpSpwTCIPNB9T1JA==} + + '@lukeed/ms@2.0.2': + resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} + engines: {node: '>=8'} + + '@types/node@20.12.12': + resolution: {integrity: sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + abstract-logging@2.0.1: + resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + + ajv-formats@2.1.1: + resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + + archy@1.0.0: + resolution: {integrity: sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + + avvio@8.3.0: + resolution: {integrity: sha512-VBVH0jubFr9LdFASy/vNtm5giTrnbVquWBhT0fyizuNK2rQ7e7ONU2plZQWUNqtE1EmxFEb+kbSkFRkstiaS9Q==} + + axios-retry-after@2.0.0: + resolution: {integrity: sha512-tSB1DEF1bSwXmRNyPcopFsiHAF+PWVq5w2mAK7J0bTltn8x2UnfoSJzTVXPySt/WdrbQL4ES5AXtG9i016+CaA==} + engines: {node: '>= 14'} + peerDependencies: + axios: ^1.0.0 + + axios@1.6.7: + resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + cookie@0.5.0: + resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} + engines: {node: '>= 0.6'} + + dateformat@4.6.3: + resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + + debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dotenv@16.4.4: + resolution: {integrity: sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg==} + engines: {node: '>=12'} + + end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + fast-content-type-parse@1.1.0: + resolution: {integrity: sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==} + + fast-copy@3.0.1: + resolution: {integrity: sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA==} + + fast-decode-uri-component@1.0.1: + resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stringify@5.12.0: + resolution: {integrity: sha512-7Nnm9UPa7SfHRbHVA1kJQrGXCRzB7LMlAAqHXQFkEQqueJm1V8owm0FsE/2Do55/4CcdhwiLQERaKomOnKQkyA==} + + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + + fast-redact@3.3.0: + resolution: {integrity: sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==} + engines: {node: '>=6'} + + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + + fast-uri@2.3.0: + resolution: {integrity: sha512-eel5UKGn369gGEWOqBShmFJWfq/xSJvsgDzgLYC845GneayWvXBf0lJCBn5qTABfewy1ZDPoaR5OZCP+kssfuw==} + + fastify-plugin@4.5.1: + resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==} + + fastify@4.26.1: + resolution: {integrity: sha512-tznA/G55dsxzM5XChBfcvVSloG2ejeeotfPPJSFaWmHyCDVGMpvf3nRNbsCb/JTBF9RmQFBfuujWt3Nphjesng==} + + fastq@1.17.1: + resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} + + find-my-way@8.1.0: + resolution: {integrity: sha512-41QwjCGcVTODUmLLqTMeoHeiozbMXYMAE1CKFiDyi9zVZ2Vjh0yz3MF0WQZoIb+cmzP/XlbFjlF2NtJmvZHznA==} + engines: {node: '>=14'} + + follow-redirects@1.15.5: + resolution: {integrity: sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + help-me@5.0.0: + resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + + json-schema-ref-resolver@1.0.1: + resolution: {integrity: sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + light-my-request@5.11.0: + resolution: {integrity: sha512-qkFCeloXCOMpmEdZ/MV91P8AT4fjwFXWaAFz3lUeStM8RcoM1ks4J/F8r1b3r6y/H4u3ACEJ1T+Gv5bopj7oDA==} + + lru-cache@10.2.0: + resolution: {integrity: sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==} + engines: {node: 14 || >=16.14} + + lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + + lru-ttl-cache@2.4.8: + resolution: {integrity: sha512-24fyw4EHvHKSGXc6DMpcA5Iet9UnPxrXzcXA9bvkTs/9kQL6m9SllRyWAtZdxzUabKWdm38vOHavXwdDx2Zl8g==} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + pino-abstract-transport@1.1.0: + resolution: {integrity: sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==} + + pino-pretty@10.3.1: + resolution: {integrity: sha512-az8JbIYeN/1iLj2t0jR9DV48/LQ3RC6hZPpapKPkb84Q+yTidMCpgWxIT3N0flnBDilyBQ1luWNpOeJptjdp/g==} + hasBin: true + + pino-std-serializers@6.2.2: + resolution: {integrity: sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==} + + pino@8.18.0: + resolution: {integrity: sha512-Mz/gKiRyuXu4HnpHgi1YWdHQCoWMufapzooisvFn78zl4dZciAxS+YeRkUxXl1ee/SzU80YCz1zpECCh4oC6Aw==} + hasBin: true + + process-warning@2.3.2: + resolution: {integrity: sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==} + + process-warning@3.0.0: + resolution: {integrity: sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + pump@3.0.0: + resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + + readable-stream@4.5.2: + resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + ret@0.2.2: + resolution: {integrity: sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ==} + engines: {node: '>=4'} + + reusify@1.0.4: + resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + rfdc@1.3.1: + resolution: {integrity: sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-regex2@2.0.0: + resolution: {integrity: sha512-PaUSFsUaNNuKwkBijoAPHAK6/eM6VirvyPWlZ7BAQy4D+hCvh4B6lIG+nPdhbFfIbP+gTGBcrdsOaUs0F+ZBOQ==} + + safe-stable-stringify@2.4.3: + resolution: {integrity: sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==} + engines: {node: '>=10'} + + secure-json-parse@2.7.0: + resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} + + semver@7.6.0: + resolution: {integrity: sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==} + engines: {node: '>=10'} + hasBin: true + + set-cookie-parser@2.6.0: + resolution: {integrity: sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==} + + sonic-boom@3.8.0: + resolution: {integrity: sha512-ybz6OYOUjoQQCQ/i4LU8kaToD8ACtYP+Cj5qd2AO36bwbdewxWJ3ArmJ2cr6AvxlL2o0PqnCcPGUgkILbfkaCA==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + thread-stream@2.4.1: + resolution: {integrity: sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==} + + toad-cache@3.7.0: + resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} + engines: {node: '>=12'} + + typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + +snapshots: + + '@fastify/ajv-compiler@3.5.0': + dependencies: + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + fast-uri: 2.3.0 + + '@fastify/error@3.4.1': {} + + '@fastify/fast-json-stringify-compiler@4.3.0': + dependencies: + fast-json-stringify: 5.12.0 + + '@fastify/merge-json-schemas@0.1.1': + dependencies: + fast-deep-equal: 3.1.3 + + '@fastify/rate-limit@9.1.0': + dependencies: + '@lukeed/ms': 2.0.2 + fastify-plugin: 4.5.1 + toad-cache: 3.7.0 + + '@lukeed/ms@2.0.2': {} + + '@types/node@20.12.12': + dependencies: + undici-types: 5.26.5 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + abstract-logging@2.0.1: {} + + ajv-formats@2.1.1(ajv@8.12.0): + optionalDependencies: + ajv: 8.12.0 + + ajv@8.12.0: + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js: 4.4.1 + + archy@1.0.0: {} + + asynckit@0.4.0: {} + + atomic-sleep@1.0.0: {} + + avvio@8.3.0: + dependencies: + '@fastify/error': 3.4.1 + archy: 1.0.0 + debug: 4.3.4 + fastq: 1.17.1 + transitivePeerDependencies: + - supports-color + + axios-retry-after@2.0.0(axios@1.6.7): + dependencies: + axios: 1.6.7 + + axios@1.6.7: + dependencies: + follow-redirects: 1.15.5 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + base64-js@1.5.1: {} + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bytes@3.1.2: {} + + colorette@2.0.20: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + cookie@0.5.0: {} + + dateformat@4.6.3: {} + + debug@4.3.4: + dependencies: + ms: 2.1.2 + + delayed-stream@1.0.0: {} + + dotenv@16.4.4: {} + + end-of-stream@1.4.4: + dependencies: + once: 1.4.0 + + event-target-shim@5.0.1: {} + + events@3.3.0: {} + + fast-content-type-parse@1.1.0: {} + + fast-copy@3.0.1: {} + + fast-decode-uri-component@1.0.1: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stringify@5.12.0: + dependencies: + '@fastify/merge-json-schemas': 0.1.1 + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + fast-deep-equal: 3.1.3 + fast-uri: 2.3.0 + json-schema-ref-resolver: 1.0.1 + rfdc: 1.3.1 + + fast-querystring@1.1.2: + dependencies: + fast-decode-uri-component: 1.0.1 + + fast-redact@3.3.0: {} + + fast-safe-stringify@2.1.1: {} + + fast-uri@2.3.0: {} + + fastify-plugin@4.5.1: {} + + fastify@4.26.1: + dependencies: + '@fastify/ajv-compiler': 3.5.0 + '@fastify/error': 3.4.1 + '@fastify/fast-json-stringify-compiler': 4.3.0 + abstract-logging: 2.0.1 + avvio: 8.3.0 + fast-content-type-parse: 1.1.0 + fast-json-stringify: 5.12.0 + find-my-way: 8.1.0 + light-my-request: 5.11.0 + pino: 8.18.0 + process-warning: 3.0.0 + proxy-addr: 2.0.7 + rfdc: 1.3.1 + secure-json-parse: 2.7.0 + semver: 7.6.0 + toad-cache: 3.7.0 + transitivePeerDependencies: + - supports-color + + fastq@1.17.1: + dependencies: + reusify: 1.0.4 + + find-my-way@8.1.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-querystring: 1.1.2 + safe-regex2: 2.0.0 + + follow-redirects@1.15.5: {} + + form-data@4.0.0: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + + forwarded@0.2.0: {} + + help-me@5.0.0: {} + + ieee754@1.2.1: {} + + ipaddr.js@1.9.1: {} + + joycon@3.1.1: {} + + json-schema-ref-resolver@1.0.1: + dependencies: + fast-deep-equal: 3.1.3 + + json-schema-traverse@1.0.0: {} + + light-my-request@5.11.0: + dependencies: + cookie: 0.5.0 + process-warning: 2.3.2 + set-cookie-parser: 2.6.0 + + lru-cache@10.2.0: {} + + lru-cache@6.0.0: + dependencies: + yallist: 4.0.0 + + lru-ttl-cache@2.4.8: + dependencies: + bytes: 3.1.2 + ms: 2.1.3 + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + minimist@1.2.8: {} + + ms@2.1.2: {} + + ms@2.1.3: {} + + on-exit-leak-free@2.1.2: {} + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + pino-abstract-transport@1.1.0: + dependencies: + readable-stream: 4.5.2 + split2: 4.2.0 + + pino-pretty@10.3.1: + dependencies: + colorette: 2.0.20 + dateformat: 4.6.3 + fast-copy: 3.0.1 + fast-safe-stringify: 2.1.1 + help-me: 5.0.0 + joycon: 3.1.1 + minimist: 1.2.8 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 1.1.0 + pump: 3.0.0 + readable-stream: 4.5.2 + secure-json-parse: 2.7.0 + sonic-boom: 3.8.0 + strip-json-comments: 3.1.1 + + pino-std-serializers@6.2.2: {} + + pino@8.18.0: + dependencies: + atomic-sleep: 1.0.0 + fast-redact: 3.3.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 1.1.0 + pino-std-serializers: 6.2.2 + process-warning: 3.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.4.3 + sonic-boom: 3.8.0 + thread-stream: 2.4.1 + + process-warning@2.3.2: {} + + process-warning@3.0.0: {} + + process@0.11.10: {} + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + proxy-from-env@1.1.0: {} + + pump@3.0.0: + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + + punycode@2.3.1: {} + + quick-format-unescaped@4.0.4: {} + + readable-stream@4.5.2: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + real-require@0.2.0: {} + + require-from-string@2.0.2: {} + + ret@0.2.2: {} + + reusify@1.0.4: {} + + rfdc@1.3.1: {} + + safe-buffer@5.2.1: {} + + safe-regex2@2.0.0: + dependencies: + ret: 0.2.2 + + safe-stable-stringify@2.4.3: {} + + secure-json-parse@2.7.0: {} + + semver@7.6.0: + dependencies: + lru-cache: 6.0.0 + + set-cookie-parser@2.6.0: {} + + sonic-boom@3.8.0: + dependencies: + atomic-sleep: 1.0.0 + + split2@4.2.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-json-comments@3.1.1: {} + + thread-stream@2.4.1: + dependencies: + real-require: 0.2.0 + + toad-cache@3.7.0: {} + + typescript@5.3.3: {} + + undici-types@5.26.5: {} + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + wrappy@1.0.2: {} + + yallist@4.0.0: {} diff --git a/proxy/src/index.ts b/proxy/src/index.ts new file mode 100644 index 0000000..020c5ef --- /dev/null +++ b/proxy/src/index.ts @@ -0,0 +1,89 @@ + +import fastify, { + FastifyInstance, + FastifyListenOptions, + FastifyReply, + FastifyRequest, +} from "fastify" +import { PinoLoggerOptions } from "fastify/types/logger" +import { Proxy } from "./proxy" +import { Logger } from "pino" +import 'dotenv/config' + +const host = process.env.HOST +const port = parseInt(process.env.PORT ?? "8080", 10) +const baseUrl = process.env.NITTER_BASE_URL +const concurrency = parseInt(process.env.CONCURRENCY ?? "1", 10) +const retryAfterMillis = process.env.RETRY_AFTER_MILLIS ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) : null +const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? "100000", 10) +const logLevel = process.env.LOG_LEVEL ?? "debug" + +const server = fastify({ + logger: { + name: "app", + level: logLevel, + ...( logLevel == "trace" ? { transport: { target: 'pino-pretty' } } : {}) + } as PinoLoggerOptions +}) + +const log = server.log as Logger +const proxy = new Proxy(log, baseUrl, concurrency, retryAfterMillis, maxCacheSize) + +async function main() { + + server.register((fastify: FastifyInstance, opts, done) => { + + fastify.get(`/user/:username`, {}, + async (request: FastifyRequest, reply: FastifyReply) => { + log.debug({ + headers: request.headers, + reqId: request.id, + params: request.params }, 'incoming request /user/:username') + const { username } = request.params as any + const { status, data } = await proxy.getUser(username, { reqId: request.id }) + reply.status(status).send(data) + }); + + fastify.get(`/user/:userId/tweets`, {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { userId } = request.params as any + const { cursor } = request.query as any + const { status, data } = await proxy.getUserTweets(userId, cursor, { reqId: request.id }) + reply.status(status).send(data) + }); + + fastify.get(`/tweet/:id`, {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as any + const { status, data } = await proxy.getTweetById(id, { reqId: request.id }) + reply.status(status).send(data) + }); + + done() + + }, { prefix: '/api' }) + + server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => { + reply.status(404) + .send({ message: `Method not found` }) + }) + + server.setErrorHandler((err: Error, request: FastifyRequest, reply: FastifyReply) => { + const { log } = request + log.error(err) + // Send error response + reply.status(500).send({ message: `Internal server error` }) + }) + + // await server.register(import('@fastify/rate-limit'), { + // max: 100, + // timeWindow: '1 minute' + // }) + + await server.listen({ port, host } as FastifyListenOptions); +} + +main().catch(err => { + log.fatal(err) + process.exit(1) +}) diff --git a/proxy/src/proxy.ts b/proxy/src/proxy.ts new file mode 100644 index 0000000..872f481 --- /dev/null +++ b/proxy/src/proxy.ts @@ -0,0 +1,209 @@ +// noinspection TypeScriptUnresolvedReference + +import axios from "axios" +import { AxiosInstance, AxiosRequestConfig } from "axios" +import fastq from "fastq" +import { Logger } from "pino" +import retry from "axios-retry-after" +import { LRUCache } from 'lru-cache' + +const GET_USER_POSITIVE_TTL_MS = process.env.GET_USER_POSITIVE_TTL + ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000 + : 30 * 24 * 3600 * 1000 +const GET_USER_NEGATIVE_TTL_MS = process.env.GET_USER_NEGATIVE_TTL + ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000 + : 3600 * 1000 +const GET_TWEETS_POSITIVE_TTL_MS = process.env.GET_TWEETS_POSITIVE_TTL + ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEETS_NEGATIVE_TTL_MS = process.env.GET_TWEETS_NEGATIVE_TTL + ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEET_POSITIVE_TTL_MS = process.env.GET_TWEET_POSITIVE_TTL + ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEET_NEGATIVE_TTL_MS = process.env.GET_TWEET_NEGATIVE_TTL + ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 + +export interface Job { + reqId: string + url: string + params?: Record +} + +export interface JobResponse { + status: number, + data: any +} + +export class Proxy { + + private readonly cache: LRUCache + private readonly client: AxiosInstance + private readonly queue: fastq.queueAsPromised + private counter: { requests: number } + private timeWindowMillis = 15 * 60 * 1000 + private maxRequestsPerAccount = 15 * 60 + + constructor( + private log: Logger, + private baseUrl: string, + private concurrency: number, + retryAfterMillis: number, + maxCacheSize: number + ) { + this.cache = new LRUCache({ max: maxCacheSize }) + this.queue = fastq.promise(this, this.sendRequest, concurrency) + this.client = axios.create() + this.counter = { + requests: 0 + } + + setInterval(() => { + this.counter.requests = 0 + }, this.timeWindowMillis) + + if ( retryAfterMillis ) { + this.client.interceptors.response.use(null, retry(this.client, { + // Determine when we should attempt to retry + isRetryable (error) { + log.debug({ status: error.response?.status, headers: error.response?.headers }, 'checking retryable') + return ( + error.response && error.response.status === 429 + // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds + // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60 + ) + }, + // Customize the wait behavior + wait (error) { + log.debug({ status: error.response?.status, headers: error.response?.headers }, 'waiting for retry') + return new Promise( + // Use X-Retry-After rather than Retry-After + // resolve => setTimeout(resolve, error.response.headers['x-retry-after']) + resolve => setTimeout(resolve, retryAfterMillis) + ) + } + })) + } + } + + async getUser(username: string, options?: { reqId?: string }) { + const key = `usernames:${username}` + + if ( this.cache.has(key)) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/user/${ username }`, + reqId: options?.reqId + }) + + if ( result.status === 200 ) { + this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS }) + } + if ( result.status === 404 ) { + this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS }) + } + + return result + } + + async getUserTweets(userId: string, cursor?: string, options?: { reqId?: string }) { + const key = `users:${userId}:tweets:${cursor ?? 'last'}` + + if ( this.cache.has(key) ) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/user/${ userId }/tweets`, + params: { cursor }, + reqId: options?.reqId + }) + + if ( result.status === 200 ) { + this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS }) + } + if ( result.status === 404 ) { + this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS }) + } + + return result + } + + async getTweetById(tweetId: string, options?: { reqId?: string }) { + const key = `tweets:${tweetId}` + + if ( this.cache.has(key) ) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/tweet/${ tweetId }`, + reqId: options?.reqId + }) + + if ( result.status === 200 ) { + this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS }) + } + if ( result.status === 404 ) { + this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS }) + } + + return result + } + + private async sendRequest(job: Job): Promise { + + const { reqId, url, params } = job + + if ( this.counter.requests > this.concurrency * this.maxRequestsPerAccount ) { + return { + status: 429 + } + } + + let config = { + url, + method: "get", + baseURL: this.baseUrl, + params, + } as AxiosRequestConfig + + this.log.trace({ config, reqId: reqId }, 'sending request to nitter') + + try { + const response = await this.client.request(config) + + this.log.trace({ + status: response.status, + data: response.data, + reqId: reqId + }, 'nitter response') + + return { + status: response.status, + data: response.data, + } as JobResponse + + } catch(err) { + + this.log.warn({ err, reqId }, 'nitter error') + + if ( err.name === "AxiosError" ) { + + this.counter.requests = Number.MAX_SAFE_INTEGER + + return { + status: 429 + } as JobResponse + } + + return { + status: 500 + } + } + } +} diff --git a/proxy/src/types.d.ts b/proxy/src/types.d.ts new file mode 100644 index 0000000..a0c18fa --- /dev/null +++ b/proxy/src/types.d.ts @@ -0,0 +1,39 @@ +declare module 'axios-retry-after' { + + import { AxiosError, AxiosInstance } from "axios"; + + /** + * Function to enhance Axios instance with retry-after functionality. + * @param axios Axios instance to be enhanced. + * @param options Configuration options for retry behavior. + */ + export default function( + axios: AxiosInstance, + options?: AxiosRetryAfterOptions + ): (error: AxiosError) => Promise; + + /** + * Configuration options for axios-retry-after. + */ + export interface AxiosRetryAfterOptions { + /** + * Function to determine if an error response is retryable. + * @param error The Axios error to evaluate. + */ + isRetryable?: (error: AxiosError) => boolean; + + /** + * Function to wait for a specified amount of time. + * @param error The Axios error that contains retry-after header. + */ + wait?: (error: AxiosError) => Promise; + + /** + * Function to retry the original request. + * @param axios The Axios instance used for retrying the request. + * @param error The Axios error to retry. + */ + retry?: (axios: AxiosInstance, error: AxiosError) => Promise; + } +} + diff --git a/proxy/tsconfig.json b/proxy/tsconfig.json new file mode 100644 index 0000000..12c5931 --- /dev/null +++ b/proxy/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "lib": ["esnext"], + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "module": "commonjs", + "target": "esnext", + "allowJs": true, + "sourceMap": true, + "outDir": "./build" + }, + "include": ["src/**/*"], + "exclude": [ + "node_modules" + ] +}