diff --git a/.gitignore b/.gitignore index ea520dc..55031d6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ nitter nitter.conf guest_accounts.json* dump.rdb +proxy/node_modules diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 deleted file mode 100644 index 70024b2..0000000 --- a/Dockerfile.arm64 +++ /dev/null @@ -1,25 +0,0 @@ -FROM alpine:3.18 as nim -LABEL maintainer="setenforce@protonmail.com" - -RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.14-r0" "nimble=0.13.1-r2" - -WORKDIR /src/nitter - -COPY nitter.nimble . -RUN nimble install -y --depsOnly - -COPY . . -RUN nimble build -d:danger -d:lto -d:strip \ - && nimble scss \ - && nimble md - -FROM alpine:3.18 -WORKDIR /src/ -RUN apk --no-cache add pcre ca-certificates openssl1.1-compat -COPY --from=nim /src/nitter/nitter ./ -COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf -COPY --from=nim /src/nitter/public ./public -EXPOSE 8080 -RUN adduser -h /src/ -D -s /bin/sh nitter -USER nitter -CMD ./nitter diff --git a/proxy/.npmrc b/proxy/.npmrc new file mode 100644 index 0000000..3bd3b7d --- /dev/null +++ b/proxy/.npmrc @@ -0,0 +1 @@ +shell-emulator=true diff --git a/proxy/.prettierignore b/proxy/.prettierignore new file mode 100644 index 0000000..18f1a04 --- /dev/null +++ b/proxy/.prettierignore @@ -0,0 +1,2 @@ +**/*.md +pnpm-lock.yaml diff --git a/proxy/.prettierrc.yaml b/proxy/.prettierrc.yaml new file mode 100644 index 0000000..88e0a15 --- /dev/null +++ b/proxy/.prettierrc.yaml @@ -0,0 +1,6 @@ +proseWrap: always +semi: false +singleQuote: true +printWidth: 80 +trailingComma: none +htmlWhitespaceSensitivity: ignore diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 0000000..000ead8 --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,21 @@ +# this is our first build stage, it will not persist in the final image +FROM node:20.11.0-buster-slim as intermediate + +# installation required packages +RUN apt-get update && apt-get install -y --no-install-recommends ssh git python python3 build-essential +RUN npm install -g pnpm@9.1.1 +RUN mkdir -p /opt +WORKDIR /opt + +COPY tsconfig.json /opt +COPY package.json pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile + +COPY ./src /opt/src + +RUN pnpm run build + +# copy just the package form the previous image +FROM node:20.11.0-buster-slim +COPY --from=intermediate /opt /opt +ENTRYPOINT ["node", "/opt/build/index.js"] diff --git a/proxy/package.json b/proxy/package.json new file mode 100644 index 0000000..0aa594b --- /dev/null +++ b/proxy/package.json @@ -0,0 +1,29 @@ +{ + "name": "nitter-proxy", + "version": "1.6.7", + "scripts": { + "clean": "rm -rf build", + "prebuild": "npm run clean", + "format": "prettier -w --cache --check .", + "build": "tsc --build" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@fastify/rate-limit": "^9.1.0", + "axios": "^1.6.7", + "axios-retry-after": "^2.0.0", + "fastify": "^4.21.0", + "fastq": "^1.17.1", + "lru-cache": "^10.2.0", + "lru-ttl-cache": "^2.4.8", + "pino": "^8.14.2", + "pino-pretty": "^10.2.0", + "typescript": "^5.3.3" + }, + "devDependencies": { + "@types/node": "^20.12.12", + "dotenv": "^16.4.4", + "prettier": "^3.2.5" + } +} diff --git a/proxy/pnpm-lock.yaml b/proxy/pnpm-lock.yaml new file mode 100644 index 0000000..1876821 --- /dev/null +++ b/proxy/pnpm-lock.yaml @@ -0,0 +1,765 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@fastify/rate-limit': + specifier: ^9.1.0 + version: 9.1.0 + axios: + specifier: ^1.6.7 + version: 1.6.7 + axios-retry-after: + specifier: ^2.0.0 + version: 2.0.0(axios@1.6.7) + fastify: + specifier: ^4.21.0 + version: 4.26.1 + fastq: + specifier: ^1.17.1 + version: 1.17.1 + lru-cache: + specifier: ^10.2.0 + version: 10.2.0 + lru-ttl-cache: + specifier: ^2.4.8 + version: 2.4.8 + pino: + specifier: ^8.14.2 + version: 8.18.0 + pino-pretty: + specifier: ^10.2.0 + version: 10.3.1 + typescript: + specifier: ^5.3.3 + version: 5.3.3 + devDependencies: + '@types/node': + specifier: ^20.12.12 + version: 20.12.12 + dotenv: + specifier: ^16.4.4 + version: 16.4.4 + prettier: + specifier: ^3.2.5 + version: 3.2.5 + +packages: + + '@fastify/ajv-compiler@3.5.0': + resolution: {integrity: sha512-ebbEtlI7dxXF5ziNdr05mOY8NnDiPB1XvAlLHctRt/Rc+C3LCOVW5imUVX+mhvUhnNzmPBHewUkOFgGlCxgdAA==} + + '@fastify/error@3.4.1': + resolution: {integrity: sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ==} + + '@fastify/fast-json-stringify-compiler@4.3.0': + resolution: {integrity: sha512-aZAXGYo6m22Fk1zZzEUKBvut/CIIQe/BapEORnxiD5Qr0kPHqqI69NtEMCme74h+at72sPhbkb4ZrLd1W3KRLA==} + + '@fastify/merge-json-schemas@0.1.1': + resolution: {integrity: sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA==} + + '@fastify/rate-limit@9.1.0': + resolution: {integrity: sha512-h5dZWCkuZXN0PxwqaFQLxeln8/LNwQwH9popywmDCFdKfgpi4b/HoMH1lluy6P+30CG9yzzpSpwTCIPNB9T1JA==} + + '@lukeed/ms@2.0.2': + resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} + engines: {node: '>=8'} + + '@types/node@20.12.12': + resolution: {integrity: sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + abstract-logging@2.0.1: + resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + + ajv-formats@2.1.1: + resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + + archy@1.0.0: + resolution: {integrity: sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + + avvio@8.3.0: + resolution: {integrity: sha512-VBVH0jubFr9LdFASy/vNtm5giTrnbVquWBhT0fyizuNK2rQ7e7ONU2plZQWUNqtE1EmxFEb+kbSkFRkstiaS9Q==} + + axios-retry-after@2.0.0: + resolution: {integrity: sha512-tSB1DEF1bSwXmRNyPcopFsiHAF+PWVq5w2mAK7J0bTltn8x2UnfoSJzTVXPySt/WdrbQL4ES5AXtG9i016+CaA==} + engines: {node: '>= 14'} + peerDependencies: + axios: ^1.0.0 + + axios@1.6.7: + resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + cookie@0.5.0: + resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} + engines: {node: '>= 0.6'} + + dateformat@4.6.3: + resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + + debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dotenv@16.4.4: + resolution: {integrity: sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg==} + engines: {node: '>=12'} + + end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + fast-content-type-parse@1.1.0: + resolution: {integrity: sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==} + + fast-copy@3.0.1: + resolution: {integrity: sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA==} + + fast-decode-uri-component@1.0.1: + resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stringify@5.12.0: + resolution: {integrity: sha512-7Nnm9UPa7SfHRbHVA1kJQrGXCRzB7LMlAAqHXQFkEQqueJm1V8owm0FsE/2Do55/4CcdhwiLQERaKomOnKQkyA==} + + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + + fast-redact@3.3.0: + resolution: {integrity: sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==} + engines: {node: '>=6'} + + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + + fast-uri@2.3.0: + resolution: {integrity: sha512-eel5UKGn369gGEWOqBShmFJWfq/xSJvsgDzgLYC845GneayWvXBf0lJCBn5qTABfewy1ZDPoaR5OZCP+kssfuw==} + + fastify-plugin@4.5.1: + resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==} + + fastify@4.26.1: + resolution: {integrity: sha512-tznA/G55dsxzM5XChBfcvVSloG2ejeeotfPPJSFaWmHyCDVGMpvf3nRNbsCb/JTBF9RmQFBfuujWt3Nphjesng==} + + fastq@1.17.1: + resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} + + find-my-way@8.1.0: + resolution: {integrity: sha512-41QwjCGcVTODUmLLqTMeoHeiozbMXYMAE1CKFiDyi9zVZ2Vjh0yz3MF0WQZoIb+cmzP/XlbFjlF2NtJmvZHznA==} + engines: {node: '>=14'} + + follow-redirects@1.15.5: + resolution: {integrity: sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + help-me@5.0.0: + resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + + json-schema-ref-resolver@1.0.1: + resolution: {integrity: sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + light-my-request@5.11.0: + resolution: {integrity: sha512-qkFCeloXCOMpmEdZ/MV91P8AT4fjwFXWaAFz3lUeStM8RcoM1ks4J/F8r1b3r6y/H4u3ACEJ1T+Gv5bopj7oDA==} + + lru-cache@10.2.0: + resolution: {integrity: sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==} + engines: {node: 14 || >=16.14} + + lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + + lru-ttl-cache@2.4.8: + resolution: {integrity: sha512-24fyw4EHvHKSGXc6DMpcA5Iet9UnPxrXzcXA9bvkTs/9kQL6m9SllRyWAtZdxzUabKWdm38vOHavXwdDx2Zl8g==} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + pino-abstract-transport@1.1.0: + resolution: {integrity: sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==} + + pino-pretty@10.3.1: + resolution: {integrity: sha512-az8JbIYeN/1iLj2t0jR9DV48/LQ3RC6hZPpapKPkb84Q+yTidMCpgWxIT3N0flnBDilyBQ1luWNpOeJptjdp/g==} + hasBin: true + + pino-std-serializers@6.2.2: + resolution: {integrity: sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==} + + pino@8.18.0: + resolution: {integrity: sha512-Mz/gKiRyuXu4HnpHgi1YWdHQCoWMufapzooisvFn78zl4dZciAxS+YeRkUxXl1ee/SzU80YCz1zpECCh4oC6Aw==} + hasBin: true + + prettier@3.2.5: + resolution: {integrity: sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==} + engines: {node: '>=14'} + hasBin: true + + process-warning@2.3.2: + resolution: {integrity: sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==} + + process-warning@3.0.0: + resolution: {integrity: sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + pump@3.0.0: + resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + + readable-stream@4.5.2: + resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + ret@0.2.2: + resolution: {integrity: sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ==} + engines: {node: '>=4'} + + reusify@1.0.4: + resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + rfdc@1.3.1: + resolution: {integrity: sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-regex2@2.0.0: + resolution: {integrity: sha512-PaUSFsUaNNuKwkBijoAPHAK6/eM6VirvyPWlZ7BAQy4D+hCvh4B6lIG+nPdhbFfIbP+gTGBcrdsOaUs0F+ZBOQ==} + + safe-stable-stringify@2.4.3: + resolution: {integrity: sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==} + engines: {node: '>=10'} + + secure-json-parse@2.7.0: + resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} + + semver@7.6.0: + resolution: {integrity: sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==} + engines: {node: '>=10'} + hasBin: true + + set-cookie-parser@2.6.0: + resolution: {integrity: sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==} + + sonic-boom@3.8.0: + resolution: {integrity: sha512-ybz6OYOUjoQQCQ/i4LU8kaToD8ACtYP+Cj5qd2AO36bwbdewxWJ3ArmJ2cr6AvxlL2o0PqnCcPGUgkILbfkaCA==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + thread-stream@2.4.1: + resolution: {integrity: sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==} + + toad-cache@3.7.0: + resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} + engines: {node: '>=12'} + + typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + +snapshots: + + '@fastify/ajv-compiler@3.5.0': + dependencies: + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + fast-uri: 2.3.0 + + '@fastify/error@3.4.1': {} + + '@fastify/fast-json-stringify-compiler@4.3.0': + dependencies: + fast-json-stringify: 5.12.0 + + '@fastify/merge-json-schemas@0.1.1': + dependencies: + fast-deep-equal: 3.1.3 + + '@fastify/rate-limit@9.1.0': + dependencies: + '@lukeed/ms': 2.0.2 + fastify-plugin: 4.5.1 + toad-cache: 3.7.0 + + '@lukeed/ms@2.0.2': {} + + '@types/node@20.12.12': + dependencies: + undici-types: 5.26.5 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + abstract-logging@2.0.1: {} + + ajv-formats@2.1.1(ajv@8.12.0): + optionalDependencies: + ajv: 8.12.0 + + ajv@8.12.0: + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js: 4.4.1 + + archy@1.0.0: {} + + asynckit@0.4.0: {} + + atomic-sleep@1.0.0: {} + + avvio@8.3.0: + dependencies: + '@fastify/error': 3.4.1 + archy: 1.0.0 + debug: 4.3.4 + fastq: 1.17.1 + transitivePeerDependencies: + - supports-color + + axios-retry-after@2.0.0(axios@1.6.7): + dependencies: + axios: 1.6.7 + + axios@1.6.7: + dependencies: + follow-redirects: 1.15.5 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + base64-js@1.5.1: {} + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bytes@3.1.2: {} + + colorette@2.0.20: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + cookie@0.5.0: {} + + dateformat@4.6.3: {} + + debug@4.3.4: + dependencies: + ms: 2.1.2 + + delayed-stream@1.0.0: {} + + dotenv@16.4.4: {} + + end-of-stream@1.4.4: + dependencies: + once: 1.4.0 + + event-target-shim@5.0.1: {} + + events@3.3.0: {} + + fast-content-type-parse@1.1.0: {} + + fast-copy@3.0.1: {} + + fast-decode-uri-component@1.0.1: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stringify@5.12.0: + dependencies: + '@fastify/merge-json-schemas': 0.1.1 + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + fast-deep-equal: 3.1.3 + fast-uri: 2.3.0 + json-schema-ref-resolver: 1.0.1 + rfdc: 1.3.1 + + fast-querystring@1.1.2: + dependencies: + fast-decode-uri-component: 1.0.1 + + fast-redact@3.3.0: {} + + fast-safe-stringify@2.1.1: {} + + fast-uri@2.3.0: {} + + fastify-plugin@4.5.1: {} + + fastify@4.26.1: + dependencies: + '@fastify/ajv-compiler': 3.5.0 + '@fastify/error': 3.4.1 + '@fastify/fast-json-stringify-compiler': 4.3.0 + abstract-logging: 2.0.1 + avvio: 8.3.0 + fast-content-type-parse: 1.1.0 + fast-json-stringify: 5.12.0 + find-my-way: 8.1.0 + light-my-request: 5.11.0 + pino: 8.18.0 + process-warning: 3.0.0 + proxy-addr: 2.0.7 + rfdc: 1.3.1 + secure-json-parse: 2.7.0 + semver: 7.6.0 + toad-cache: 3.7.0 + transitivePeerDependencies: + - supports-color + + fastq@1.17.1: + dependencies: + reusify: 1.0.4 + + find-my-way@8.1.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-querystring: 1.1.2 + safe-regex2: 2.0.0 + + follow-redirects@1.15.5: {} + + form-data@4.0.0: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + + forwarded@0.2.0: {} + + help-me@5.0.0: {} + + ieee754@1.2.1: {} + + ipaddr.js@1.9.1: {} + + joycon@3.1.1: {} + + json-schema-ref-resolver@1.0.1: + dependencies: + fast-deep-equal: 3.1.3 + + json-schema-traverse@1.0.0: {} + + light-my-request@5.11.0: + dependencies: + cookie: 0.5.0 + process-warning: 2.3.2 + set-cookie-parser: 2.6.0 + + lru-cache@10.2.0: {} + + lru-cache@6.0.0: + dependencies: + yallist: 4.0.0 + + lru-ttl-cache@2.4.8: + dependencies: + bytes: 3.1.2 + ms: 2.1.3 + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + minimist@1.2.8: {} + + ms@2.1.2: {} + + ms@2.1.3: {} + + on-exit-leak-free@2.1.2: {} + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + pino-abstract-transport@1.1.0: + dependencies: + readable-stream: 4.5.2 + split2: 4.2.0 + + pino-pretty@10.3.1: + dependencies: + colorette: 2.0.20 + dateformat: 4.6.3 + fast-copy: 3.0.1 + fast-safe-stringify: 2.1.1 + help-me: 5.0.0 + joycon: 3.1.1 + minimist: 1.2.8 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 1.1.0 + pump: 3.0.0 + readable-stream: 4.5.2 + secure-json-parse: 2.7.0 + sonic-boom: 3.8.0 + strip-json-comments: 3.1.1 + + pino-std-serializers@6.2.2: {} + + pino@8.18.0: + dependencies: + atomic-sleep: 1.0.0 + fast-redact: 3.3.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 1.1.0 + pino-std-serializers: 6.2.2 + process-warning: 3.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.4.3 + sonic-boom: 3.8.0 + thread-stream: 2.4.1 + + prettier@3.2.5: {} + + process-warning@2.3.2: {} + + process-warning@3.0.0: {} + + process@0.11.10: {} + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + proxy-from-env@1.1.0: {} + + pump@3.0.0: + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + + punycode@2.3.1: {} + + quick-format-unescaped@4.0.4: {} + + readable-stream@4.5.2: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + real-require@0.2.0: {} + + require-from-string@2.0.2: {} + + ret@0.2.2: {} + + reusify@1.0.4: {} + + rfdc@1.3.1: {} + + safe-buffer@5.2.1: {} + + safe-regex2@2.0.0: + dependencies: + ret: 0.2.2 + + safe-stable-stringify@2.4.3: {} + + secure-json-parse@2.7.0: {} + + semver@7.6.0: + dependencies: + lru-cache: 6.0.0 + + set-cookie-parser@2.6.0: {} + + sonic-boom@3.8.0: + dependencies: + atomic-sleep: 1.0.0 + + split2@4.2.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-json-comments@3.1.1: {} + + thread-stream@2.4.1: + dependencies: + real-require: 0.2.0 + + toad-cache@3.7.0: {} + + typescript@5.3.3: {} + + undici-types@5.26.5: {} + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + wrappy@1.0.2: {} + + yallist@4.0.0: {} diff --git a/proxy/src/index.ts b/proxy/src/index.ts new file mode 100644 index 0000000..b7c121d --- /dev/null +++ b/proxy/src/index.ts @@ -0,0 +1,116 @@ +import fastify, { + FastifyInstance, + FastifyListenOptions, + FastifyReply, + FastifyRequest +} from 'fastify' +import { PinoLoggerOptions } from 'fastify/types/logger' +import { Proxy } from './proxy' +import { Logger } from 'pino' +import 'dotenv/config' + +const host = process.env.HOST +const port = parseInt(process.env.PORT ?? '8080', 10) +const baseUrl = process.env.NITTER_BASE_URL +const concurrency = parseInt(process.env.CONCURRENCY ?? '1', 10) +const retryAfterMillis = process.env.RETRY_AFTER_MILLIS + ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) + : null +const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? '100000', 10) +const logLevel = process.env.LOG_LEVEL ?? 'debug' + +const server = fastify({ + logger: { + name: 'app', + level: logLevel, + ...(logLevel == 'trace' ? { transport: { target: 'pino-pretty' } } : {}) + } as PinoLoggerOptions +}) + +const log = server.log as Logger +const proxy = new Proxy( + log, + baseUrl, + concurrency, + retryAfterMillis, + maxCacheSize +) + +async function main() { + server.register( + (fastify: FastifyInstance, opts, done) => { + fastify.get( + `/user/:username`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + log.debug( + { + headers: request.headers, + reqId: request.id, + params: request.params + }, + 'incoming request /user/:username' + ) + const { username } = request.params as any + const { status, data } = await proxy.getUser(username, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) + + fastify.get( + `/user/:userId/tweets`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { userId } = request.params as any + const { cursor } = request.query as any + const { status, data } = await proxy.getUserTweets(userId, cursor, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) + + fastify.get( + `/tweet/:id`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as any + const { status, data } = await proxy.getTweetById(id, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) + + done() + }, + { prefix: '/api' } + ) + + server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => { + reply.status(404).send({ message: `Method not found` }) + }) + + server.setErrorHandler( + (err: Error, request: FastifyRequest, reply: FastifyReply) => { + const { log } = request + log.error(err) + // Send error response + reply.status(500).send({ message: `Internal server error` }) + } + ) + + // await server.register(import('@fastify/rate-limit'), { + // max: 100, + // timeWindow: '1 minute' + // }) + + await server.listen({ port, host } as FastifyListenOptions) +} + +main().catch((err) => { + log.fatal(err) + process.exit(1) +}) diff --git a/proxy/src/proxy.ts b/proxy/src/proxy.ts new file mode 100644 index 0000000..a9fb9e5 --- /dev/null +++ b/proxy/src/proxy.ts @@ -0,0 +1,226 @@ +// noinspection TypeScriptUnresolvedReference + +import axios from 'axios' +import { AxiosInstance, AxiosRequestConfig } from 'axios' +import fastq from 'fastq' +import { Logger } from 'pino' +import retry from 'axios-retry-after' +import { LRUCache } from 'lru-cache' + +const GET_USER_POSITIVE_TTL_MS = process.env.GET_USER_POSITIVE_TTL + ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000 + : 30 * 24 * 3600 * 1000 +const GET_USER_NEGATIVE_TTL_MS = process.env.GET_USER_NEGATIVE_TTL + ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000 + : 3600 * 1000 +const GET_TWEETS_POSITIVE_TTL_MS = process.env.GET_TWEETS_POSITIVE_TTL + ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEETS_NEGATIVE_TTL_MS = process.env.GET_TWEETS_NEGATIVE_TTL + ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEET_POSITIVE_TTL_MS = process.env.GET_TWEET_POSITIVE_TTL + ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEET_NEGATIVE_TTL_MS = process.env.GET_TWEET_NEGATIVE_TTL + ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 + +export interface Job { + reqId: string + url: string + params?: Record +} + +export interface JobResponse { + status: number + data: any +} + +export class Proxy { + private readonly cache: LRUCache + private readonly client: AxiosInstance + private readonly queue: fastq.queueAsPromised + private counter: { requests: number } + private timeWindowMillis = 15 * 60 * 1000 + private maxRequestsPerAccount = 15 * 60 + + constructor( + private log: Logger, + private baseUrl: string, + private concurrency: number, + retryAfterMillis: number, + maxCacheSize: number + ) { + this.cache = new LRUCache({ max: maxCacheSize }) + this.queue = fastq.promise(this, this.sendRequest, concurrency) + this.client = axios.create() + this.counter = { + requests: 0 + } + + setInterval(() => { + this.counter.requests = 0 + }, this.timeWindowMillis) + + if (retryAfterMillis) { + this.client.interceptors.response.use( + null, + retry(this.client, { + // Determine when we should attempt to retry + isRetryable(error) { + log.debug( + { + status: error.response?.status, + headers: error.response?.headers + }, + 'checking retryable' + ) + return ( + error.response && error.response.status === 429 + // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds + // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60 + ) + }, + // Customize the wait behavior + wait(error) { + log.debug( + { + status: error.response?.status, + headers: error.response?.headers + }, + 'waiting for retry' + ) + return new Promise( + // Use X-Retry-After rather than Retry-After + // resolve => setTimeout(resolve, error.response.headers['x-retry-after']) + (resolve) => setTimeout(resolve, retryAfterMillis) + ) + } + }) + ) + } + } + + async getUser(username: string, options?: { reqId?: string }) { + const key = `usernames:${username}` + + if (this.cache.has(key)) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/user/${username}`, + reqId: options?.reqId + }) + + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS }) + } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS }) + } + + return result + } + + async getUserTweets( + userId: string, + cursor?: string, + options?: { reqId?: string } + ) { + const key = `users:${userId}:tweets:${cursor ?? 'last'}` + + if (this.cache.has(key)) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/user/${userId}/tweets`, + params: { cursor }, + reqId: options?.reqId + }) + + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS }) + } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS }) + } + + return result + } + + async getTweetById(tweetId: string, options?: { reqId?: string }) { + const key = `tweets:${tweetId}` + + if (this.cache.has(key)) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/tweet/${tweetId}`, + reqId: options?.reqId + }) + + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS }) + } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS }) + } + + return result + } + + private async sendRequest(job: Job): Promise { + const { reqId, url, params } = job + + if (this.counter.requests > this.concurrency * this.maxRequestsPerAccount) { + return { + status: 429 + } + } + + let config = { + url, + method: 'get', + baseURL: this.baseUrl, + params + } as AxiosRequestConfig + + this.log.trace({ config, reqId: reqId }, 'sending request to nitter') + + try { + const response = await this.client.request(config) + + this.log.trace( + { + status: response.status, + data: response.data, + reqId: reqId + }, + 'nitter response' + ) + + return { + status: response.status, + data: response.data + } as JobResponse + } catch (err) { + this.log.warn({ err, reqId }, 'nitter error') + + if (err.name === 'AxiosError') { + this.counter.requests = Number.MAX_SAFE_INTEGER + + return { + status: 429 + } as JobResponse + } + + return { + status: 500 + } + } + } +} diff --git a/proxy/src/types.d.ts b/proxy/src/types.d.ts new file mode 100644 index 0000000..cd9cf13 --- /dev/null +++ b/proxy/src/types.d.ts @@ -0,0 +1,37 @@ +declare module 'axios-retry-after' { + import { AxiosError, AxiosInstance } from 'axios' + + /** + * Function to enhance Axios instance with retry-after functionality. + * @param axios Axios instance to be enhanced. + * @param options Configuration options for retry behavior. + */ + export default function ( + axios: AxiosInstance, + options?: AxiosRetryAfterOptions + ): (error: AxiosError) => Promise + + /** + * Configuration options for axios-retry-after. + */ + export interface AxiosRetryAfterOptions { + /** + * Function to determine if an error response is retryable. + * @param error The Axios error to evaluate. + */ + isRetryable?: (error: AxiosError) => boolean + + /** + * Function to wait for a specified amount of time. + * @param error The Axios error that contains retry-after header. + */ + wait?: (error: AxiosError) => Promise + + /** + * Function to retry the original request. + * @param axios The Axios instance used for retrying the request. + * @param error The Axios error to retry. + */ + retry?: (axios: AxiosInstance, error: AxiosError) => Promise + } +} diff --git a/proxy/tsconfig.json b/proxy/tsconfig.json new file mode 100644 index 0000000..ee6e859 --- /dev/null +++ b/proxy/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "lib": ["esnext"], + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "module": "commonjs", + "target": "esnext", + "allowJs": true, + "sourceMap": true, + "outDir": "./build" + }, + "include": ["src/**/*"], + "exclude": ["node_modules"] +} diff --git a/public/css/baguetteBox.min.css b/public/css/baguetteBox.min.css new file mode 100644 index 0000000..f1c5ca9 --- /dev/null +++ b/public/css/baguetteBox.min.css @@ -0,0 +1,6 @@ +/*! + * baguetteBox.js + * @author feimosi + * @version 1.11.1 + * @url https://github.com/feimosi/baguetteBox.js + */#baguetteBox-overlay{display:none;opacity:0;position:fixed;overflow:hidden;top:0;left:0;width:100%;height:100%;z-index:1000000;background-color:#222;background-color:rgba(0,0,0,.8);-webkit-transition:opacity .5s ease;transition:opacity .5s ease}#baguetteBox-overlay.visible{opacity:1}#baguetteBox-overlay .full-image{display:inline-block;position:relative;width:100%;height:100%;text-align:center}#baguetteBox-overlay .full-image figure{display:inline;margin:0;height:100%}#baguetteBox-overlay .full-image img{display:inline-block;width:auto;height:auto;max-height:100%;max-width:100%;vertical-align:middle;-webkit-box-shadow:0 0 8px rgba(0,0,0,.6);-moz-box-shadow:0 0 8px rgba(0,0,0,.6);box-shadow:0 0 8px rgba(0,0,0,.6)}#baguetteBox-overlay .full-image figcaption{display:block;position:absolute;bottom:0;width:100%;text-align:center;line-height:1.8;white-space:normal;color:#ccc;background-color:#000;background-color:rgba(0,0,0,.6);font-family:sans-serif}#baguetteBox-overlay .full-image:before{content:"";display:inline-block;height:50%;width:1px;margin-right:-1px}#baguetteBox-slider{position:absolute;left:0;top:0;height:100%;width:100%;white-space:nowrap;-webkit-transition:left .4s ease,-webkit-transform .4s ease;transition:left .4s ease,-webkit-transform .4s ease;transition:left .4s ease,transform .4s ease;transition:left .4s ease,transform .4s ease,-webkit-transform .4s ease,-moz-transform .4s ease}#baguetteBox-slider.bounce-from-right{-webkit-animation:bounceFromRight .4s ease-out;animation:bounceFromRight .4s ease-out}#baguetteBox-slider.bounce-from-left{-webkit-animation:bounceFromLeft .4s ease-out;animation:bounceFromLeft .4s ease-out}@-webkit-keyframes bounceFromRight{0%,100%{margin-left:0}50%{margin-left:-30px}}@keyframes bounceFromRight{0%,100%{margin-left:0}50%{margin-left:-30px}}@-webkit-keyframes bounceFromLeft{0%,100%{margin-left:0}50%{margin-left:30px}}@keyframes bounceFromLeft{0%,100%{margin-left:0}50%{margin-left:30px}}.baguetteBox-button#next-button,.baguetteBox-button#previous-button{top:50%;top:calc(50% - 30px);width:44px;height:60px}.baguetteBox-button{position:absolute;cursor:pointer;outline:0;padding:0;margin:0;border:0;-moz-border-radius:15%;border-radius:15%;background-color:#323232;background-color:rgba(50,50,50,.5);color:#ddd;font:1.6em sans-serif;-webkit-transition:background-color .4s ease;transition:background-color .4s ease}.baguetteBox-button:focus,.baguetteBox-button:hover{background-color:rgba(50,50,50,.9)}.baguetteBox-button#next-button{right:2%}.baguetteBox-button#previous-button{left:2%}.baguetteBox-button#close-button{top:20px;right:2%;right:calc(2% + 6px);width:30px;height:30px}.baguetteBox-button svg{position:absolute;left:0;top:0}.baguetteBox-spinner{width:40px;height:40px;display:inline-block;position:absolute;top:50%;left:50%;margin-top:-20px;margin-left:-20px}.baguetteBox-double-bounce1,.baguetteBox-double-bounce2{width:100%;height:100%;-moz-border-radius:50%;border-radius:50%;background-color:#fff;opacity:.6;position:absolute;top:0;left:0;-webkit-animation:bounce 2s infinite ease-in-out;animation:bounce 2s infinite ease-in-out}.baguetteBox-double-bounce2{-webkit-animation-delay:-1s;animation-delay:-1s}@-webkit-keyframes bounce{0%,100%{-webkit-transform:scale(0);transform:scale(0)}50%{-webkit-transform:scale(1);transform:scale(1)}}@keyframes bounce{0%,100%{-webkit-transform:scale(0);-moz-transform:scale(0);transform:scale(0)}50%{-webkit-transform:scale(1);-moz-transform:scale(1);transform:scale(1)}} \ No newline at end of file diff --git a/public/css/themes/catppuccin_frappe.css b/public/css/themes/catppuccin_frappe.css new file mode 100644 index 0000000..23d951e --- /dev/null +++ b/public/css/themes/catppuccin_frappe.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Frappe color palette */ + --rosewater: #f2d5cf; + --flamingo: #eebebe; + --pink: #f4b8e4; + --mauve: #ca9ee6; + --red: #e78284; + --maroon: #ea999c; + --peach: #ef9f76; + --yellow: #e5c890; + --green: #a6d189; + --teal: #81c8be; + --sky: #99d1db; + --sapphire: #85c1dc; + --blue: #8caaee; + --lavender: #babbf1; + --text: #c6d0f5; + --subtext1: #b5bfe2; + --subtext0: #a5adce; + --overlay2: #949cbb; + --overlay1: #838ba7; + --overlay0: #737994; + --surface2: #626880; + --surface1: #51576d; + --surface0: #414559; + --base: #303446; + --mantle: #292c3c; + --crust: #232634; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} diff --git a/public/css/themes/catppuccin_latte.css b/public/css/themes/catppuccin_latte.css new file mode 100644 index 0000000..be9101c --- /dev/null +++ b/public/css/themes/catppuccin_latte.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Latte color palette */ + --rosewater: #dc8a78; + --flamingo: #dd7878; + --pink: #ea76cb; + --mauve: #8839ef; + --red: #d20f39; + --maroon: #e64553; + --peach: #fe640b; + --yellow: #df8e1d; + --green: #40a02b; + --teal: #179299; + --sky: #04a5e5; + --sapphire: #209fb5; + --blue: #1e66f5; + --lavender: #7287fd; + --text: #4c4f69; + --subtext1: #5c5f77; + --subtext0: #6c6f85; + --overlay2: #7c7f93; + --overlay1: #8c8fa1; + --overlay0: #9ca0b0; + --surface2: #acb0be; + --surface1: #bcc0cc; + --surface0: #ccd0da; + --base: #eff1f5; + --mantle: #e6e9ef; + --crust: #dce0e8; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} diff --git a/public/css/themes/catppuccin_macchiato.css b/public/css/themes/catppuccin_macchiato.css new file mode 100644 index 0000000..c1cb1bb --- /dev/null +++ b/public/css/themes/catppuccin_macchiato.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Macchiato color palette */ + --rosewater: #f4dbd6; + --flamingo: #f0c6c6; + --pink: #f5bde6; + --mauve: #c6a0f6; + --red: #ed8796; + --maroon: #ee99a0; + --peach: #f5a97f; + --yellow: #eed49f; + --green: #a6da95; + --teal: #8bd5ca; + --sky: #91d7e3; + --sapphire: #7dc4e4; + --blue: #8aadf4; + --lavender: #b7bdf8; + --text: #cad3f5; + --subtext1: #b8c0e0; + --subtext0: #a5adcb; + --overlay2: #939ab7; + --overlay1: #8087a2; + --overlay0: #6e738d; + --surface2: #5b6078; + --surface1: #494d64; + --surface0: #363a4f; + --base: #24273a; + --mantle: #1e2030; + --crust: #181926; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} diff --git a/public/css/themes/catppuccin_mocha.css b/public/css/themes/catppuccin_mocha.css new file mode 100644 index 0000000..c2bfa87 --- /dev/null +++ b/public/css/themes/catppuccin_mocha.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Mocha color palette */ + --rosewater: #f5e0dc; + --flamingo: #f2cdcd; + --pink: #f5c2e7; + --mauve: #cba6f7; + --red: #f38ba8; + --maroon: #eba0ac; + --peach: #fab387; + --yellow: #f9e2af; + --green: #a6e3a1; + --teal: #94e2d5; + --sky: #89dceb; + --sapphire: #74c7ec; + --blue: #89b4fa; + --lavender: #b4befe; + --text: #cdd6f4; + --subtext1: #bac2de; + --subtext0: #a6adc8; + --overlay2: #9399b2; + --overlay1: #7f849c; + --overlay0: #6c7086; + --surface2: #585b70; + --surface1: #45475a; + --surface0: #313244; + --base: #1e1e2e; + --mantle: #181825; + --crust: #11111b; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} diff --git a/public/css/themes/tokyonight.css b/public/css/themes/tokyonight.css new file mode 100644 index 0000000..c5df1cf --- /dev/null +++ b/public/css/themes/tokyonight.css @@ -0,0 +1,81 @@ +body { + /* Catppuccin Mocha color palette */ + --rosewater: #f5e0dc; + --flamingo: #f2cdcd; + --pink: #f5c2e7; + --mauve: #cba6f7; + --red: #f38ba8; + --maroon: #eba0ac; + --peach: #fab387; + --yellow: #f9e2af; + --green: #a6e3a1; + --teal: #94e2d5; + --sky: #89dceb; + --sapphire: #74c7ec; + --blue: #89b4fa; + --lavender: #b4befe; + --text: #cdd6f4; + --subtext1: #bac2de; + --subtext0: #a6adc8; + --overlay2: #9399b2; + --overlay1: #7f849c; + --overlay0: #6c7086; + --surface2: #585b70; + --surface1: #45475a; + --surface0: #313244; + --base: #1e1e2e; + + /* Predefined colors */ + --bg_color: #1a1b26; + --fg_color: #c0caf5; + --fg_faded: #a9b1d6; + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + --bg_panel: #171722; + --bg_elements: #16161e; + --bg_overlays: #171722; + --bg_hover: #16161e; + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: #171722; + --border_grey: var(--base); + + --accent: #7aa2f7; + --accent_light: #7aa2f7; + --accent_dark: #7aa2f7; + --accent_border: #7aa2f7; + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: #cba6f7; + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } + + ::selection { + background-color: #515c7e40; + color: #a9b1d6; + } +} diff --git a/public/js/baguetteBox.min.js b/public/js/baguetteBox.min.js new file mode 100644 index 0000000..a6c7b04 --- /dev/null +++ b/public/js/baguetteBox.min.js @@ -0,0 +1,7 @@ +/*! + * baguetteBox.js + * @author feimosi + * @version 1.11.1 + * @url https://github.com/feimosi/baguetteBox.js + */ +!function(e,t){"use strict";"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():e.baguetteBox=t()}(this,function(){"use strict";var r,l,u,c,d,f='',g='',p='',b={},v={captions:!0,buttons:"auto",fullScreen:!1,noScrollbars:!1,bodyClass:"baguetteBox-open",titleTag:!1,async:!1,preload:2,animation:"slideIn",afterShow:null,afterHide:null,onChange:null,overlayBackgroundColor:"rgba(0,0,0,.8)"},m={},h=[],o=0,n=!1,i={},a=!1,y=/.+\.(gif|jpe?g|png|webp)/i,w={},k=[],s=null,x=function(e){-1!==e.target.id.indexOf("baguette-img")&&j()},E=function(e){e.stopPropagation?e.stopPropagation():e.cancelBubble=!0,D()},C=function(e){e.stopPropagation?e.stopPropagation():e.cancelBubble=!0,X()},B=function(e){e.stopPropagation?e.stopPropagation():e.cancelBubble=!0,j()},T=function(e){i.count++,1