From 984d4362a5ce5a45ba903383a0104191f2e1963c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 11 Feb 2025 12:51:58 +1300 Subject: [PATCH] Remove node-fetch --- package-lock.json | 91 -------------------------- package.json | 2 - readme.md | 3 +- scripts/setup.js | 1 - src/d2m/actions/register-pk-user.js | 3 +- src/m2d/actions/emoji-sheet.js | 18 +++-- src/m2d/actions/send-event.js | 1 - src/m2d/converters/emoji-sheet.js | 2 +- src/matrix/api.js | 4 +- src/matrix/file.js | 8 ++- src/matrix/mreq.js | 16 +++-- src/web/routes/download-matrix.test.js | 3 +- src/web/routes/oauth.js | 1 - test/test.js | 8 +-- 14 files changed, 30 insertions(+), 131 deletions(-) diff --git a/package-lock.json b/package-lock.json index db61cd0..d7bddfd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,7 +33,6 @@ "heatsync": "^2.6.0", "lru-cache": "^10.4.3", "minimist": "^1.2.8", - "node-fetch": "^2.6.7", "prettier-bytes": "^1.0.4", "sharp": "^0.33.4", "snowtransfer": "^0.10.5", @@ -46,7 +45,6 @@ "devDependencies": { "@cloudrac3r/tap-dot": "^2.0.3", "@types/node": "^18.16.0", - "@types/node-fetch": "^2.6.3", "c8": "^10.1.2", "cross-env": "^7.0.3", "discord-api-types": "^0.37.60", @@ -1083,16 +1081,6 @@ "undici-types": "~5.26.4" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.11", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", - "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", - "dev": true, - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.0" - } - }, "node_modules/@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -1173,12 +1161,6 @@ "resolved": "https://registry.npmjs.org/assert-never/-/assert-never-1.3.0.tgz", "integrity": "sha512-9Z3vxQ+berkL/JJo0dK+EY3Lp0s3NtSnP3VCLsh5HDcZPrh0M+KQRK5sWhUeyPPH+/RCxZqOxLMR+YC6vlviEQ==" }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true - }, "node_modules/babel-walk": { "version": "3.0.0-canary-5", "resolved": "https://registry.npmjs.org/babel-walk/-/babel-walk-3.0.0-canary-5.tgz", @@ -1478,18 +1460,6 @@ "simple-swizzle": "^0.2.2" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/consola": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/consola/-/consola-3.2.3.tgz", @@ -1605,15 +1575,6 @@ "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==", "license": "MIT" }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/destr": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.3.tgz", @@ -1801,20 +1762,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -2256,25 +2203,6 @@ "node": ">=10" } }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-fetch-native": { "version": "1.6.4", "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.4.tgz", @@ -3168,11 +3096,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/try-catch": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/try-catch/-/try-catch-3.0.1.tgz", @@ -3281,20 +3204,6 @@ "node": ">=0.10.0" } }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index fde5aa2..24555a3 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,6 @@ "heatsync": "^2.6.0", "lru-cache": "^10.4.3", "minimist": "^1.2.8", - "node-fetch": "^2.6.7", "prettier-bytes": "^1.0.4", "sharp": "^0.33.4", "snowtransfer": "^0.10.5", @@ -55,7 +54,6 @@ "devDependencies": { "@cloudrac3r/tap-dot": "^2.0.3", "@types/node": "^18.16.0", - "@types/node-fetch": "^2.6.3", "c8": "^10.1.2", "cross-env": "^7.0.3", "discord-api-types": "^0.37.60", diff --git a/readme.md b/readme.md index 8ab6bb4..2fa64a4 100644 --- a/readme.md +++ b/readme.md @@ -164,7 +164,7 @@ To get into the rooms on your Matrix account, use the `/invite [your mxid here]` ## Dependency justification -Total transitive production dependencies: 148 +Total transitive production dependencies: 144 ### 🦕 @@ -197,7 +197,6 @@ Total transitive production dependencies: 148 * (1) js-yaml: Will be removed in the future after registration.yaml is converted to JSON. * (0) lru-cache: For holding unused nonce in memory and letting them be overwritten later if never used. * (0) minimist: It's already pulled in by better-sqlite3->prebuild-install. -* (3) node-fetch@2: I like it and it does what I want. Version 2 is used because version 3 is ESM-only. * (0) prettier-bytes: It does what I want and has no dependencies. * (2) snowtransfer: Discord API library with bring-your-own-caching that I trust. * (0) try-to-catch: Not strictly necessary, but it's already pulled in by supertape, so I may as well. diff --git a/scripts/setup.js b/scripts/setup.js index 6ee8c19..1174861 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -11,7 +11,6 @@ const {join} = require("path") const {prompt} = require("enquirer") const Input = require("enquirer/lib/prompts/input") -const fetch = require("node-fetch").default const {magenta, bold, cyan} = require("ansi-colors") const HeatSync = require("heatsync") const {SnowTransfer} = require("snowtransfer") diff --git a/src/d2m/actions/register-pk-user.js b/src/d2m/actions/register-pk-user.js index ce1665c..cfd51af 100644 --- a/src/d2m/actions/register-pk-user.js +++ b/src/d2m/actions/register-pk-user.js @@ -3,10 +3,9 @@ const assert = require("assert") const {reg} = require("../../matrix/read-registration") const Ty = require("../../types") -const fetch = require("node-fetch").default const passthrough = require("../../passthrough") -const {discord, sync, db, select} = passthrough +const {sync, db, select} = passthrough /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** @type {import("../../matrix/file")} */ diff --git a/src/m2d/actions/emoji-sheet.js b/src/m2d/actions/emoji-sheet.js index c81960d..a7a4498 100644 --- a/src/m2d/actions/emoji-sheet.js +++ b/src/m2d/actions/emoji-sheet.js @@ -1,10 +1,8 @@ // @ts-check -const assert = require("assert") -const fetch = require("node-fetch").default - -const utils = require("../converters/utils") +const {Readable} = require("stream") const {sync} = require("../../passthrough") +const assert = require("assert").strict /** @type {import("../converters/emoji-sheet")} */ const emojiSheetConverter = sync.require("../converters/emoji-sheet") @@ -18,16 +16,16 @@ const api = sync.require("../../matrix/api") */ async function getAndConvertEmoji(mxc) { const abortController = new AbortController() - /** @type {import("node-fetch").Response} */ // If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing. // If we were using connection pooling, we would be forced to download the entire GIF. // So we set no agent to ensure we are not connection pooling. - // @ts-ignore the signal is slightly different from the type it wants (still works fine) - const res = await api.getMedia(mxc, {agent: false, signal: abortController.signal}) - return emojiSheetConverter.convertImageStream(res.body, () => { + const res = await api.getMedia(mxc, {signal: abortController.signal}) + // @ts-ignore + const readable = Readable.fromWeb(res.body) + return emojiSheetConverter.convertImageStream(readable, () => { abortController.abort() - res.body.pause() - res.body.emit("end") + readable.emit("end") + readable.on("error", () => {}) // DOMException [AbortError]: This operation was aborted }) } diff --git a/src/m2d/actions/send-event.js b/src/m2d/actions/send-event.js index 35fcfda..66c2728 100644 --- a/src/m2d/actions/send-event.js +++ b/src/m2d/actions/send-event.js @@ -5,7 +5,6 @@ const DiscordTypes = require("discord-api-types/v10") const {Readable} = require("stream") const assert = require("assert").strict const crypto = require("crypto") -const fetch = require("node-fetch").default const passthrough = require("../../passthrough") const {sync, discord, db, select} = passthrough diff --git a/src/m2d/converters/emoji-sheet.js b/src/m2d/converters/emoji-sheet.js index db5b06f..17098e0 100644 --- a/src/m2d/converters/emoji-sheet.js +++ b/src/m2d/converters/emoji-sheet.js @@ -48,7 +48,7 @@ async function compositeMatrixEmojis(mxcs, mxcDownloader) { } /** - * @param {import("node-fetch").Response["body"]} streamIn + * @param {NodeJS.ReadableStream} streamIn * @param {() => any} stopStream * @returns {Promise} Uncompressed PNG image */ diff --git a/src/matrix/api.js b/src/matrix/api.js index 13295e0..cea4307 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -3,8 +3,6 @@ const Ty = require("../types") const assert = require("assert").strict -const fetch = require("node-fetch").default - const passthrough = require("../passthrough") const {sync} = passthrough /** @type {import("./mreq")} */ @@ -344,7 +342,7 @@ async function ping() { /** * @param {string} mxc - * @param {fetch.RequestInit} [init] + * @param {RequestInit} [init] */ function getMedia(mxc, init = {}) { const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) diff --git a/src/matrix/file.js b/src/matrix/file.js index f0ee29a..6eb75e0 100644 --- a/src/matrix/file.js +++ b/src/matrix/file.js @@ -1,7 +1,5 @@ // @ts-check -const fetch = require("node-fetch").default - const passthrough = require("../passthrough") const {sync, db, select} = passthrough /** @type {import("./mreq")} */ @@ -47,7 +45,7 @@ async function uploadDiscordFileToMxc(path) { } // Download from Discord - const promise = fetch(url, {}).then(/** @param {import("node-fetch").Response} res */ async res => { + const promise = fetch(url, {}).then(async res => { // Upload to Matrix const root = await module.exports._actuallyUploadDiscordFileToMxc(urlNoExpiry, res) @@ -62,6 +60,10 @@ async function uploadDiscordFileToMxc(path) { return promise } +/** + * @param {string} url + * @param {Response} res + */ async function _actuallyUploadDiscordFileToMxc(url, res) { const body = res.body /** @type {import("../types").R.FileUploaded} */ diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index 8a76b11..feb13b2 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -1,6 +1,5 @@ // @ts-check -const fetch = require("node-fetch").default const mixin = require("@cloudrac3r/mixin-deep") const stream = require("stream") const getStream = require("get-stream") @@ -22,7 +21,7 @@ class MatrixServerError extends Error { /** * @param {string} method * @param {string} url - * @param {any} [body] + * @param {string | object | ReadableStream | stream.Readable} [body] * @param {any} [extra] */ async function mreq(method, url, body, extra = {}) { @@ -30,16 +29,19 @@ async function mreq(method, url, body, extra = {}) { body = JSON.stringify(body) } else if (body instanceof stream.Readable && reg.ooye.content_length_workaround) { body = await getStream.buffer(body) + } else if (body instanceof ReadableStream && reg.ooye.content_length_workaround) { + body = await stream.consumers.buffer(body) } - const opts = mixin({ + /** @type {RequestInit} */ + const opts = { method, body, headers: { Authorization: `Bearer ${reg.as_token}` - } - }, extra) - + }, + ...extra + } // console.log(baseUrl + url, opts) const res = await fetch(baseUrl + url, opts) const root = await res.json() @@ -55,7 +57,7 @@ async function mreq(method, url, body, extra = {}) { writeRegistration(reg) return root } - delete opts.headers.Authorization + delete opts.headers?.["Authorization"] throw new MatrixServerError(root, {baseUrl, url, ...opts}) } return root diff --git a/src/web/routes/download-matrix.test.js b/src/web/routes/download-matrix.test.js index d44271a..421d2da 100644 --- a/src/web/routes/download-matrix.test.js +++ b/src/web/routes/download-matrix.test.js @@ -3,7 +3,6 @@ const tryToCatch = require("try-to-catch") const {test} = require("supertape") const {router} = require("../../../test/web") -const fetch = require("node-fetch") test("web download matrix: access denied if not a known attachment", async t => { const [error] = await tryToCatch(() => @@ -27,7 +26,7 @@ test("web download matrix: works if a known attachment", async t => { event, api: { async getMedia(mxc, init) { - return new fetch.Response("", {status: 200, headers: {"content-type": "image/png"}}) + return new Response("", {status: 200, headers: {"content-type": "image/png"}}) } } }) diff --git a/src/web/routes/oauth.js b/src/web/routes/oauth.js index a6e31f0..ed0da8a 100644 --- a/src/web/routes/oauth.js +++ b/src/web/routes/oauth.js @@ -5,7 +5,6 @@ const {randomUUID} = require("crypto") const {defineEventHandler, getValidatedQuery, sendRedirect, createError} = require("h3") const {SnowTransfer} = require("snowtransfer") const DiscordTypes = require("discord-api-types/v10") -const fetch = require("node-fetch") const getRelativePath = require("get-relative-path") const {as, db, sync} = require("../../passthrough") diff --git a/test/test.js b/test/test.js index bc226a3..5d771a8 100644 --- a/test/test.js +++ b/test/test.js @@ -2,15 +2,12 @@ const fs = require("fs") const {join} = require("path") -const stp = require("stream").promises const sqlite = require("better-sqlite3") +const {Writable} = require("stream") const migrate = require("../src/db/migrate") const HeatSync = require("heatsync") const {test, extend} = require("supertape") const data = require("./data") -/** @type {import("node-fetch").default} */ -// @ts-ignore -const fetch = require("node-fetch") const {green} = require("ansi-colors") const passthrough = require("../src/passthrough") @@ -86,7 +83,8 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not async function download({url, to}) { if (await fs.existsSync(to)) return const res = await fetch(url) - await stp.pipeline(res.body, fs.createWriteStream(to, {encoding: "binary"})) + // @ts-ignore + await res.body.pipeTo(Writable.toWeb(fs.createWriteStream(to, {encoding: "binary"}))) } await allReporter([ {url: "https://cadence.moe/friends/ooye_test/RLMgJGfgTPjIQtvvWZsYjhjy.png", to: "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png"},