From 845f93e5d05b67af1c28bf8acddb3d8b278f7079 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 17:03:03 +1300 Subject: [PATCH 1/4] Work around the Content-Length issue --- matrix/mreq.js | 15 +++++++++++++-- package-lock.json | 12 ++++++++++++ package.json | 1 + readme.md | 1 + registration.example.yaml | 1 + scripts/seed.js | 12 +++++++++++- types.d.ts | 1 + 7 files changed, 40 insertions(+), 3 deletions(-) diff --git a/matrix/mreq.js b/matrix/mreq.js index 0bd7505..70d9ac7 100644 --- a/matrix/mreq.js +++ b/matrix/mreq.js @@ -2,6 +2,8 @@ const fetch = require("node-fetch").default const mixin = require("mixin-deep") +const stream = require("stream") +const getStream = require("get-stream") const passthrough = require("../passthrough") const { sync } = passthrough @@ -27,9 +29,15 @@ class MatrixServerError extends Error { * @param {any} [extra] */ async function mreq(method, url, body, extra = {}) { + if (body == undefined || Object.is(body.constructor, Object)) { + body = JSON.stringify(body) + } else if (body instanceof stream.Readable && reg.ooye.content_length_workaround) { + body = await getStream.buffer(body) + } + const opts = mixin({ method, - body: (body == undefined || Object.is(body.constructor, Object)) ? JSON.stringify(body) : body, + body, headers: { Authorization: `Bearer ${reg.as_token}` } @@ -39,7 +47,10 @@ async function mreq(method, url, body, extra = {}) { const res = await fetch(baseUrl + url, opts) const root = await res.json() - if (!res.ok || root.errcode) throw new MatrixServerError(root, {baseUrl, url, ...opts}) + if (!res.ok || root.errcode) { + delete opts.headers.Authorization + throw new MatrixServerError(root, {baseUrl, url, ...opts}) + } return root } diff --git a/package-lock.json b/package-lock.json index 054e701..4fb1269 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", + "get-stream": "^6.0.1", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0", @@ -1456,6 +1457,17 @@ "source-map": "^0.6.1" } }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/giframe": { "version": "0.3.0", "resolved": "git+ssh://git@github.com/cloudrac3r/giframe.git#1630f4d3b2bf5acd197409c85edd11e0da72d0a1", diff --git a/package.json b/package.json index 58a8674..4aba845 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", + "get-stream": "^6.0.1", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0", diff --git a/readme.md b/readme.md index e7baf18..df42234 100644 --- a/readme.md +++ b/readme.md @@ -164,6 +164,7 @@ Follow these steps: * (8) snowtransfer: Discord API library with bring-your-own-caching that I trust. * (0) deep-equal: It's already pulled in by supertape. * (1) discord-markdown: This is my fork! +* (0) get-stream: Only needed if content_length_workaround is true. * (0) giframe: This is my fork! * (1) heatsync: Module hot-reloader that I trust. * (0) entities: Looks fine. No dependencies. diff --git a/registration.example.yaml b/registration.example.yaml index 9e7cd2c..9a562cd 100644 --- a/registration.example.yaml +++ b/registration.example.yaml @@ -18,6 +18,7 @@ ooye: max_file_size: 5000000 server_name: [the part after the colon in your matrix id, like cadence.moe] server_origin: [the full protocol and domain of your actual matrix server's location, with no trailing slash, like https://matrix.cadence.moe] + content_length_workaround: false invite: # uncomment this to auto-invite the named user to newly created spaces and mark them as admin (PL 100) everywhere # - @cadence:cadence.moe diff --git a/scripts/seed.js b/scripts/seed.js index 4dc08e0..20cb5dd 100644 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -71,7 +71,17 @@ async function uploadAutoEmoji(guild, name, filename) { try { await api.register(reg.sender_localpart) } catch (e) { - if (e.errcode === "M_USER_IN_USE" || e.data?.error !== "Internal server error") throw e // "Internal server error" is the only OK error because Synapse says this if you try to register the same username twice. + if (e.errcode === "M_USER_IN_USE" || e.data?.error === "Internal server error") { + // "Internal server error" is the only OK error because older versions of Synapse say this if you try to register the same username twice. + } else if (e.data?.error?.includes("Content-Length")) { + die(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:` + + `\n * Run an nginx reverse proxy to Synapse, and point registration.yaml's` + + `\n \`server_origin\` to nginx` + + `\n * Set \`content_length_workaround: true\` in registration.yaml (this will` + + `\n halve the speed of bridging d->m files)`) + } else { + throw e + } } // upload initial images... diff --git a/types.d.ts b/types.d.ts index 9e9d72b..477b41d 100644 --- a/types.d.ts +++ b/types.d.ts @@ -21,6 +21,7 @@ export type AppServiceRegistrationConfig = { max_file_size: number server_name: string server_origin: string + content_length_workaround: boolean invite: string[] } old_bridge?: { From 706b37669b59835f678d431bc28372f40005c344 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 17:25:58 +1300 Subject: [PATCH 2/4] Move Content-Length detection logic --- matrix/mreq.js | 8 ++++++++ scripts/seed.js | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/matrix/mreq.js b/matrix/mreq.js index 70d9ac7..d515846 100644 --- a/matrix/mreq.js +++ b/matrix/mreq.js @@ -48,6 +48,14 @@ async function mreq(method, url, body, extra = {}) { const root = await res.json() if (!res.ok || root.errcode) { + if (root.error?.includes("Content-Length")) { + console.error(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:` + + `\n * Run an nginx reverse proxy to Synapse, and point registration.yaml's` + + `\n \`server_origin\` to nginx` + + `\n * Set \`content_length_workaround: true\` in registration.yaml (this will` + + `\n halve the speed of bridging d->m files)`) + throw new Error("Synapse is not accepting stream uploads, see the message above.") + } delete opts.headers.Authorization throw new MatrixServerError(root, {baseUrl, url, ...opts}) } diff --git a/scripts/seed.js b/scripts/seed.js index 20cb5dd..1a21490 100644 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -73,12 +73,6 @@ async function uploadAutoEmoji(guild, name, filename) { } catch (e) { if (e.errcode === "M_USER_IN_USE" || e.data?.error === "Internal server error") { // "Internal server error" is the only OK error because older versions of Synapse say this if you try to register the same username twice. - } else if (e.data?.error?.includes("Content-Length")) { - die(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:` - + `\n * Run an nginx reverse proxy to Synapse, and point registration.yaml's` - + `\n \`server_origin\` to nginx` - + `\n * Set \`content_length_workaround: true\` in registration.yaml (this will` - + `\n halve the speed of bridging d->m files)`) } else { throw e } From 67dc31f747140c8ea64e5fc796d1bc6674270358 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 18:23:48 +1300 Subject: [PATCH 3/4] Fix tests that hard-coded cadence.moe --- test/test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test.js b/test/test.js index 4663b18..a57231d 100644 --- a/test/test.js +++ b/test/test.js @@ -14,6 +14,7 @@ const db = new sqlite(":memory:") const reg = require("../matrix/read-registration") reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded +reg.ooye.server_name = "cadence.moe" reg.ooye.invite = ["@test_auto_invite:example.org"] const sync = new HeatSync({watchFS: false}) From 8591ea5c1f67a5fa6ba615f9f731d3605171355c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 23:13:39 +1300 Subject: [PATCH 4/4] Explain how to get in the rooms --- readme.md | 4 ++++ registration.example.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index df42234..05d8f9c 100644 --- a/readme.md +++ b/readme.md @@ -96,6 +96,10 @@ Follow these steps: * $ `npm run addbot` * $ `./addbot.sh` +Now any message on Discord will create the corresponding rooms on Matrix-side. After the rooms have been created, Matrix and Discord users can chat back and forth. + +To get into the rooms on your Matrix account, either add yourself to `invite` in `registration.yaml`, or use the `//invite [your mxid here]` command on Discord. + # Development setup * Be sure to install dependencies with `--save-dev` so you can run the tests. diff --git a/registration.example.yaml b/registration.example.yaml index 9a562cd..dd217db 100644 --- a/registration.example.yaml +++ b/registration.example.yaml @@ -21,4 +21,4 @@ ooye: content_length_workaround: false invite: # uncomment this to auto-invite the named user to newly created spaces and mark them as admin (PL 100) everywhere - # - @cadence:cadence.moe + # - '@cadence:cadence.moe'