From 411491b405c92c7817ee552745429f6e59d16958 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 21 Feb 2026 12:04:42 +1300 Subject: [PATCH 1/7] Remove live dependency on cadence.moe --- scripts/setup.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/setup.js b/scripts/setup.js index 4e6de0a..69b62a2 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -1,6 +1,7 @@ #!/usr/bin/env node // @ts-check +const Ty = require("../src/types") const assert = require("assert").strict const fs = require("fs") const sqlite = require("better-sqlite3") @@ -285,8 +286,8 @@ function defineEchoHandler() { console.log() // Done with user prompts, reg is now guaranteed to be valid + const mreq = require("../src/matrix/mreq") const api = require("../src/matrix/api") - const file = require("../src/matrix/file") const DiscordClient = require("../src/d2m/discord-client") const discord = new DiscordClient(reg.ooye.discord_token, "no") passthrough.discord = discord @@ -343,7 +344,13 @@ function defineEchoHandler() { await api.register(reg.sender_localpart) // upload initial images... - const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png") + const avatarBuffer = await fs.promises.readFile(join(__dirname, "..", "docs", "img", "icon.png"), null) + /** @type {Ty.R.FileUploaded} */ + const root = await mreq.mreq("POST", "/media/v3/upload", avatarBuffer, { + headers: {"Content-Type": "image/png"} + }) + const avatarUrl = root.content_uri + assert(avatarUrl) console.log("✅ Matrix appservice login works...") @@ -352,8 +359,7 @@ function defineEchoHandler() { console.log("✅ Emojis are ready...") // set profile data on discord... - const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer()) - await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")}) + await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + avatarBuffer.toString("base64")}) console.log("✅ Discord profile updated...") // set profile data on homeserver... From 9b3707baa1dc7f9308f3152068d69f0678d16d30 Mon Sep 17 00:00:00 2001 From: Abdul <32655037-CamperThumper@users.noreply.gitlab.com> Date: Wed, 25 Feb 2026 01:03:30 +0300 Subject: [PATCH 2/7] Link sticker instead of file upload --- src/m2d/actions/sticker.js | 53 ++++++++++++++++++++++++++ src/m2d/converters/event-to-message.js | 25 +++++------- src/web/routes/download-matrix.js | 19 +++++++++ 3 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 src/m2d/actions/sticker.js diff --git a/src/m2d/actions/sticker.js b/src/m2d/actions/sticker.js new file mode 100644 index 0000000..f9c06bf --- /dev/null +++ b/src/m2d/actions/sticker.js @@ -0,0 +1,53 @@ +// @ts-check + +const streamr = require("stream") +const {pipeline} = require("stream").promises + +const {sync} = require("../../passthrough") +const sharp = require("sharp") +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/mreq")} */ +const mreq = sync.require("../../matrix/mreq") +const streamMimeType = require("stream-mime-type") + +const WIDTH = 160 +const HEIGHT = 160 +/** + * Downloads the sticker from the web and converts to webp data. + * @param {string} mxc a single mxc:// URL + * @returns {Promise} sticker webp data, or undefined if the downloaded sticker is not valid + */ +async function getAndResizeSticker(mxc) { + const res = await api.getMedia(mxc) + if (res.status !== 200) { + const root = await res.json() + throw new mreq.MatrixServerError(root, {mxc}) + } + const streamIn = streamr.Readable.fromWeb(res.body) + + const { stream, mime } = await streamMimeType.getMimeType(streamIn) + let animated = false + if (mime === "image/gif" || mime === "image/webp") { + animated = true + } + + const result = await new Promise((resolve, reject) => { + const transformer = sharp({animated: animated}) + .resize(WIDTH, HEIGHT, {fit: "inside", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .webp() + .toBuffer((err, buffer, info) => { + /* c8 ignore next */ + if (err) return reject(err) + resolve({info, buffer}) + }) + pipeline( + stream, + transformer + ) + }) + return result.buffer +} + + +module.exports.getAndResizeSticker = getAndResizeSticker diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 2add279..91c2400 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -632,23 +632,16 @@ async function eventToMessage(event, guild, channel, di) { if (event.type === "m.sticker") { content = "" - let filename = event.content.body - if (event.type === "m.sticker") { - let mimetype - if (event.content.info?.mimetype?.includes("/")) { - mimetype = event.content.info.mimetype - } else { - const res = await di.api.getMedia(event.content.url, {method: "HEAD"}) - if (res.status === 200) { - mimetype = res.headers.get("content-type") - } - if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`) - } - filename += "." + mimetype.split("/")[1] - } - attachments.push({id: "0", filename}) - pendingFiles.push({name: filename, mxc: event.content.url}) + content += `[${event.content.body}](` // sticker title for fallback if the url preview fails + const afterLink = ")" + // Make sticker URL params + const params = new URLSearchParams() + const withoutMxc = mxUtils.makeMxcPublic(event.content.url) + assert(withoutMxc) + params.append("mxc", withoutMxc) + const url = `${reg.ooye.bridge_origin}/download/sticker.webp?${params.toString()}` + content += url + afterLink } else if (event.type === "org.matrix.msc3381.poll.start") { const pollContent = event.content["org.matrix.msc3381.poll.start"] // just for convenience const isClosed = false; diff --git a/src/web/routes/download-matrix.js b/src/web/routes/download-matrix.js index bb6b850..1fbc1d6 100644 --- a/src/web/routes/download-matrix.js +++ b/src/web/routes/download-matrix.js @@ -16,6 +16,9 @@ const emojiSheet = sync.require("../../m2d/actions/emoji-sheet") /** @type {import("../../m2d/converters/emoji-sheet")} */ const emojiSheetConverter = sync.require("../../m2d/converters/emoji-sheet") +/** @type {import("../../m2d/actions/sticker")} */ +const sticker = sync.require("../../m2d/actions/sticker") + const schema = { params: z.object({ server_name: z.string(), @@ -23,6 +26,9 @@ const schema = { }), sheet: z.object({ e: z.array(z.string()).or(z.string()) + }), + sticker: z.object({ + mxc: z.string() }) } @@ -90,3 +96,16 @@ as.router.get(`/download/sheet`, defineEventHandler(async event => { setResponseHeader(event, "Content-Type", "image/png") return buffer })) + +as.router.get(`/download/sticker.webp`, defineEventHandler(async event => { + const query = await getValidatedQuery(event, schema.sticker.parse) + + /** remember that these have no mxc:// protocol in the string */ + verifyMediaHash(query.mxc) + const mxc = `mxc://${query.mxc}` + + setResponseHeader(event, "Content-Type", 'image/webp') + const buffer = await sticker.getAndResizeSticker(mxc) + return buffer +})) + From d1aa8f01e70e78d7ae438059872d4f7ef52bff1a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 25 Feb 2026 18:21:35 +1300 Subject: [PATCH 3/7] Change sticker URL and stream response --- src/m2d/actions/sticker.js | 33 ++++++++------------------ src/m2d/converters/event-to-message.js | 12 +++------- src/web/routes/download-matrix.js | 21 ++++++++-------- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/src/m2d/actions/sticker.js b/src/m2d/actions/sticker.js index f9c06bf..341d8b0 100644 --- a/src/m2d/actions/sticker.js +++ b/src/m2d/actions/sticker.js @@ -1,7 +1,7 @@ // @ts-check -const streamr = require("stream") -const {pipeline} = require("stream").promises +const {Readable} = require("stream") +const {ReadableStream} = require("stream/web") const {sync} = require("../../passthrough") const sharp = require("sharp") @@ -16,7 +16,7 @@ const HEIGHT = 160 /** * Downloads the sticker from the web and converts to webp data. * @param {string} mxc a single mxc:// URL - * @returns {Promise} sticker webp data, or undefined if the downloaded sticker is not valid + * @returns {Promise} sticker webp data, or undefined if the downloaded sticker is not valid */ async function getAndResizeSticker(mxc) { const res = await api.getMedia(mxc) @@ -24,29 +24,16 @@ async function getAndResizeSticker(mxc) { const root = await res.json() throw new mreq.MatrixServerError(root, {mxc}) } - const streamIn = streamr.Readable.fromWeb(res.body) + const streamIn = Readable.fromWeb(res.body) const { stream, mime } = await streamMimeType.getMimeType(streamIn) - let animated = false - if (mime === "image/gif" || mime === "image/webp") { - animated = true - } + const animated = ["image/gif", "image/webp"].includes(mime) - const result = await new Promise((resolve, reject) => { - const transformer = sharp({animated: animated}) - .resize(WIDTH, HEIGHT, {fit: "inside", background: {r: 0, g: 0, b: 0, alpha: 0}}) - .webp() - .toBuffer((err, buffer, info) => { - /* c8 ignore next */ - if (err) return reject(err) - resolve({info, buffer}) - }) - pipeline( - stream, - transformer - ) - }) - return result.buffer + const transformer = sharp({animated: animated}) + .resize(WIDTH, HEIGHT, {fit: "inside", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .webp() + stream.pipe(transformer) + return Readable.toWeb(transformer) } diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 91c2400..81ad48c 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -631,17 +631,11 @@ async function eventToMessage(event, guild, channel, di) { } if (event.type === "m.sticker") { - content = "" - content += `[${event.content.body}](` // sticker title for fallback if the url preview fails - const afterLink = ")" - - // Make sticker URL params - const params = new URLSearchParams() const withoutMxc = mxUtils.makeMxcPublic(event.content.url) assert(withoutMxc) - params.append("mxc", withoutMxc) - const url = `${reg.ooye.bridge_origin}/download/sticker.webp?${params.toString()}` - content += url + afterLink + const url = `${reg.ooye.bridge_origin}/download/sticker/${withoutMxc}/_.webp` + content = `[${event.content.body || "\u2800"}](${url})` + } else if (event.type === "org.matrix.msc3381.poll.start") { const pollContent = event.content["org.matrix.msc3381.poll.start"] // just for convenience const isClosed = false; diff --git a/src/web/routes/download-matrix.js b/src/web/routes/download-matrix.js index 1fbc1d6..82e2f7e 100644 --- a/src/web/routes/download-matrix.js +++ b/src/web/routes/download-matrix.js @@ -28,7 +28,8 @@ const schema = { e: z.array(z.string()).or(z.string()) }), sticker: z.object({ - mxc: z.string() + server_name: z.string().regex(/^[^/]+$/), + media_id: z.string().regex(/^[A-Za-z0-9_-]+$/) }) } @@ -97,15 +98,13 @@ as.router.get(`/download/sheet`, defineEventHandler(async event => { return buffer })) -as.router.get(`/download/sticker.webp`, defineEventHandler(async event => { - const query = await getValidatedQuery(event, schema.sticker.parse) +as.router.get(`/download/sticker/:server_name/:media_id/_.webp`, defineEventHandler(async event => { + const {server_name, media_id} = await getValidatedRouterParams(event, schema.sticker.parse) + /** remember that this has no mxc:// protocol in the string */ + const mxc = server_name + "/" + media_id + verifyMediaHash(mxc) - /** remember that these have no mxc:// protocol in the string */ - verifyMediaHash(query.mxc) - const mxc = `mxc://${query.mxc}` - - setResponseHeader(event, "Content-Type", 'image/webp') - const buffer = await sticker.getAndResizeSticker(mxc) - return buffer + const stream = await sticker.getAndResizeSticker(`mxc://${mxc}`) + setResponseHeader(event, "Content-Type", "image/webp") + return stream })) - From ea261e825b8b9a80ebae44d3048042af44df8e55 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 27 Feb 2026 18:33:29 +1300 Subject: [PATCH 4/7] Slashes not allowed in MXID --- src/d2m/converters/user-to-mxid.js | 6 +++--- src/d2m/converters/user-to-mxid.test.js | 8 ++++++-- src/db/migrations/0034-slash-not-allowed-in-mxid.sql | 5 +++++ 3 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 src/db/migrations/0034-slash-not-allowed-in-mxid.sql diff --git a/src/d2m/converters/user-to-mxid.js b/src/d2m/converters/user-to-mxid.js index 12891c0..7705aff 100644 --- a/src/d2m/converters/user-to-mxid.js +++ b/src/d2m/converters/user-to-mxid.js @@ -20,10 +20,10 @@ const SPECIAL_USER_MAPPINGS = new Map([ function downcaseUsername(user) { // First, try to convert the username to the set of allowed characters let downcased = user.username.toLowerCase() - // spaces to underscores... - .replace(/ /g, "_") + // spaces and slashes to underscores... + .replace(/[ /]/g, "_") // remove disallowed characters... - .replace(/[^a-z0-9._=/-]*/g, "") + .replace(/[^a-z0-9._=-]*/g, "") // remove leading and trailing dashes and underscores... .replace(/(?:^[_-]*|[_-]*$)/g, "") // If requested, also make the Discord user ID part of the username diff --git a/src/d2m/converters/user-to-mxid.test.js b/src/d2m/converters/user-to-mxid.test.js index 387d472..f8cf16a 100644 --- a/src/d2m/converters/user-to-mxid.test.js +++ b/src/d2m/converters/user-to-mxid.test.js @@ -21,8 +21,12 @@ test("user2name: works on single emoji at the end", t => { t.equal(userToSimName({username: "Melody 🎵", discriminator: "2192"}), "melody") }) -test("user2name: works on crazy name", t => { - t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7//") +test("user2name: works on really weird name", t => { + t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7") +}) + +test("user2name: treats slashes", t => { + t.equal(userToSimName({username: "Evil Lillith (she/her)", discriminator: "5892"}), "evil_lillith_she_her") }) test("user2name: adds discriminator if name is unavailable (old tag format)", t => { diff --git a/src/db/migrations/0034-slash-not-allowed-in-mxid.sql b/src/db/migrations/0034-slash-not-allowed-in-mxid.sql new file mode 100644 index 0000000..ea2d031 --- /dev/null +++ b/src/db/migrations/0034-slash-not-allowed-in-mxid.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +DELETE FROM sim WHERE sim_name like '%/%'; + +COMMIT; From 780154fd09beda223a21bf5d17d407d07b2c7192 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 27 Feb 2026 18:34:30 +1300 Subject: [PATCH 5/7] Bots with Administrator may access all channels --- src/web/routes/guild.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index dfb393b..a5508c4 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -115,7 +115,7 @@ function getChannelRoomsLinks(guild, rooms, roles) { let removedWrongTypeChannels = dUtils.filterTo(unlinkedChannels, c => c && [0, 5].includes(c.type)) let removedPrivateChannels = dUtils.filterTo(unlinkedChannels, c => { const permissions = dUtils.getPermissions(guild.id, roles, guild.roles, botID, c["permission_overwrites"]) - return dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) + return dUtils.hasSomePermissions(permissions, ["Administrator", "ViewChannel"]) }) unlinkedChannels.sort((a, b) => getPosition(a, discord.channels) - getPosition(b, discord.channels)) From e275d4c928b9bcbc7d3c855b7ce6900dae1e8686 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 27 Feb 2026 18:35:48 +1300 Subject: [PATCH 6/7] Add script to estimate total channel file size --- scripts/estimate-size.js | 65 ++++++++++++++++++++++++++++++++++++++++ src/matrix/api.js | 2 +- src/types.d.ts | 8 ++++- 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 scripts/estimate-size.js diff --git a/scripts/estimate-size.js b/scripts/estimate-size.js new file mode 100644 index 0000000..341abc0 --- /dev/null +++ b/scripts/estimate-size.js @@ -0,0 +1,65 @@ +// @ts-check + +const pb = require("prettier-bytes") +const sqlite = require("better-sqlite3") +const HeatSync = require("heatsync") + +const {reg} = require("../src/matrix/read-registration") +const passthrough = require("../src/passthrough") + +const sync = new HeatSync({watchFS: false}) +Object.assign(passthrough, {reg, sync}) + +const DiscordClient = require("../src/d2m/discord-client") + +const discord = new DiscordClient(reg.ooye.discord_token, "no") +passthrough.discord = discord + +const db = new sqlite("ooye.db") +passthrough.db = db + +const api = require("../src/matrix/api") + +const {room: roomID} = require("minimist")(process.argv.slice(2), {string: ["room"]}) +if (!roomID) { + console.error("Usage: ./scripts/estimate-size.js --room=") + process.exit(1) +} + +const {channel_id, guild_id} = db.prepare("SELECT channel_id, guild_id FROM channel_room WHERE room_id = ?").get(roomID) + +const max = 1000 + +;(async () => { + let total = 0 + let size = 0 + let from + + while (total < max) { + const events = await api.getEvents(roomID, "b", {limit: 1000, from}) + total += events.chunk.length + from = events.end + console.log(`Fetched ${total} events so far`) + + for (const e of events.chunk) { + if (e.content?.info?.size) { + size += e.content.info.size + } + } + + if (events.chunk.length === 0 || !events.end) break + } + + console.log(`Total size of uploads: ${pb(size)}`) + + const searchResults = await discord.snow.requestHandler.request(`/guilds/${guild_id}/messages/search`, { + channel_id, + offset: "0", + limit: "1" + }, "get", "json") + + const totalAllTime = searchResults.total_results + const fractionCounted = total / totalAllTime + console.log(`That counts for ${(fractionCounted*100).toFixed(2)}% of the history on Discord (${totalAllTime.toLocaleString()} messages)`) + console.log(`The size of uploads for the whole history would be approx: ${pb(Math.floor(size/total*totalAllTime))}`) +})() diff --git a/src/matrix/api.js b/src/matrix/api.js index 70cb50b..87bbf0c 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -136,7 +136,7 @@ async function getEventForTimestamp(roomID, ts) { */ async function getEvents(roomID, dir, pagination = {}, filter) { filter = filter && JSON.stringify(filter) - /** @type {Ty.Pagination>} */ + /** @type {Ty.MessagesPagination>} */ const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/messages`, null, {...pagination, dir, filter})) return root } diff --git a/src/types.d.ts b/src/types.d.ts index 6ee2eb1..a85907d 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -498,7 +498,13 @@ export type Membership = "invite" | "knock" | "join" | "leave" | "ban" export type Pagination = { chunk: T[] next_batch?: string - prev_match?: string + prev_batch?: string +} + +export type MessagesPagination = { + chunk: T[] + start: string + end?: string } export type HierarchyPagination = { From c68bac5476dd6809dffacfd09d467db1f257ba83 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 1 Mar 2026 22:05:46 +1300 Subject: [PATCH 7/7] Document encryption as unsupported --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 8c9fc90..e8a8e7e 100644 --- a/readme.md +++ b/readme.md @@ -38,6 +38,7 @@ For more information about features, [see the user guide.](https://gitdab.com/ca * This bridge is not designed for puppetting. * Direct Messaging is not supported until I figure out a good way of doing it. +* Encrypted messages are not supported. Decryption is often unreliable on Matrix, and your messages end up in plaintext on Discord anyway, so there's not much advantage. ## Get started!