From dc7b44408684379a2c41916c2efb849fc96d93f0 Mon Sep 17 00:00:00 2001 From: Elliu Date: Thu, 21 Aug 2025 21:38:34 +0900 Subject: [PATCH 001/153] Fix matrix api joinRoom() for remote rooms When using self-service mode and trying to link with a remote matrix room (room not in the same HS as the bridge user), then we need to add the "via" HSs to join the room with, or else it fails. We get it from the "m.space.child" in the "children_state" of the space hierarchy. It seems like the "via" information can also be stored in the "m.space.parent" in the states of the room, but hopefully this shouldn't be needed in sane implementations --- src/matrix/api.js | 4 ++-- src/web/routes/link.js | 22 +++++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/matrix/api.js b/src/matrix/api.js index 709d70c5..9daf65a1 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -64,9 +64,9 @@ async function createRoom(content) { /** * @returns {Promise} room ID */ -async function joinRoom(roomIDOrAlias, mxid) { +async function joinRoom(roomIDOrAlias, mxid, via) { /** @type {Ty.R.RoomJoined} */ - const root = await mreq.mreq("POST", path(`/client/v3/join/${roomIDOrAlias}`, mxid), {}) + const root = await mreq.mreq("POST", path(`/client/v3/join/${roomIDOrAlias}`, mxid, via), {}) return root.room_id } diff --git a/src/web/routes/link.js b/src/web/routes/link.js index c5f404eb..7654adf5 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -77,7 +77,7 @@ as.router.post("/api/link-space", defineEventHandler(async event => { // Check space exists and bridge is joined try { - await api.joinRoom(parsedBody.space_id) + await api.joinRoom(parsedBody.space_id, null, via) } catch (e) { throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) } @@ -135,19 +135,31 @@ as.router.post("/api/link", defineEventHandler(async event => { // Check room is part of the guild's space let found = false + let via = undefined for await (const room of api.generateFullHierarchy(spaceID)) { - if (room.room_id === parsedBody.matrix && !room.room_type) { + if (via === undefined && room.room_type === "m.space") { + for (state of room.children_state) { + if (state.state_key === parsedBody.matrix){ + via = {via: state.content.via} + if (found === true) + break + } + } + } + + if (!found && room.room_id === parsedBody.matrix && !room.room_type) { found = true - break + if (via !== undefined) + break } } if (!found) throw createError({status: 400, message: "Bad Request", data: "Matrix room needs to be part of the bridged space"}) // Check room exists and bridge is joined try { - await api.joinRoom(parsedBody.matrix) + await api.joinRoom(parsedBody.matrix, null, via ?? {}) } catch (e) { - throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) + throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}${via === null ? " (hint: couln't find a \"via\" in the space children_state for this room in order to help joining this room)" : ""}`}) } // Check bridge has PL 100 From 1efd301e1d21764b96c0b57a680d74c962ab197f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 22 Aug 2025 21:25:05 +1200 Subject: [PATCH 002/153] Cleanup --- src/matrix/api.js | 11 +++++++++-- src/types.d.ts | 10 +++++++++- src/web/routes/link.js | 34 ++++++++++++++++++---------------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/matrix/api.js b/src/matrix/api.js index 9daf65a1..edffc456 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -22,7 +22,11 @@ function path(p, mxid, otherParams = {}) { const u = new URL(p, "http://localhost") if (mxid) u.searchParams.set("user_id", mxid) for (const entry of Object.entries(otherParams)) { - if (entry[1] != undefined) { + if (Array.isArray(entry[1])) { + for (const element of entry[1]) { + u.searchParams.append(entry[0], element) + } + } else if (entry[1] != undefined) { u.searchParams.set(entry[0], entry[1]) } } @@ -62,11 +66,14 @@ async function createRoom(content) { } /** + * @param {string} roomIDOrAlias + * @param {string?} [mxid] + * @param {string[]?} [via] * @returns {Promise} room ID */ async function joinRoom(roomIDOrAlias, mxid, via) { /** @type {Ty.R.RoomJoined} */ - const root = await mreq.mreq("POST", path(`/client/v3/join/${roomIDOrAlias}`, mxid, via), {}) + const root = await mreq.mreq("POST", path(`/client/v3/join/${roomIDOrAlias}`, mxid, {via}), {}) return root.room_id } diff --git a/src/types.d.ts b/src/types.d.ts index 27dfddfb..a34095d9 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -149,6 +149,14 @@ export namespace Event { prev_content?: any } + export type Outer_StrippedChildStateEvent = { + type: string + state_key: string + sender: string + origin_server_ts: number + content: any + } + export type M_Room_Message = { msgtype: "m.text" | "m.emote" body: string @@ -345,7 +353,7 @@ export namespace R { export type Hierarchy = { avatar_url?: string canonical_alias?: string - children_state: {} + children_state: Event.Outer_StrippedChildStateEvent[] guest_can_join: boolean join_rule?: string name?: string diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 7654adf5..f85e66dc 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -134,32 +134,34 @@ as.router.post("/api/link", defineEventHandler(async event => { if (row) throw createError({status: 400, message: "Bad Request", data: `Channel ID ${row.channel_id} or room ID ${parsedBody.matrix} are already bridged and cannot be reused`}) // Check room is part of the guild's space - let found = false - let via = undefined + let foundRoom = false + /** @type {string[]?} */ + let foundVia = null for await (const room of api.generateFullHierarchy(spaceID)) { - if (via === undefined && room.room_type === "m.space") { - for (state of room.children_state) { - if (state.state_key === parsedBody.matrix){ - via = {via: state.content.via} - if (found === true) - break - } + // When finding a space during iteration, look at space's children state, because we need a `via` to join the room (when we find it later) + for (const state of room.children_state) { + if (state.type === "m.space.child" && state.state_key === parsedBody.matrix) { + foundVia = state.content.via } } - if (!found && room.room_id === parsedBody.matrix && !room.room_type) { - found = true - if (via !== undefined) - break + // When finding a room during iteration, see if it was the requested room (to confirm that the room is in the space) + if (room.room_id === parsedBody.matrix && !room.room_type) { + foundRoom = true } + + if (foundRoom && foundVia) break } - if (!found) throw createError({status: 400, message: "Bad Request", data: "Matrix room needs to be part of the bridged space"}) + if (!foundRoom) throw createError({status: 400, message: "Bad Request", data: "Matrix room needs to be part of the bridged space"}) // Check room exists and bridge is joined try { - await api.joinRoom(parsedBody.matrix, null, via ?? {}) + await api.joinRoom(parsedBody.matrix, null, foundVia) } catch (e) { - throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}${via === null ? " (hint: couln't find a \"via\" in the space children_state for this room in order to help joining this room)" : ""}`}) + if (!foundVia) { + throw createError({status: 403, message: "Unable To Join", data: `Unable to join the requested Matrix room. Please invite the bridge to the room and try again. (Server said: ${e.errcode} - ${e.message})`}) + } + throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) } // Check bridge has PL 100 From ea08e16963d012943e631506be28e407f6737ebb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 22 Aug 2025 21:42:13 +1200 Subject: [PATCH 003/153] Update tests for new types and code path --- src/web/routes/link.test.js | 58 +++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/src/web/routes/link.test.js b/src/web/routes/link.test.js index 0d8d366d..068bc9b8 100644 --- a/src/web/routes/link.test.js +++ b/src/web/routes/link.test.js @@ -360,7 +360,7 @@ test("web link room: check that room is part of space (not in hierarchy)", async t.equal(called, 1) }) -test("web link room: check that bridge can join room", async t => { +test("web link room: check that bridge can join room (notices lack of via and asks for invite instead)", async t => { let called = 0 const [error] = await tryToCatch(() => router.test("post", "/api/link", { sessionData: { @@ -381,7 +381,55 @@ test("web link room: check that bridge can join room", async t => { t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], + guest_can_join: false, + num_joined_members: 2 + } + /* c8 ignore next */ + } + } + })) + t.equal(error.data, "Unable to join the requested Matrix room. Please invite the bridge to the room and try again. (Server said: M_FORBIDDEN - not allowed to join I guess)") + t.equal(called, 2) +}) + +test("web link room: check that bridge can join room (uses via for join attempt)", async t => { + let called = 0 + const [error] = await tryToCatch(() => router.test("post", "/api/link", { + sessionData: { + managedGuilds: ["665289423482519565"] + }, + body: { + discord: "665310973967597573", + matrix: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + guild_id: "665289423482519565" + }, + api: { + async joinRoom(roomID, _, via) { + called++ + t.deepEqual(via, ["cadence.moe", "hashi.re"]) + throw new MatrixServerError({errcode: "M_FORBIDDEN", error: "not allowed to join I guess"}) + }, + async *generateFullHierarchy(spaceID) { + called++ + t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + yield { + room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + children_state: [], + guest_can_join: false, + num_joined_members: 2 + } + yield { + room_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", + children_state: [{ + type: "m.space.child", + state_key: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + sender: "@elliu:hashi.re", + content: { + via: ["cadence.moe", "hashi.re"] + }, + origin_server_ts: 0 + }], guest_can_join: false, num_joined_members: 2 } @@ -414,7 +462,7 @@ test("web link room: check that bridge has PL 100 in target room (event missing) t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], guest_can_join: false, num_joined_members: 2 } @@ -454,7 +502,7 @@ test("web link room: check that bridge has PL 100 in target room (users default) t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], guest_can_join: false, num_joined_members: 2 } @@ -494,7 +542,7 @@ test("web link room: successfully calls createRoom", async t => { t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], guest_can_join: false, num_joined_members: 2 } From e7b4dfea9cb48598e9102764c6077797f222c930 Mon Sep 17 00:00:00 2001 From: Elliu Date: Sat, 23 Aug 2025 00:01:30 +0900 Subject: [PATCH 004/153] Fix /api/link-space joinRoom() for remote spaces --- src/web/routes/link.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/web/routes/link.js b/src/web/routes/link.js index f85e66dc..b46b78f6 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -12,6 +12,20 @@ const auth = sync.require("../auth") const mreq = sync.require("../../matrix/mreq") const {reg} = require("../../matrix/read-registration") +/** + * @param {string} UserID + * @returns {string} the HS of the user, or "" if the user ID is malformed + */ +function getHSOfUser(user) { + domainStartIndex = user.indexOf(":"); + if (domainStartIndex >= 1) { + return user.slice(domainStartIndex + 1) + } + + return "" +} + + /** * @param {H3Event} event * @returns {import("../../matrix/api")} @@ -75,6 +89,9 @@ as.router.post("/api/link-space", defineEventHandler(async event => { const existing = select("guild_space", "guild_id", {}, "WHERE guild_id = ? OR space_id = ?").get(guildID, spaceID) if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) + const inviteSender = select("invite", "mxid", {mxid: session.data.mxid, room_id: spaceID}).pluck().get() + via = [ getHSOfUser(inviteSender) ] + // Check space exists and bridge is joined try { await api.joinRoom(parsedBody.space_id, null, via) From 0776cc6ccdddef5da82caaa090121bf3809bf0c0 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 23 Aug 2025 23:46:51 +1200 Subject: [PATCH 005/153] Fill in more of reg for other people to test with --- src/matrix/api.test.js | 4 ++++ test/test.js | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/matrix/api.test.js b/src/matrix/api.test.js index 82565ebe..da923858 100644 --- a/src/matrix/api.test.js +++ b/src/matrix/api.test.js @@ -24,3 +24,7 @@ test("api path: real world mxid", t => { test("api path: extras number works", t => { t.equal(path(`/client/v3/rooms/!example/timestamp_to_event`, null, {ts: 1687324651120}), "/client/v3/rooms/!example/timestamp_to_event?ts=1687324651120") }) + +test("api path: multiple via params", t => { + t.equal(path(`/client/v3/rooms/!example/join`, null, {via: ["cadence.moe", "matrix.org"], ts: 1687324651120}), "/client/v3/rooms/!example/join?via=cadence.moe&via=matrix.org&ts=1687324651120") +}) diff --git a/test/test.js b/test/test.js index 233fd940..b01f0ce2 100644 --- a/test/test.js +++ b/test/test.js @@ -17,6 +17,8 @@ const {reg} = require("../src/matrix/read-registration") reg.ooye.discord_token = "Njg0MjgwMTkyNTUzODQ0NzQ3.Xl3zlw.baby" reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded reg.ooye.server_name = "cadence.moe" +reg.ooye.namespace_prefix = "_ooye_" +reg.sender_localpart = "_ooye_bot" reg.id = "baby" reg.as_token = "don't actually take authenticated actions on the server" reg.hs_token = "don't actually take authenticated actions on the server" From 7932f8af851148bb7a10a4e5ac8f9305f15d2be1 Mon Sep 17 00:00:00 2001 From: Elliu Date: Sun, 31 Aug 2025 20:11:07 +0900 Subject: [PATCH 006/153] Add "please try invite" message when joinRoom in /api/link-space fails --- src/web/routes/link.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/web/routes/link.js b/src/web/routes/link.js index b46b78f6..d16ecea5 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -96,6 +96,9 @@ as.router.post("/api/link-space", defineEventHandler(async event => { try { await api.joinRoom(parsedBody.space_id, null, via) } catch (e) { + if (via.join("") == "") { + throw createError({status: 403, message: "Unable To Join", data: `Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: ${e.errcode} - ${e.message})`}) + } throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) } From 717dc185e5b68def578b1d31de5a158b386e0ab6 Mon Sep 17 00:00:00 2001 From: Elliu Date: Thu, 9 Oct 2025 21:54:15 +0900 Subject: [PATCH 007/153] Misc. fixes for remote join --- src/web/routes/link.js | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/src/web/routes/link.js b/src/web/routes/link.js index d16ecea5..a7ae8ecf 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -12,20 +12,6 @@ const auth = sync.require("../auth") const mreq = sync.require("../../matrix/mreq") const {reg} = require("../../matrix/read-registration") -/** - * @param {string} UserID - * @returns {string} the HS of the user, or "" if the user ID is malformed - */ -function getHSOfUser(user) { - domainStartIndex = user.indexOf(":"); - if (domainStartIndex >= 1) { - return user.slice(domainStartIndex + 1) - } - - return "" -} - - /** * @param {H3Event} event * @returns {import("../../matrix/api")} @@ -90,16 +76,13 @@ as.router.post("/api/link-space", defineEventHandler(async event => { if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) const inviteSender = select("invite", "mxid", {mxid: session.data.mxid, room_id: spaceID}).pluck().get() - via = [ getHSOfUser(inviteSender) ] + const via = [ inviteSender?.match(/:(.*)/)?.[1] ?? "" ] // Check space exists and bridge is joined try { await api.joinRoom(parsedBody.space_id, null, via) } catch (e) { - if (via.join("") == "") { - throw createError({status: 403, message: "Unable To Join", data: `Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: ${e.errcode} - ${e.message})`}) - } - throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) + throw createError({status: 400, message: "Unable To Join", data: `Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: ${e.errcode} - ${e.message})`}) } // Check bridge has PL 100 @@ -179,7 +162,7 @@ as.router.post("/api/link", defineEventHandler(async event => { await api.joinRoom(parsedBody.matrix, null, foundVia) } catch (e) { if (!foundVia) { - throw createError({status: 403, message: "Unable To Join", data: `Unable to join the requested Matrix room. Please invite the bridge to the room and try again. (Server said: ${e.errcode} - ${e.message})`}) + throw createError({status: 400, message: "Unable To Join", data: `Unable to join the requested Matrix room. Please invite the bridge to the room and try again. (Server said: ${e.errcode} - ${e.message})`}) } throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) } From ab69eab8a492e99d7e30f4e3196c5a6d09debe0b Mon Sep 17 00:00:00 2001 From: Elliu Date: Sat, 1 Nov 2025 21:01:15 +0900 Subject: [PATCH 008/153] Fix tests for new link space error message --- src/web/routes/link.js | 4 ++-- src/web/routes/link.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/web/routes/link.js b/src/web/routes/link.js index a7ae8ecf..d4d13da1 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -76,13 +76,13 @@ as.router.post("/api/link-space", defineEventHandler(async event => { if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) const inviteSender = select("invite", "mxid", {mxid: session.data.mxid, room_id: spaceID}).pluck().get() - const via = [ inviteSender?.match(/:(.*)/)?.[1] ?? "" ] + const via = [ inviteSender?.match(/:(.*)/)?.[1] ?? "" ] // Check space exists and bridge is joined try { await api.joinRoom(parsedBody.space_id, null, via) } catch (e) { - throw createError({status: 400, message: "Unable To Join", data: `Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: ${e.errcode} - ${e.message})`}) + throw createError({status: 400, message: "Unable To Join", data: `Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: ${e.errcode} - ${e.message})`}) } // Check bridge has PL 100 diff --git a/src/web/routes/link.test.js b/src/web/routes/link.test.js index 068bc9b8..ffe4e5ee 100644 --- a/src/web/routes/link.test.js +++ b/src/web/routes/link.test.js @@ -77,7 +77,7 @@ test("web link space: check that OOYE is joined", async t => { } } })) - t.equal(error.data, "M_FORBIDDEN - not allowed to join I guess") + t.equal(error.data, "Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: M_FORBIDDEN - not allowed to join I guess)") t.equal(called, 1) }) From a55d7a16325c7783223bdcc54358c46d02aa57db Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 2 Nov 2025 20:46:31 +1300 Subject: [PATCH 009/153] Style match --- jsconfig.json | 9 +++++++++ src/d2m/converters/message-to-event.js | 1 + src/types.d.ts | 4 ++-- src/web/routes/link.js | 3 ++- 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 jsconfig.json diff --git a/jsconfig.json b/jsconfig.json new file mode 100644 index 00000000..4106061c --- /dev/null +++ b/jsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "target": "es2024", + "module": "nodenext", + "strict": true, + "noImplicitAny": false, + "useUnknownInCatchVariables": false + } +} diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 30a20fe2..93e120e0 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -207,6 +207,7 @@ async function attachmentToEvent(mentions, attachment) { * - alwaysReturnFormattedBody: false - formatted_body will be skipped if it is the same as body because the message is plaintext. if you want the formatted_body to be returned anyway, for example to merge it with another message, then set this to true. * - scanTextForMentions: true - needs to be set to false when converting forwarded messages etc which may be from a different channel that can't be scanned. * @param {{api: import("../../matrix/api"), snow?: import("snowtransfer").SnowTransfer}} di simple-as-nails dependency injection for the matrix API + * @returns {Promise<{$type: string, $sender?: string, [x: string]: any}[]>} */ async function messageToEvent(message, guild, options = {}, di) { const events = [] diff --git a/src/types.d.ts b/src/types.d.ts index a34095d9..c7cb006f 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -149,7 +149,7 @@ export namespace Event { prev_content?: any } - export type Outer_StrippedChildStateEvent = { + export type StrippedChildStateEvent = { type: string state_key: string sender: string @@ -353,7 +353,7 @@ export namespace R { export type Hierarchy = { avatar_url?: string canonical_alias?: string - children_state: Event.Outer_StrippedChildStateEvent[] + children_state: Event.StrippedChildStateEvent[] guest_can_join: boolean join_rule?: string name?: string diff --git a/src/web/routes/link.js b/src/web/routes/link.js index d4d13da1..0afbc495 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -76,7 +76,8 @@ as.router.post("/api/link-space", defineEventHandler(async event => { if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) const inviteSender = select("invite", "mxid", {mxid: session.data.mxid, room_id: spaceID}).pluck().get() - const via = [ inviteSender?.match(/:(.*)/)?.[1] ?? "" ] + const inviteSenderServer = inviteSender?.match(/:(.*)/)?.[1] + const via = [inviteSenderServer || ""] // Check space exists and bridge is joined try { From d95a1143774e69b8619d2e00e2d70c5186b99c8e Mon Sep 17 00:00:00 2001 From: Elliu Date: Sun, 2 Nov 2025 07:50:16 +0000 Subject: [PATCH 010/153] Fix matrix api joinRoom() for remote rooms (#60) When using self-service mode and trying to link with a remote matrix room (room not in the same HS as the bridge user), then we need to add the "via" HSs to join the room with, or else it fails. We get it from the "m.space.child" in the "children_state" of the space hierarchy. Co-authored-by: Cadence Ember Reviewed-on: https://gitdab.com/cadence/out-of-your-element/pulls/60 Co-authored-by: Elliu Co-committed-by: Elliu --- jsconfig.json | 9 ++++ src/d2m/converters/message-to-event.js | 1 + src/matrix/api.js | 13 ++++-- src/matrix/api.test.js | 4 ++ src/types.d.ts | 10 ++++- src/web/routes/link.js | 34 +++++++++++---- src/web/routes/link.test.js | 60 +++++++++++++++++++++++--- test/test.js | 2 + 8 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 jsconfig.json diff --git a/jsconfig.json b/jsconfig.json new file mode 100644 index 00000000..4106061c --- /dev/null +++ b/jsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "target": "es2024", + "module": "nodenext", + "strict": true, + "noImplicitAny": false, + "useUnknownInCatchVariables": false + } +} diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 30a20fe2..93e120e0 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -207,6 +207,7 @@ async function attachmentToEvent(mentions, attachment) { * - alwaysReturnFormattedBody: false - formatted_body will be skipped if it is the same as body because the message is plaintext. if you want the formatted_body to be returned anyway, for example to merge it with another message, then set this to true. * - scanTextForMentions: true - needs to be set to false when converting forwarded messages etc which may be from a different channel that can't be scanned. * @param {{api: import("../../matrix/api"), snow?: import("snowtransfer").SnowTransfer}} di simple-as-nails dependency injection for the matrix API + * @returns {Promise<{$type: string, $sender?: string, [x: string]: any}[]>} */ async function messageToEvent(message, guild, options = {}, di) { const events = [] diff --git a/src/matrix/api.js b/src/matrix/api.js index 709d70c5..edffc456 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -22,7 +22,11 @@ function path(p, mxid, otherParams = {}) { const u = new URL(p, "http://localhost") if (mxid) u.searchParams.set("user_id", mxid) for (const entry of Object.entries(otherParams)) { - if (entry[1] != undefined) { + if (Array.isArray(entry[1])) { + for (const element of entry[1]) { + u.searchParams.append(entry[0], element) + } + } else if (entry[1] != undefined) { u.searchParams.set(entry[0], entry[1]) } } @@ -62,11 +66,14 @@ async function createRoom(content) { } /** + * @param {string} roomIDOrAlias + * @param {string?} [mxid] + * @param {string[]?} [via] * @returns {Promise} room ID */ -async function joinRoom(roomIDOrAlias, mxid) { +async function joinRoom(roomIDOrAlias, mxid, via) { /** @type {Ty.R.RoomJoined} */ - const root = await mreq.mreq("POST", path(`/client/v3/join/${roomIDOrAlias}`, mxid), {}) + const root = await mreq.mreq("POST", path(`/client/v3/join/${roomIDOrAlias}`, mxid, {via}), {}) return root.room_id } diff --git a/src/matrix/api.test.js b/src/matrix/api.test.js index 82565ebe..da923858 100644 --- a/src/matrix/api.test.js +++ b/src/matrix/api.test.js @@ -24,3 +24,7 @@ test("api path: real world mxid", t => { test("api path: extras number works", t => { t.equal(path(`/client/v3/rooms/!example/timestamp_to_event`, null, {ts: 1687324651120}), "/client/v3/rooms/!example/timestamp_to_event?ts=1687324651120") }) + +test("api path: multiple via params", t => { + t.equal(path(`/client/v3/rooms/!example/join`, null, {via: ["cadence.moe", "matrix.org"], ts: 1687324651120}), "/client/v3/rooms/!example/join?via=cadence.moe&via=matrix.org&ts=1687324651120") +}) diff --git a/src/types.d.ts b/src/types.d.ts index 27dfddfb..c7cb006f 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -149,6 +149,14 @@ export namespace Event { prev_content?: any } + export type StrippedChildStateEvent = { + type: string + state_key: string + sender: string + origin_server_ts: number + content: any + } + export type M_Room_Message = { msgtype: "m.text" | "m.emote" body: string @@ -345,7 +353,7 @@ export namespace R { export type Hierarchy = { avatar_url?: string canonical_alias?: string - children_state: {} + children_state: Event.StrippedChildStateEvent[] guest_can_join: boolean join_rule?: string name?: string diff --git a/src/web/routes/link.js b/src/web/routes/link.js index c5f404eb..0afbc495 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -75,11 +75,15 @@ as.router.post("/api/link-space", defineEventHandler(async event => { const existing = select("guild_space", "guild_id", {}, "WHERE guild_id = ? OR space_id = ?").get(guildID, spaceID) if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) + const inviteSender = select("invite", "mxid", {mxid: session.data.mxid, room_id: spaceID}).pluck().get() + const inviteSenderServer = inviteSender?.match(/:(.*)/)?.[1] + const via = [inviteSenderServer || ""] + // Check space exists and bridge is joined try { - await api.joinRoom(parsedBody.space_id) + await api.joinRoom(parsedBody.space_id, null, via) } catch (e) { - throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) + throw createError({status: 400, message: "Unable To Join", data: `Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: ${e.errcode} - ${e.message})`}) } // Check bridge has PL 100 @@ -134,19 +138,33 @@ as.router.post("/api/link", defineEventHandler(async event => { if (row) throw createError({status: 400, message: "Bad Request", data: `Channel ID ${row.channel_id} or room ID ${parsedBody.matrix} are already bridged and cannot be reused`}) // Check room is part of the guild's space - let found = false + let foundRoom = false + /** @type {string[]?} */ + let foundVia = null for await (const room of api.generateFullHierarchy(spaceID)) { - if (room.room_id === parsedBody.matrix && !room.room_type) { - found = true - break + // When finding a space during iteration, look at space's children state, because we need a `via` to join the room (when we find it later) + for (const state of room.children_state) { + if (state.type === "m.space.child" && state.state_key === parsedBody.matrix) { + foundVia = state.content.via + } } + + // When finding a room during iteration, see if it was the requested room (to confirm that the room is in the space) + if (room.room_id === parsedBody.matrix && !room.room_type) { + foundRoom = true + } + + if (foundRoom && foundVia) break } - if (!found) throw createError({status: 400, message: "Bad Request", data: "Matrix room needs to be part of the bridged space"}) + if (!foundRoom) throw createError({status: 400, message: "Bad Request", data: "Matrix room needs to be part of the bridged space"}) // Check room exists and bridge is joined try { - await api.joinRoom(parsedBody.matrix) + await api.joinRoom(parsedBody.matrix, null, foundVia) } catch (e) { + if (!foundVia) { + throw createError({status: 400, message: "Unable To Join", data: `Unable to join the requested Matrix room. Please invite the bridge to the room and try again. (Server said: ${e.errcode} - ${e.message})`}) + } throw createError({status: 403, message: e.errcode, data: `${e.errcode} - ${e.message}`}) } diff --git a/src/web/routes/link.test.js b/src/web/routes/link.test.js index 0d8d366d..ffe4e5ee 100644 --- a/src/web/routes/link.test.js +++ b/src/web/routes/link.test.js @@ -77,7 +77,7 @@ test("web link space: check that OOYE is joined", async t => { } } })) - t.equal(error.data, "M_FORBIDDEN - not allowed to join I guess") + t.equal(error.data, "Unable to join the requested Matrix space. Please invite the bridge to the space and try again. (Server said: M_FORBIDDEN - not allowed to join I guess)") t.equal(called, 1) }) @@ -360,7 +360,7 @@ test("web link room: check that room is part of space (not in hierarchy)", async t.equal(called, 1) }) -test("web link room: check that bridge can join room", async t => { +test("web link room: check that bridge can join room (notices lack of via and asks for invite instead)", async t => { let called = 0 const [error] = await tryToCatch(() => router.test("post", "/api/link", { sessionData: { @@ -381,7 +381,55 @@ test("web link room: check that bridge can join room", async t => { t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], + guest_can_join: false, + num_joined_members: 2 + } + /* c8 ignore next */ + } + } + })) + t.equal(error.data, "Unable to join the requested Matrix room. Please invite the bridge to the room and try again. (Server said: M_FORBIDDEN - not allowed to join I guess)") + t.equal(called, 2) +}) + +test("web link room: check that bridge can join room (uses via for join attempt)", async t => { + let called = 0 + const [error] = await tryToCatch(() => router.test("post", "/api/link", { + sessionData: { + managedGuilds: ["665289423482519565"] + }, + body: { + discord: "665310973967597573", + matrix: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + guild_id: "665289423482519565" + }, + api: { + async joinRoom(roomID, _, via) { + called++ + t.deepEqual(via, ["cadence.moe", "hashi.re"]) + throw new MatrixServerError({errcode: "M_FORBIDDEN", error: "not allowed to join I guess"}) + }, + async *generateFullHierarchy(spaceID) { + called++ + t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + yield { + room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + children_state: [], + guest_can_join: false, + num_joined_members: 2 + } + yield { + room_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", + children_state: [{ + type: "m.space.child", + state_key: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + sender: "@elliu:hashi.re", + content: { + via: ["cadence.moe", "hashi.re"] + }, + origin_server_ts: 0 + }], guest_can_join: false, num_joined_members: 2 } @@ -414,7 +462,7 @@ test("web link room: check that bridge has PL 100 in target room (event missing) t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], guest_can_join: false, num_joined_members: 2 } @@ -454,7 +502,7 @@ test("web link room: check that bridge has PL 100 in target room (users default) t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], guest_can_join: false, num_joined_members: 2 } @@ -494,7 +542,7 @@ test("web link room: successfully calls createRoom", async t => { t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") yield { room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: {}, + children_state: [], guest_can_join: false, num_joined_members: 2 } diff --git a/test/test.js b/test/test.js index 233fd940..b01f0ce2 100644 --- a/test/test.js +++ b/test/test.js @@ -17,6 +17,8 @@ const {reg} = require("../src/matrix/read-registration") reg.ooye.discord_token = "Njg0MjgwMTkyNTUzODQ0NzQ3.Xl3zlw.baby" reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded reg.ooye.server_name = "cadence.moe" +reg.ooye.namespace_prefix = "_ooye_" +reg.sender_localpart = "_ooye_bot" reg.id = "baby" reg.as_token = "don't actually take authenticated actions on the server" reg.hs_token = "don't actually take authenticated actions on the server" From bc32910cc8bff99b145bbf2e9a7e11eed22e2257 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 9 Nov 2025 00:24:24 +1300 Subject: [PATCH 011/153] Silence errors if generated embeds can't be sent --- src/d2m/actions/edit-message.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/d2m/actions/edit-message.js b/src/d2m/actions/edit-message.js index 1afcb35e..7f1bff7d 100644 --- a/src/d2m/actions/edit-message.js +++ b/src/d2m/actions/edit-message.js @@ -10,6 +10,8 @@ const editToChanges = sync.require("../converters/edit-to-changes") const registerPkUser = sync.require("./register-pk-user") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/mreq")} */ +const mreq = sync.require("../../matrix/mreq") /** * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message @@ -70,8 +72,17 @@ async function editMessage(message, guild, row) { const part = sendNewEventParts.has("part") && eventsToSend[0] === content ? 0 : 1 const reactionPart = sendNewEventParts.has("reaction_part") && eventsToSend[eventsToSend.length - 1] === content ? 0 : 1 - const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord + + try { + const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid) + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord + } catch (e) { + if (e instanceof mreq.MatrixServerError && e.errcode === "M_FORBIDDEN") { + // sending user doesn't have permission to update message, e.g. because Discord generated an embed in a read-only room + } else { + throw e + } + } } } From b4dd66fbec4b277eea06d2baa09b27b61fe1b1dc Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 9 Nov 2025 01:07:03 +1300 Subject: [PATCH 012/153] More consistently generate embeds for Matrix --- src/d2m/converters/edit-to-changes.js | 20 +++++++++++++------- src/d2m/event-dispatcher.js | 5 ----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index c615a3fc..fe0e790f 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -32,17 +32,21 @@ function eventIsText(ev) { * @param {import("../../matrix/api")} api simple-as-nails dependency injection for the matrix API */ async function editToChanges(message, guild, api) { - // If it is a user edit, allow deleting old messages (e.g. they might have removed text from an image). - // If it is the system adding a generated embed to a message, don't delete old messages since the system only sends partial data. - // Since an update in August 2024, the system always provides the full data of message updates. I'll leave in the old code since it won't cause problems. - - const isGeneratedEmbed = !("content" in message) - // Figure out what events we will be replacing const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get() assert(roomID) - const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all() + const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part", "source"], {message_id: message.id}).all() + + // If it is a user edit, allow deleting old messages (e.g. they might have removed text from an image). + // If it is the system adding a generated embed to a message, don't delete old messages since the system only sends partial data. + // Since an update in August 2024, the system always provides the full data of message updates. + // Now, this code path is only used by generated embeds for messages that were originally sent from Matrix. + + const originallyFromMatrix = oldEventRows.find(r => r.part === 0)?.source === 0 + const isGeneratedEmbed = !("content" in message) || originallyFromMatrix + + // Figure out who to send as /** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */ let senderMxid = null @@ -117,6 +121,8 @@ async function editToChanges(message, guild, api) { if (isGeneratedEmbed) { unchangedEvents.push(...eventsToRedact.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToRedact to unchangedEvents. eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice") + unchangedEvents.push(...eventsToReplace.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToReplace to unchangedEvents. + eventsToReplace = eventsToReplace.filter(e => e.old.event_subtype === "m.notice") } // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 1698317a..49352d79 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -272,11 +272,6 @@ module.exports = { // Otherwise, if there are embeds, then the system generated URL preview embeds. if (!(typeof data.content === "string" || "embeds" in data)) return - if (data.webhook_id) { - const row = select("webhook", "webhook_id", {webhook_id: data.webhook_id}).pluck().get() - if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. - } - if (dUtils.isEphemeralMessage(data)) return // Ephemeral messages are for the eyes of the receiver only! // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. From cba79658fae0675574ba3d4b9e23c945c7d5bb51 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Nov 2025 00:45:56 +1300 Subject: [PATCH 013/153] Don't post new embeds for old messages --- src/d2m/converters/edit-to-changes.js | 5 ++++ src/d2m/converters/edit-to-changes.test.js | 9 +++++++ test/data.js | 30 ++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index fe0e790f..e5234080 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -123,6 +123,11 @@ async function editToChanges(message, guild, api) { eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice") unchangedEvents.push(...eventsToReplace.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToReplace to unchangedEvents. eventsToReplace = eventsToReplace.filter(e => e.old.event_subtype === "m.notice") + + // Don't post new generated embeds for messages if it's been a while since the message was sent. Detached embeds look weird. + if (message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 120 * 1000) { // older than 2 minutes ago + eventsToSend = eventsToSend.filter(e => e.msgtype !== "m.notice") + } } // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! diff --git a/src/d2m/converters/edit-to-changes.test.js b/src/d2m/converters/edit-to-changes.test.js index 30549c78..f6674c04 100644 --- a/src/d2m/converters/edit-to-changes.test.js +++ b/src/d2m/converters/edit-to-changes.test.js @@ -360,3 +360,12 @@ test("edit2changes: generated embed on a reply", async t => { t.equal(senderMxid, "@_ooye_cadence:cadence.moe") t.equal(called, 1) }) + +test("edit2changes: don't generate embed if it's been too long since the message", async t => { + const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.embed_generated_social_media_image_for_matrix_user, data.guild.general) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToReplace, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(promotions, []) + t.equal(senderMxid, null) +}) diff --git a/test/data.js b/test/data.js index e64b9c2f..c9f58d0e 100644 --- a/test/data.js +++ b/test/data.js @@ -5296,6 +5296,36 @@ module.exports = { guild_id: "112760669178241024", id: "1210387798297682020" }, + embed_generated_social_media_image_for_matrix_user: { + channel_id: "112760669178241024", + embeds: [ + { + color: 8594767, + description: "1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:\n\n * Both players draw eight cards\n * Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand\n * Both players present their best five-or-less-card pok...", + provider: { + name: "hthrflwrs on cohost" + }, + thumbnail: { + height: 1587, + placeholder: "GpoKP5BJZphshnhwmmmYlmh3l7+m+mwJ", + placeholder_version: 1, + proxy_url: "https://images-ext-2.discordapp.net/external/9vTXIzlXU4wyUZvWfmlmQkck8nGLUL-A090W4lWsZ48/https/staging.cohostcdn.org/avatar/292-6b64b03c-4ada-42f6-8452-109275bfe68d-profile.png", + url: "https://staging.cohostcdn.org/avatar/292-6b64b03c-4ada-42f6-8452-109275bfe68d-profile.png", + width: 1644 + }, + title: "This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO", + type: "link", + url: "https://cohost.org/jkap/post/4794219-empty" + } + ], + author: { + name: "Matrix Bridge", + id: "684280192553844747" + }, + guild_id: "112760669178241024", + id: "1128118177155526666", + timestamp: "2025-01-01T00:00:00Z" + }, embed_generated_on_reply: { attachments: [], author: { From 03aa4e086668954aed8624e38e089c19eea3557e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Nov 2025 00:55:11 +1300 Subject: [PATCH 014/153] Update dependencies --- package-lock.json | 414 ++++++++++++-------- package.json | 4 +- src/m2d/converters/event-to-message.test.js | 4 +- 3 files changed, 250 insertions(+), 172 deletions(-) diff --git a/package-lock.json b/package-lock.json index fda73e3f..832b2017 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.14.0", - "discord-api-types": "^0.38.19", + "discord-api-types": "^0.38.31", "domino": "^2.1.6", "enquirer": "^2.4.1", "entities": "^5.0.0", @@ -34,7 +34,7 @@ "htmx.org": "^2.0.4", "lru-cache": "^11.0.2", "prettier-bytes": "^1.0.4", - "sharp": "^0.33.4", + "sharp": "^0.34.5", "snowtransfer": "^0.14.2", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", @@ -119,9 +119,9 @@ } }, "node_modules/@chriscdn/promise-semaphore": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-3.1.1.tgz", - "integrity": "sha512-ALLLLYlPfd/QZLptcVi6HQRK1zaCDWZoqYYw+axLmCatFs4gVTSZ5nqlyxwFe4qwR/K84HvOMa9hxda881FqMA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-3.1.2.tgz", + "integrity": "sha512-rELbH6FSr9wr5J249Ax8dpzQdTaqEgcW+lilDKZxB13Hz0Bz3Iyx4q/7qZxPMnra9FUW4ZOkVf+bx5tbi6Goog==", "license": "MIT" }, "node_modules/@cloudcmd/stub": { @@ -355,9 +355,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", - "integrity": "sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.0.tgz", + "integrity": "sha512-oAYoQnCYaQZKVS53Fq23ceWMRxq5EhQsE0x0RdQ55jT7wagMu5k+fS39v1fiSLrtrLQlXwVINenqhLMtTrV/1Q==", "license": "MIT", "optional": true, "dependencies": { @@ -369,10 +369,19 @@ "resolved": "https://registry.npmjs.org/@hotwired/stimulus/-/stimulus-3.2.2.tgz", "integrity": "sha512-eGeIqNOQpXoPAIP7tC1+1Yc1yl1xnwYqg+3mzqxyrbE5pg5YFBZcA6YoTiByJB6DKAEsiWtl6tjTJS4IYtbB7A==" }, + "node_modules/@img/colour": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", + "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", - "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", "cpu": [ "arm64" ], @@ -388,13 +397,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.4" + "@img/sharp-libvips-darwin-arm64": "1.2.4" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", - "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", "cpu": [ "x64" ], @@ -410,13 +419,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.4" + "@img/sharp-libvips-darwin-x64": "1.2.4" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", - "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", "cpu": [ "arm64" ], @@ -430,9 +439,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", - "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", "cpu": [ "x64" ], @@ -446,9 +455,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", - "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", "cpu": [ "arm" ], @@ -462,9 +471,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", - "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", "cpu": [ "arm64" ], @@ -477,10 +486,42 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", - "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", "cpu": [ "s390x" ], @@ -494,9 +535,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", - "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", "cpu": [ "x64" ], @@ -510,9 +551,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", - "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", "cpu": [ "arm64" ], @@ -526,9 +567,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", - "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", "cpu": [ "x64" ], @@ -542,9 +583,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", - "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", "cpu": [ "arm" ], @@ -560,13 +601,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.5" + "@img/sharp-libvips-linux-arm": "1.2.4" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", - "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", "cpu": [ "arm64" ], @@ -582,13 +623,57 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.4" + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", - "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", "cpu": [ "s390x" ], @@ -604,13 +689,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.4" + "@img/sharp-libvips-linux-s390x": "1.2.4" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", - "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", "cpu": [ "x64" ], @@ -626,13 +711,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.4" + "@img/sharp-libvips-linux-x64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", - "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", "cpu": [ "arm64" ], @@ -648,13 +733,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", - "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", "cpu": [ "x64" ], @@ -670,20 +755,20 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", - "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", "cpu": [ "wasm32" ], "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.2.0" + "@emnapi/runtime": "^1.7.0" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -692,10 +777,29 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", - "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", "cpu": [ "ia32" ], @@ -712,9 +816,9 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", - "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", "cpu": [ "x64" ], @@ -949,19 +1053,18 @@ "license": "MIT" }, "node_modules/@stackoverflow/stacks": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/@stackoverflow/stacks/-/stacks-2.8.4.tgz", - "integrity": "sha512-FfA7Bw7a0AQrMw3/bG6G4BUrZ698F7Cdk6HkR9T7jdaufORkiX5d16wI4j4b5Sqm1FwkaZAF+ZSKLL1w0tAsew==", - "license": "MIT", + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/@stackoverflow/stacks/-/stacks-2.8.6.tgz", + "integrity": "sha512-pR0vMDBA5rNV5Cb/McG+2F1nG68fZk3UiNwuOvQrP82WoQy41k9PsBjRjD7wHDVVZyDrWHLb/WnosXX7uK3IdA==", "dependencies": { "@hotwired/stimulus": "^3.2.2", "@popperjs/core": "^2.11.8" } }, "node_modules/@stackoverflow/stacks-icons": { - "version": "6.6.1", - "resolved": "https://registry.npmjs.org/@stackoverflow/stacks-icons/-/stacks-icons-6.6.1.tgz", - "integrity": "sha512-upa2jajYTKAHfILFbPWMsml0nlh4fbIEb2V9SS0txjOJEoZE2oBnNJXbg29vShp7Nyn1VwrMjaraX63WkKT07w==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@stackoverflow/stacks-icons/-/stacks-icons-6.7.2.tgz", + "integrity": "sha512-zEJDPDt7eYyAOMSnJFEPKkRoKydBWsg8LfEAX3TaF0UHI7N6vrVuOW6YeDIR2/uo0NahI9rf+Avg4+BADJmRhw==", "license": "MIT" }, "node_modules/@supertape/engine-loader": { @@ -1107,9 +1210,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "22.18.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.0.tgz", - "integrity": "sha512-m5ObIqwsUp6BZzyiy4RdZpzWGub9bqLJMvZDD0QMXhxjqMHMENlj+SqF5QxoUwaQNFe+8kz8XM8ZQhqkQPTgMQ==", + "version": "22.19.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.0.tgz", + "integrity": "sha512-xpr/lmLPQEj+TUnHmR+Ab91/glhJvsqcjB+yY0Ix9GO70H6Lb4FHH5GeqdOE5btAx7eIMwuHkp4H2MSkLcqWbA==", "dev": true, "license": "MIT", "dependencies": { @@ -1239,9 +1342,9 @@ ] }, "node_modules/better-sqlite3": { - "version": "12.2.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.2.0.tgz", - "integrity": "sha512-eGbYq2CT+tos1fBwLQ/tkBt9J5M3JEHjku4hbvQUePCckkvVf14xWj+1m7dGoK81M/fOjFT7yM9UMeKT/+vFLQ==", + "version": "12.4.1", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.4.1.tgz", + "integrity": "sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ==", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -1476,22 +1579,11 @@ "node": ">=16.15.0" } }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -1502,16 +1594,8 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "node_modules/constantinople": { "version": "4.0.1", @@ -1620,17 +1704,18 @@ "license": "MIT" }, "node_modules/detect-libc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", - "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", "engines": { "node": ">=8" } }, "node_modules/discord-api-types": { - "version": "0.38.22", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.22.tgz", - "integrity": "sha512-2gnYrgXN3yTlv2cKBISI/A8btZwsSZLwKpIQXeI1cS8a7W7wP3sFVQOm3mPuuinTD8jJCKGPGNH399zE7Un1kA==", + "version": "0.38.32", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.32.tgz", + "integrity": "sha512-UhIqkFuUVwBzejLPPWF18qixYPucMf718RnGh1NxZYNS7czXUmcUsWWkzWR7lRWj5pjfj4LwrnN9McvpfLvGqQ==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" @@ -1973,11 +2058,6 @@ "url": "https://github.com/sponsors/brc-dd" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -2116,9 +2196,9 @@ } }, "node_modules/lru-cache": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", - "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", + "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", "license": "ISC", "engines": { "node": "20 || >=22" @@ -2589,9 +2669,10 @@ ] }, "node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -2600,15 +2681,15 @@ } }, "node_modules/sharp": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", - "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "color": "^4.2.3", - "detect-libc": "^2.0.3", - "semver": "^7.6.3" + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -2617,25 +2698,30 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.5", - "@img/sharp-darwin-x64": "0.33.5", - "@img/sharp-libvips-darwin-arm64": "1.0.4", - "@img/sharp-libvips-darwin-x64": "1.0.4", - "@img/sharp-libvips-linux-arm": "1.0.5", - "@img/sharp-libvips-linux-arm64": "1.0.4", - "@img/sharp-libvips-linux-s390x": "1.0.4", - "@img/sharp-libvips-linux-x64": "1.0.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", - "@img/sharp-libvips-linuxmusl-x64": "1.0.4", - "@img/sharp-linux-arm": "0.33.5", - "@img/sharp-linux-arm64": "0.33.5", - "@img/sharp-linux-s390x": "0.33.5", - "@img/sharp-linux-x64": "0.33.5", - "@img/sharp-linuxmusl-arm64": "0.33.5", - "@img/sharp-linuxmusl-x64": "0.33.5", - "@img/sharp-wasm32": "0.33.5", - "@img/sharp-win32-ia32": "0.33.5", - "@img/sharp-win32-x64": "0.33.5" + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" } }, "node_modules/shebang-command": { @@ -2722,14 +2808,6 @@ "@types/react": ">=16.0.0" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, "node_modules/snowtransfer": { "version": "0.14.2", "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.14.2.tgz", @@ -2941,9 +3019,9 @@ } }, "node_modules/supertape": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/supertape/-/supertape-11.3.0.tgz", - "integrity": "sha512-2LP36xLtxsb3bBYrfvWIilhWpA/vs7/vIgElpsqEhZZ0vcOAMlhMIxH6eHAl5u9KcxGD28IrJrw8lREqeMtZeQ==", + "version": "11.3.1", + "resolved": "https://registry.npmjs.org/supertape/-/supertape-11.3.1.tgz", + "integrity": "sha512-jfo8kUh6ru75tTLuwfpTEjMbkP7/Pllgd/pPkKRWTtRyePxmLIzWjvSryT2j6Af5R6SZm44KJRd0aYhb3w3EEw==", "dev": true, "license": "MIT", "dependencies": { @@ -3216,9 +3294,9 @@ } }, "node_modules/tslib": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", - "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD", "optional": true }, @@ -3459,9 +3537,9 @@ } }, "node_modules/zod": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.5.tgz", - "integrity": "sha512-rcUUZqlLJgBC33IT3PNMgsCq6TzLQEG/Ei/KTCU0PedSWRMAXoOUN+4t/0H+Q8bdnLPdqUYnvboJT0bn/229qg==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", + "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 85530c10..bb9eb3e8 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.14.0", - "discord-api-types": "^0.38.19", + "discord-api-types": "^0.38.31", "domino": "^2.1.6", "enquirer": "^2.4.1", "entities": "^5.0.0", @@ -43,7 +43,7 @@ "htmx.org": "^2.0.4", "lru-cache": "^11.0.2", "prettier-bytes": "^1.0.4", - "sharp": "^0.33.4", + "sharp": "^0.34.5", "snowtransfer": "^0.14.2", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 3d1c918c..0d65b9d4 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -4729,7 +4729,7 @@ slow()("event2message: known and unknown emojis in the end are reuploaded as a s t.deepEqual(testResult, { content: "known unknown: <:hippo:230201364309868544> [:ms_robot_dress:](https://bridge.example.org/download/matrix/cadence.moe/wcouHVjbKJJYajkhJLsyeJAA) and known unknown:", fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAYAAADuFn/PAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAeXRFWHRSYXcACklQVEMgcHJvZmlsZQogICAgICA0Ngoz" + fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAYAAADuFn/PAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAT/klEQVR4nOVcC3CVRZbuS2KAIMpDQt5PQkIScm/uvYRX" }) }) @@ -4754,6 +4754,6 @@ slow()("event2message: all unknown chess emojis are reuploaded as a sprite sheet t.deepEqual(testResult, { content: "testing", fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAYAAAABgCAYAAAAU9KWJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAGXRFWHRTb2Z0d2FyZQB3d3cuaW5rc2NhcGUub3Jnm+48" + fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAYAAAABgCAYAAAAU9KWJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAgAElEQVR4nOx9B3hUVdr/KIpKL2nT0pPpLRNQkdXddV1c" }) }) From 158921d55e8e939fd427e8d32ae18d2673379c59 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Nov 2025 19:08:25 +1300 Subject: [PATCH 015/153] Improve emoji uploading experience --- src/d2m/converters/message-to-event.js | 10 ++++++++++ src/matrix/matrix-command-handler.js | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 93e120e0..9a975d28 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -531,6 +531,16 @@ async function messageToEvent(message, guild, options = {}, di) { message.content = "changed the channel name to **" + message.content + "**" } + // Handle message type 63, new emoji announcement + // @ts-expect-error - should be changed to a DiscordTypes reference once it has been documented + if (message.type === 63) { + const match = message.content.match(/^<(a?):([^:>]{1,64}):([0-9]+)>$/) + assert(match, `message type 63, which announces a new emoji, did not include an emoji. the actual content was: "${message.content}"`) + const name = match[2] + msgtype = "m.emote" + message.content = `added a new emoji, ${message.content} :${name}:` + } + // Forwarded content appears first if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_snapshots?.length) { // Forwarded notice diff --git a/src/matrix/matrix-command-handler.js b/src/matrix/matrix-command-handler.js index 93bc3121..f712ece2 100644 --- a/src/matrix/matrix-command-handler.js +++ b/src/matrix/matrix-command-handler.js @@ -177,7 +177,7 @@ const commands = [{ .addLine(`Ⓜ️ *If you were a Discord user, you wouldn't have permission to create emojis. ${matrixOnlyConclusion}`, `Ⓜ️ If you were a Discord user, you wouldn't have permission to create emojis. ${matrixOnlyConclusion}`, matrixOnlyReason === "CAPACITY") .addLine("[Preview not available in plain text.]", "Preview:") for (const e of toUpload) { - b.add("", `:${e.name}:`) + b.add("", `:${e.name}: :${e.name}:`) } b.addLine("Hit ✅ to add it.") const sent = await api.sendEvent(event.room_id, "m.room.message", { From 56a4fe12864db0cac0f2b49640170c3d3549c40d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 15:28:14 +1300 Subject: [PATCH 016/153] m->d: link too-large files instead of uploading --- src/m2d/converters/event-to-message.js | 59 ++++++-- src/m2d/converters/event-to-message.test.js | 160 ++++++++++++++++++++ 2 files changed, 205 insertions(+), 14 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 61525e2b..fd9289dd 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -8,6 +8,8 @@ const TurndownService = require("@cloudrac3r/turndown") const domino = require("domino") const assert = require("assert").strict const entities = require("entities") +const pb = require("prettier-bytes") +const {tag} = require("@cloudrac3r/html-template-tag") const passthrough = require("../../passthrough") const {sync, db, discord, select, from} = passthrough @@ -483,6 +485,17 @@ const attachmentEmojis = new Map([ ["m.file", "📄"] ]) +/** @param {DiscordTypes.APIGuild} guild */ +function getFileSizeForGuild(guild) { + // guild.features may include strings such as MAX_FILE_SIZE_50_MB and MAX_FILE_SIZE_100_MB, which are the current server boost amounts + const fileSizeFeature = guild?.features.map(f => Number(f.match(/^MAX_FILE_SIZE_([0-9]+)_MB$/)?.[1])).filter(f => f).sort()[0] + if (fileSizeFeature) { + return fileSizeFeature * 1024 * 1024 // discord uses big megabytes + } else { + return 10 * 1024 * 1024 // default file size is 10 MB + } +} + async function getL1L2ReplyLine(called = false) { // @ts-ignore const autoEmoji = new Map(select("auto_emoji", ["name", "emoji_id"], {}, "WHERE name = 'L1' OR name = 'L2'").raw().all()) @@ -539,21 +552,39 @@ async function eventToMessage(event, guild, di) { // Handle images first - might need to handle their `body`/`formatted_body` as well, which will fall through to the text processor let shouldProcessTextEvent = event.type === "m.room.message" && (event.content.msgtype === "m.text" || event.content.msgtype === "m.emote") if (event.type === "m.room.message" && (event.content.msgtype === "m.file" || event.content.msgtype === "m.video" || event.content.msgtype === "m.audio" || event.content.msgtype === "m.image")) { - content = "" - const filename = event.content.filename || event.content.body - if ("file" in event.content) { - // Encrypted - assert.equal(event.content.file.key.alg, "A256CTR") - attachments.push({id: "0", filename}) - pendingFiles.push({name: filename, mxc: event.content.file.url, key: event.content.file.key.k, iv: event.content.file.iv}) - } else { - // Unencrypted - attachments.push({id: "0", filename}) - pendingFiles.push({name: filename, mxc: event.content.url}) - } - // Check if we also need to process a text event for this image - if it has a caption that's different from its filename - if ((event.content.body && event.content.filename && event.content.body !== event.content.filename) || event.content.formatted_body) { + if (!("file" in event.content) && event.content.info?.size > getFileSizeForGuild(guild)) { + // Upload (unencrypted) file as link, because it's too large for Discord + // Do this by constructing a sample Matrix message with the link and then use the text processor to convert that + the original caption. + const url = mxUtils.getPublicUrlForMxc(event.content.url) + assert(url) + const filename = event.content.filename || event.content.body + const newText = new mxUtils.MatrixStringBuilder() + const emoji = attachmentEmojis.has(event.content.msgtype) ? attachmentEmojis.get(event.content.msgtype) + " " : "" + newText.addLine(`${emoji}Uploaded file: ${url} (${pb(event.content.info.size)})`, tag`${emoji}Uploaded file: ${filename} (${pb(event.content.info.size)})`) + // Check if the event has a caption that we need to add as well + if ((event.content.body && event.content.filename && event.content.body !== event.content.filename) || event.content.formatted_body) { + newText.addLine(event.content.body || "", event.content.formatted_body || tag`${event.content.body || ""}`) + } + Object.assign(event.content, newText.get()) shouldProcessTextEvent = true + } else { + // Upload file as file + content = "" + const filename = event.content.filename || event.content.body + if ("file" in event.content) { + // Encrypted + assert.equal(event.content.file.key.alg, "A256CTR") + attachments.push({id: "0", filename}) + pendingFiles.push({name: filename, mxc: event.content.file.url, key: event.content.file.key.k, iv: event.content.file.iv}) + } else { + // Unencrypted + attachments.push({id: "0", filename}) + pendingFiles.push({name: filename, mxc: event.content.url}) + } + // Check if we also need to process a text event for this image - if it has a caption that's different from its filename + if ((event.content.body && event.content.filename && event.content.body !== event.content.filename) || event.content.formatted_body) { + shouldProcessTextEvent = true + } } } if (event.type === "m.sticker") { diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 0d65b9d4..73ca4e96 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -4041,6 +4041,166 @@ test("event2message: evil encrypted image attachment works", async t => { ) }) +test("event2message: large attachments are uploaded if the server boost level is sufficient", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + body: "cool cat.png", + filename: "cool cat.png", + info: { + size: 90_000_000, + mimetype: "image/png", + w: 480, + h: 480, + "xyz.amorgan.blurhash": "URTHsVaTpdj2eKZgkkkXp{pHl7feo@lSl9Z$" + }, + msgtype: "m.image", + url: "mxc://cadence.moe/IvxVJFLEuksCNnbojdSIeEvn" + }, + event_id: "$CXQy3Wmg1A-gL_xAesC1HQcQTEXwICLdSwwUx55FBTI", + room_id: "!BnKuBPCvyfOkhcUjEu:cadence.moe" + }, { + features: ["MAX_FILE_SIZE_100_MB"] + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "", + avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", + attachments: [{id: "0", filename: "cool cat.png"}], + pendingFiles: [{name: "cool cat.png", mxc: "mxc://cadence.moe/IvxVJFLEuksCNnbojdSIeEvn"}] + }] + } + ) +}) + +test("event2message: files too large for Discord are linked as as URL", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + body: "cool cat.png", + filename: "cool cat.png", + info: { + size: 40_000_000, + mimetype: "image/png", + w: 480, + h: 480, + "xyz.amorgan.blurhash": "URTHsVaTpdj2eKZgkkkXp{pHl7feo@lSl9Z$" + }, + msgtype: "m.image", + url: "mxc://cadence.moe/IvxVJFLEuksCNnbojdSIeEvn" + }, + event_id: "$CXQy3Wmg1A-gL_xAesC1HQcQTEXwICLdSwwUx55FBTI", + room_id: "!BnKuBPCvyfOkhcUjEu:cadence.moe" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "🖼️ _Uploaded file: [cool cat.png](https://bridge.example.org/download/matrix/cadence.moe/IvxVJFLEuksCNnbojdSIeEvn) (40 MB)_", + avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: files too large for Discord can have a plaintext caption", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + body: "Cat emoji surrounded by pink hearts", + filename: "cool cat.png", + info: { + size: 40_000_000, + mimetype: "image/png", + w: 480, + h: 480, + "xyz.amorgan.blurhash": "URTHsVaTpdj2eKZgkkkXp{pHl7feo@lSl9Z$" + }, + msgtype: "m.image", + url: "mxc://cadence.moe/IvxVJFLEuksCNnbojdSIeEvn" + }, + event_id: "$CXQy3Wmg1A-gL_xAesC1HQcQTEXwICLdSwwUx55FBTI", + room_id: "!BnKuBPCvyfOkhcUjEu:cadence.moe" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "🖼️ _Uploaded file: [cool cat.png](https://bridge.example.org/download/matrix/cadence.moe/IvxVJFLEuksCNnbojdSIeEvn) (40 MB)_\nCat emoji surrounded by pink hearts", + avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: files too large for Discord can have a formatted caption", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "this event has `formatting`", + filename: "5740.jpg", + format: "org.matrix.custom.html", + formatted_body: "this event has formatting", + info: { + h: 1340, + mimetype: "image/jpeg", + size: 40_000_000, + thumbnail_info: { + h: 670, + mimetype: "image/jpeg", + size: 80157, + w: 540 + }, + thumbnail_url: "mxc://thomcat.rocks/XhLsOCDBYyearsLQgUUrbAvw", + w: 1080, + "xyz.amorgan.blurhash": "KHJQG*55ic-.}?0M58J.9v" + }, + msgtype: "m.image", + url: "mxc://thomcat.rocks/RTHsXmcMPXmuHqVNsnbKtRbh" + }, + origin_server_ts: 1740607766895, + sender: "@cadence:cadence.moe", + type: "m.room.message", + event_id: "$NqNqVgukiQm1nynm9vIr9FIq31hZpQ3udOd7cBIW46U", + room_id: "!BnKuBPCvyfOkhcUjEu:cadence.moe" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "🖼️ _Uploaded file: [5740.jpg](https://bridge.example.org/download/matrix/thomcat.rocks/RTHsXmcMPXmuHqVNsnbKtRbh) (40 MB)_\nthis event has `formatting`", + avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + + test("event2message: stickers work", async t => { t.deepEqual( await eventToMessage({ From 3d34c9d653f316576f6f53b5d33d300ffe12421a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 15:47:42 +1300 Subject: [PATCH 017/153] Try raising errors for failed media --- src/matrix/api.js | 11 +++++++++-- src/matrix/mreq.js | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/matrix/api.js b/src/matrix/api.js index edffc456..6d135bda 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -378,19 +378,26 @@ async function ping() { } /** + * Given an mxc:// URL, and an optional height for thumbnailing, get the file from the content repository. Returns res. * @param {string} mxc - * @param {RequestInit} [init] + * @param {RequestInit & {height?: number | string}} [init] * @return {Promise}>} */ async function getMedia(mxc, init = {}) { const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) assert(mediaParts) - const res = await fetch(`${mreq.baseUrl}/client/v1/media/download/${mediaParts[1]}/${mediaParts[2]}`, { + const downloadOrThumbnail = init.height ? "thumbnail" : "download" + let url = `${mreq.baseUrl}/client/v1/media/${downloadOrThumbnail}/${mediaParts[1]}/${mediaParts[2]}` + if (init.height) url += "?" + new URLSearchParams({height: String(init.height), width: String(init.height)}) + const res = await fetch(url, { headers: { Authorization: `Bearer ${reg.as_token}` }, ...init }) + if (res.status !== 200) { + throw mreq.makeMatrixServerError(res, init) + } if (init.method !== "HEAD") { assert(res.body) } diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index 91798254..bed7951f 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -18,6 +18,21 @@ class MatrixServerError extends Error { } } +/** + * @param {Response} res + * @param {object} opts + */ +async function makeMatrixServerError(res, opts = {}) { + delete opts.headers?.["Authorization"] + if (res.headers.get("content-type") === "application/json") { + return new MatrixServerError(await res.json(), opts) + } else if (res.headers.get("content-type")?.startsWith("text/")) { + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}) + } else { + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}) + } +} + /** * @param {undefined | string | object | streamWeb.ReadableStream | stream.Readable} body * @returns {Promise} @@ -85,6 +100,7 @@ async function withAccessToken(token, callback) { } module.exports.MatrixServerError = MatrixServerError +module.exports.makeMatrixServerError = makeMatrixServerError module.exports.baseUrl = baseUrl module.exports.mreq = mreq module.exports.withAccessToken = withAccessToken From e332786fd0d804c71283d7fb15409ff34922caaa Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 15:57:12 +1300 Subject: [PATCH 018/153] forgot an await --- src/matrix/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/api.js b/src/matrix/api.js index 6d135bda..69ef78dc 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -396,7 +396,7 @@ async function getMedia(mxc, init = {}) { ...init }) if (res.status !== 200) { - throw mreq.makeMatrixServerError(res, init) + throw await mreq.makeMatrixServerError(res, init) } if (init.method !== "HEAD") { assert(res.body) From 408070f32966cd6df54954e825997d164842119f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 16:55:01 +1300 Subject: [PATCH 019/153] Version 3.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 832b2017..4c74e047 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "out-of-your-element", - "version": "3.1.0", + "version": "3.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "out-of-your-element", - "version": "3.1.0", + "version": "3.2.0", "license": "AGPL-3.0-or-later", "dependencies": { "@chriscdn/promise-semaphore": "^3.0.1", diff --git a/package.json b/package.json index bb9eb3e8..22eb9e5f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "out-of-your-element", - "version": "3.1.0", + "version": "3.2.0", "description": "A bridge between Matrix and Discord", "main": "index.js", "repository": { From 1338e6ba882657d2bd1afea633a3ed71cb83e395 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 15 Nov 2025 16:41:02 +1300 Subject: [PATCH 020/153] Update discord-markdown --- package-lock.json | 18 +++++----- scripts/text-probability.js | 65 ++++++++++++++++++++++++++++++++++ src/d2m/actions/create-room.js | 2 +- 3 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 scripts/text-probability.js diff --git a/package-lock.json b/package-lock.json index 4c74e047..bba36f42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -225,9 +225,9 @@ } }, "node_modules/@cloudrac3r/discord-markdown": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/@cloudrac3r/discord-markdown/-/discord-markdown-2.6.7.tgz", - "integrity": "sha512-bWLmBYWaNEDcQfZHDz4jaAxLKA9161ruEnHo3ms6kfRw8uYku/Uz7U1xTmQ2dQF/q1PiuBvM9I37pLiotlQj8A==", + "version": "2.6.8", + "resolved": "https://registry.npmjs.org/@cloudrac3r/discord-markdown/-/discord-markdown-2.6.8.tgz", + "integrity": "sha512-ZrSimHqmLqXR+W3U1n6ge6poAjmQaMzXyWrTkT36znrgKhfuQAYxLBtKTf7m+cmr3VlaDVM2P+iPdSeTeaM0qg==", "license": "MIT", "dependencies": { "simple-markdown": "^0.7.3" @@ -1210,9 +1210,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "22.19.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.0.tgz", - "integrity": "sha512-xpr/lmLPQEj+TUnHmR+Ab91/glhJvsqcjB+yY0Ix9GO70H6Lb4FHH5GeqdOE5btAx7eIMwuHkp4H2MSkLcqWbA==", + "version": "22.19.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.1.tgz", + "integrity": "sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1713,9 +1713,9 @@ } }, "node_modules/discord-api-types": { - "version": "0.38.32", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.32.tgz", - "integrity": "sha512-UhIqkFuUVwBzejLPPWF18qixYPucMf718RnGh1NxZYNS7czXUmcUsWWkzWR7lRWj5pjfj4LwrnN9McvpfLvGqQ==", + "version": "0.38.33", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.33.tgz", + "integrity": "sha512-oau1V7OzrNX8yNi+DfQpoLZCNCv7cTFmvPKwHfMrA/tewsO6iQKrMTzA7pa3iBSj0fED6NlklJ/1B/cC1kI08Q==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" diff --git a/scripts/text-probability.js b/scripts/text-probability.js new file mode 100644 index 00000000..cc934057 --- /dev/null +++ b/scripts/text-probability.js @@ -0,0 +1,65 @@ +// @ts-check + +const Ty = require("../src/types") +const fs = require("fs") +const domino = require("domino") +const repl = require("repl") + +const pres = (() => { + const pres = [] + for (const file of process.argv.slice(2)) { + const data = JSON.parse(fs.readFileSync(file, "utf8")) + /** @type {Ty.Event.Outer<{msgtype?: string}>[]} */ + const events = data.messages + for (const event of events) { + if (event.type !== "m.room.message" || event.content.msgtype !== "m.text") continue + /** @type {Ty.Event.M_Room_Message} */ // @ts-ignore + const content = event.content + if (content.format !== "org.matrix.custom.html") continue + if (!content.formatted_body) continue + + const document = domino.createDocument(content.formatted_body) + // @ts-ignore + for (const pre of document.querySelectorAll("pre").cache) { + const content = pre.textContent + if (content.length < 100) continue + pres.push(content) + } + } + } + return pres +})() + +// @ts-ignore +global.gc() + +/** @param {string} text */ +function probablyFixedWidthIntended(text) { + // if internal spaces are used, seems like they want a fixed-width font + if (text.match(/[^ ] {3,}[^ ]/)) return true + // if characters from Unicode General_Category "Symbol, other" are used, seems like they're doing ascii art and they want a fixed-width font + if (text.match(/\p{So}/v)) return true + // check start of line indentation + let indents = new Set() + for (const line of text.trimEnd().split("\n")) { + indents.add(line.match(/^ */)?.[0].length || 0) + // if there are more than 3 different indents (counting 0) then it's code + if (indents.size >= 3) return true + } + // if everything is indented then it's code + if (!indents.has(0)) return true + // if there is a high proportion of symbols then it's code (this filter works remarkably well on its own) + if ([...text.matchAll(/[\\`~;+|<>%$@*&"'=(){}[\]_^]|\.[a-zA-Z]|[a-z][A-Z]/g)].length / text.length >= 0.04) return true + return false +} + +Object.assign(repl.start().context, {pres, probablyFixedWidthIntended}) + +/* +if it has a lot of symbols then it's code +if it has >=3 levels of indentation then it's code +if it is all indented then it's code +if it has many spaces in a row in the middle then it's ascii art +if it has many non-latin characters then it's language +-> except if they are ascii art characters e.g. ⣿⣿⡇⢸⣿⠃ then it's ascii art +*/ diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index ff5782dc..61e79f30 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -358,7 +358,7 @@ function assertExistsOrAutocreatable(channel, guildID) { * @returns {Promise} room ID */ async function _syncRoom(channelID, shouldActuallySync) { - /** @ts-ignore @type {DiscordTypes.APIGuildChannel} */ + /** @ts-ignore @type {DiscordTypes.APIGuildTextChannel} */ const channel = discord.channels.get(channelID) assert.ok(channel) const guild = channelToGuild(channel) From a2787f7b0b29b940b5bbb8bdbd979b147b46fb28 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 15 Nov 2025 23:04:23 +1300 Subject: [PATCH 021/153] Record failed URL in more error messages --- src/matrix/api.js | 2 +- src/matrix/mreq.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/api.js b/src/matrix/api.js index 69ef78dc..e529d0f8 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -396,7 +396,7 @@ async function getMedia(mxc, init = {}) { ...init }) if (res.status !== 200) { - throw await mreq.makeMatrixServerError(res, init) + throw await mreq.makeMatrixServerError(res, {...init, url}) } if (init.method !== "HEAD") { assert(res.body) diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index bed7951f..384950ed 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -27,9 +27,9 @@ async function makeMatrixServerError(res, opts = {}) { if (res.headers.get("content-type") === "application/json") { return new MatrixServerError(await res.json(), opts) } else if (res.headers.get("content-type")?.startsWith("text/")) { - return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}) + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, opts) } else { - return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}) + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, opts) } } From d0f7d79d78e1014f9f0357dd29a32601f006b242 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 18 Nov 2025 14:48:59 +1300 Subject: [PATCH 022/153] Fix duplicated data appearing on Matrix --- src/d2m/converters/edit-to-changes.js | 8 ++++- src/d2m/converters/edit-to-changes.test.js | 9 +++++ test/data.js | 41 ++++++++++++++++++++++ test/ooye-test-data.sql | 9 +++-- test/test.js | 1 + 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index e5234080..18adc164 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -118,14 +118,20 @@ async function editToChanges(message, guild, api) { eventsToRedact = oldEventRows.map(e => ({old: e})) // If this is a generated embed update, only allow the embeds to be updated, since the system only sends data about events. Ignore changes to other things. + // This also prevents Matrix events that were re-subtyped during conversion (e.g. large image -> text link) from being mistakenly included. if (isGeneratedEmbed) { unchangedEvents.push(...eventsToRedact.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToRedact to unchangedEvents. eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice") unchangedEvents.push(...eventsToReplace.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToReplace to unchangedEvents. eventsToReplace = eventsToReplace.filter(e => e.old.event_subtype === "m.notice") + unchangedEvents.push(...eventsToSend.filter(e => e.msgtype !== "m.notice")) // Move them from eventsToSend to unchangedEvents. + eventsToSend = eventsToSend.filter(e => e.msgtype === "m.notice") // Don't post new generated embeds for messages if it's been a while since the message was sent. Detached embeds look weird. - if (message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 120 * 1000) { // older than 2 minutes ago + const messageTooOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 120 * 1000 // older than 2 minutes ago + // Don't post new generated embeds for messages if the setting was disabled. + const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 + if (messageTooOld || !embedsEnabled) { eventsToSend = eventsToSend.filter(e => e.msgtype !== "m.notice") } } diff --git a/src/d2m/converters/edit-to-changes.test.js b/src/d2m/converters/edit-to-changes.test.js index f6674c04..dfb286c1 100644 --- a/src/d2m/converters/edit-to-changes.test.js +++ b/src/d2m/converters/edit-to-changes.test.js @@ -369,3 +369,12 @@ test("edit2changes: don't generate embed if it's been too long since the message t.deepEqual(promotions, []) t.equal(senderMxid, null) }) + +test("edit2changes: don't generate new data in situations where m->d(->m) subtypes don't match, like large files", async t => { + const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message.large_file_from_matrix) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToReplace, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(promotions, []) + t.equal(senderMxid, null) +}) diff --git a/test/data.js b/test/data.js index c9f58d0e..bc7c192a 100644 --- a/test/data.js +++ b/test/data.js @@ -2657,6 +2657,47 @@ module.exports = { flags: 0, components: [] }, + large_file_from_matrix: { + type: 0, + content: "", + attachments: [ + { + id: "1439351589474140290", + filename: "image.png", + size: 5112701, + url: "https://cdn.discordapp.com/attachments/1438284564815548418/1439351589474140290/image.png?ex=691cd720&is=691b85a0&hm=671d32324ce17acb9708057f9a532a3184d02343747b32b2ad8d330a277d8f65&", + proxy_url: "https://media.discordapp.net/attachments/1438284564815548418/1439351589474140290/image.png?ex=691cd720&is=691b85a0&hm=671d32324ce17acb9708057f9a532a3184d02343747b32b2ad8d330a277d8f65&", + width: 1930, + height: 2522, + content_type: "image/png", + flags: 16, + content_scan_version: 2, + placeholder: "ZhgKDQJ6pIp2B5hmhndoZ0lgiwTJ", + placeholder_version: 1, + spoiler: false + } + ], + embeds: [], + timestamp: new Date().toISOString(), + edited_timestamp: null, + flags: 0, + components: [], + id: "1439351590262800565", + channel_id: "1438284564815548418", + author: { + id: "1438286167706701958", + username: "cibo", + discriminator: "0000", + avatar: "9c9e1d63ce093e76b9cdb99328c91201", + bot: true + }, + pinned: false, + mentions: [], + mention_roles: [], + mention_everyone: false, + tts: false, + webhook_id: "1438286167706701958" + }, simple_reply_to_reply_in_thread: { type: 19, tts: false, diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index b31f2c34..65719548 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -20,7 +20,8 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom ('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS'), ('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL), ('1160894080998461480', '!TqlyQmifxGUggEmdBN:cadence.moe', 'ooyexperiment', NULL, NULL, NULL), -('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL); +('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL), +('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL); INSERT INTO sim (user_id, username, sim_name, mxid) VALUES ('0', 'Matrix Bridge', 'bot', '@_ooye_bot:cadence.moe'), @@ -73,7 +74,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES ('1144874214311067708', '687028734322147344'), ('1339000288144658482', '176333891320283136'), ('1381212840957972480', '112760669178241024'), -('1401760355339862066', '112760669178241024'); +('1401760355339862066', '112760669178241024'), +('1439351590262800565', '1438284564815548418'); INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1), @@ -117,7 +119,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part ('$51gH61p_eJc2RylOdE2lAr4-ogP7dS0WJI62lCFzBvk', 'm.room.message', 'm.text', '1339000288144658482', 0, 0, 0), ('$AfrB8hzXkDMvuoWjSZkDdFYomjInWH7jMBPkwQMN8AI', 'm.room.message', 'm.text', '1381212840957972480', 0, 1, 1), ('$43baKEhJfD-RlsFQi0LB16Zxd8yMqp0HSVL00TDQOqM', 'm.room.message', 'm.image', '1381212840957972480', 1, 0, 1), -('$7P2O_VTQNHvavX5zNJ35DV-dbJB1Ag80tGQP_JzGdhk', 'm.room.message', 'm.text', '1401760355339862066', 0, 0, 0); +('$7P2O_VTQNHvavX5zNJ35DV-dbJB1Ag80tGQP_JzGdhk', 'm.room.message', 'm.text', '1401760355339862066', 0, 0, 0), +('$ielAnR6geu0P1Tl5UXfrbxlIf-SV9jrNprxrGXP3v7M', 'm.room.message', 'm.image', '1439351590262800565', 0, 0, 0); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), diff --git a/test/test.js b/test/test.js index b01f0ce2..2c3902a0 100644 --- a/test/test.js +++ b/test/test.js @@ -28,6 +28,7 @@ reg.namespaces = { } reg.ooye.bridge_origin = "https://bridge.example.org" reg.ooye.time_zone = "Pacific/Auckland" +reg.ooye.max_file_size = 5000000 const sync = new HeatSync({watchFS: false}) From ce30272389c15b56c98019f5688fdf7c9bef6c8b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 18 Nov 2025 15:08:14 +1300 Subject: [PATCH 023/153] Exclude vulnerable version of glob OOYE isn't impacted by the vulnerability. This is just for confidence. --- package-lock.json | 155 +++++++++++++--------------------------------- package.json | 3 + 2 files changed, 45 insertions(+), 113 deletions(-) diff --git a/package-lock.json b/package-lock.json index bba36f42..65abcf45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -876,9 +876,9 @@ } }, "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "dev": true, "license": "MIT", "engines": { @@ -998,17 +998,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=14" - } - }, "node_modules/@popperjs/core": { "version": "2.11.8", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", @@ -1933,22 +1922,41 @@ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" }, "node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-12.0.0.tgz", + "integrity": "sha512-5Qcll1z7IKgHr5g485ePDdHcNQY0k2dtv/bjYy0iuyGxQw2qSOiiXUXJ+AYQpg3HNoUMHqAruX478Jeev7UULw==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.1.1", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" + "path-scurry": "^2.0.0" }, "bin": { "glob": "dist/esm/bin.mjs" }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, "funding": { "url": "https://github.com/sponsors/isaacs" } @@ -2131,19 +2139,19 @@ } }, "node_modules/jackspeak": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", - "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", + "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" }, + "engines": { + "node": "20 || >=22" + }, "funding": { "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" } }, "node_modules/jest-diff": { @@ -2386,28 +2394,22 @@ "dev": true }, "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", + "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" }, "engines": { - "node": ">=16 || 14 >=14.18" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true - }, "node_modules/peek-readable": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", @@ -3058,79 +3060,6 @@ "node": ">=20" } }, - "node_modules/supertape/node_modules/glob": { - "version": "11.0.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.3.tgz", - "integrity": "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.3.1", - "jackspeak": "^4.1.1", - "minimatch": "^10.0.3", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/supertape/node_modules/jackspeak": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", - "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/supertape/node_modules/minimatch": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz", - "integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==", - "dev": true, - "license": "ISC", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/supertape/node_modules/path-scurry": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", - "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/supertape/node_modules/yargs-parser": { "version": "22.0.0", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", diff --git a/package.json b/package.json index 22eb9e5f..b7bce862 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,9 @@ "xxhash-wasm": "^1.0.2", "zod": "^4.0.17" }, + "overrides": { + "glob@<11.1": "^12" + }, "devDependencies": { "@cloudrac3r/tap-dot": "^2.0.3", "@types/node": "^22.17.1", From edd4f988ecdbc4416eaa865f4421d5369d7d2f57 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 19 Nov 2025 16:37:39 +1300 Subject: [PATCH 024/153] Support persistent Matrix sims for webhooks --- src/d2m/actions/register-pk-user.js | 11 +- src/d2m/actions/register-webhook-user.js | 146 ++++++++++++++++++ src/d2m/actions/send-message.js | 22 ++- src/d2m/converters/user-to-mxid.js | 48 +++++- ...025-add-webhook-profile-to-guild-space.sql | 5 + src/db/orm-defs.d.ts | 1 + src/types.d.ts | 7 + src/web/pug/guild.pug | 9 ++ src/web/routes/guild-settings.js | 2 + 9 files changed, 234 insertions(+), 17 deletions(-) create mode 100644 src/d2m/actions/register-webhook-user.js create mode 100644 src/db/migrations/0025-add-webhook-profile-to-guild-space.sql diff --git a/src/d2m/actions/register-pk-user.js b/src/d2m/actions/register-pk-user.js index 27e949ce..e17f0613 100644 --- a/src/d2m/actions/register-pk-user.js +++ b/src/d2m/actions/register-pk-user.js @@ -13,13 +13,6 @@ const file = sync.require("../../matrix/file") /** @type {import("./register-user")} */ const registerUser = sync.require("./register-user") -/** - * @typedef WebhookAuthor Discord API message->author. A webhook as an author. - * @prop {string} username - * @prop {string?} avatar - * @prop {string} id - */ - /** @returns {Promise} */ async function fetchMessage(messageID) { try { @@ -111,7 +104,7 @@ async function ensureSimJoined(pkMessage, roomID) { /** * Generate profile data based on webhook displayname and configured avatar. * @param {Ty.PkMessage} pkMessage - * @param {WebhookAuthor} author + * @param {Ty.WebhookAuthor} author */ async function memberToStateContent(pkMessage, author) { // We prefer to use the member's avatar URL data since the image upload can be cached across channels, @@ -137,7 +130,7 @@ async function memberToStateContent(pkMessage, author) { * 5. Compare against the previously known state content, which is helpfully stored in the database * 6. If the state content has changed, send it to Matrix and update it in the database for next time * @param {string} messageID to call API with - * @param {WebhookAuthor} author for profile data + * @param {Ty.WebhookAuthor} author for profile data * @param {string} roomID room to join member to * @param {boolean} shouldActuallySync whether to actually sync updated user data or just ensure it's joined * @returns {Promise} mxid of the updated sim diff --git a/src/d2m/actions/register-webhook-user.js b/src/d2m/actions/register-webhook-user.js new file mode 100644 index 00000000..869d7d85 --- /dev/null +++ b/src/d2m/actions/register-webhook-user.js @@ -0,0 +1,146 @@ +// @ts-check + +const assert = require("assert") +const {reg} = require("../../matrix/read-registration") +const Ty = require("../../types") + +const passthrough = require("../../passthrough") +const {sync, db, select, from} = passthrough +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/file")} */ +const file = sync.require("../../matrix/file") +/** @type {import("./register-user")} */ +const registerUser = sync.require("./register-user") +/** @type {import("../converters/user-to-mxid")} */ +const userToMxid = sync.require("../converters/user-to-mxid") + +/** + * A sim is an account that is being simulated by the bridge to copy events from the other side. + * @param {string} fakeUserID + * @param {Ty.WebhookAuthor} author + * @returns mxid + */ +async function createSim(fakeUserID, author) { + // Choose sim name + const simName = userToMxid.webhookAuthorToSimName(author) + const localpart = reg.ooye.namespace_prefix + simName + const mxid = `@${localpart}:${reg.ooye.server_name}` + + // Save chosen name in the database forever + db.prepare("INSERT INTO sim (user_id, username, sim_name, mxid) VALUES (?, ?, ?, ?)").run(fakeUserID, author.username, simName, mxid) + + // Register matrix user with that name + try { + await api.register(localpart) + } catch (e) { + // If user creation fails, manually undo the database change. Still isn't perfect, but should help. + // (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.) + db.prepare("DELETE FROM sim WHERE user_id = ?").run(fakeUserID) + throw e + } + return mxid +} + +/** + * Ensure a sim is registered for the user. + * If there is already a sim, use that one. If there isn't one yet, register a new sim. + * @param {string} fakeUserID + * @param {Ty.WebhookAuthor} author + * @returns {Promise} mxid + */ +async function ensureSim(fakeUserID, author) { + let mxid = null + const existing = select("sim", "mxid", {user_id: fakeUserID}).pluck().get() + if (existing) { + mxid = existing + } else { + mxid = await createSim(fakeUserID, author) + } + return mxid +} + +/** + * Ensure a sim is registered for the user and is joined to the room. + * @param {string} fakeUserID + * @param {Ty.WebhookAuthor} author + * @param {string} roomID + * @returns {Promise} mxid + */ +async function ensureSimJoined(fakeUserID, author, roomID) { + // Ensure room ID is really an ID, not an alias + assert.ok(roomID[0] === "!") + + // Ensure user + const mxid = await ensureSim(fakeUserID, author) + + // Ensure joined + const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() + if (!existing) { + try { + await api.inviteToRoom(roomID, mxid) + await api.joinRoom(roomID, mxid) + } catch (e) { + if (e.message.includes("is already in the room.")) { + // Sweet! + } else { + throw e + } + } + db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid) + } + return mxid +} + +/** + * Generate profile data based on webhook displayname and configured avatar. + * @param {Ty.WebhookAuthor} author + */ +async function authorToStateContent(author) { + // We prefer to use the member's avatar URL data since the image upload can be cached across channels, + // unlike the userAvatar URL which is unique per channel, due to the webhook ID being in the URL. + const avatar = file.userAvatar(author) + + const content = { + displayname: author.username, + membership: "join", + } + if (avatar) content.avatar_url = await file.uploadDiscordFileToMxc(avatar) + + return content +} + +/** + * Sync profile data for a sim webhook user. This function follows the following process: + * 1. Create and join the sim to the room if needed + * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before + * 3. Compare against the previously known state content, which is helpfully stored in the database + * 4. If the state content has changed, send it to Matrix and update it in the database for next time + * @param {Ty.WebhookAuthor} author for profile data + * @param {string} roomID room to join member to + * @param {boolean} shouldActuallySync whether to actually sync updated user data or just ensure it's joined + * @returns {Promise} mxid of the updated sim + */ +async function syncUser(author, roomID, shouldActuallySync) { + const fakeUserID = userToMxid.webhookAuthorToFakeUserID(author) + + // Create and join the sim to the room if needed + const mxid = await ensureSimJoined(fakeUserID, author, roomID) + + if (shouldActuallySync) { + // Build current profile data + const content = await authorToStateContent(author) + const currentHash = registerUser._hashProfileContent(content, 0) + const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() + + // Only do the actual sync if the hash has changed since we last looked + if (existingHash !== currentHash) { + await api.sendState(roomID, "m.room.member", mxid, content, mxid) + db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) + } + } + + return mxid +} + +module.exports.syncUser = syncUser diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index b1cb6801..dcbd5767 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -4,7 +4,7 @@ const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") -const { discord, sync, db } = passthrough +const { discord, sync, db, select } = passthrough /** @type {import("../converters/message-to-event")} */ const messageToEvent = sync.require("../converters/message-to-event") /** @type {import("../../matrix/api")} */ @@ -13,6 +13,8 @@ const api = sync.require("../../matrix/api") const registerUser = sync.require("./register-user") /** @type {import("./register-pk-user")} */ const registerPkUser = sync.require("./register-pk-user") +/** @type {import("./register-webhook-user")} */ +const registerWebhookUser = sync.require("./register-webhook-user") /** @type {import("../actions/create-room")} */ const createRoom = sync.require("../actions/create-room") /** @type {import("../../discord/utils")} */ @@ -28,17 +30,23 @@ async function sendMessage(message, channel, guild, row) { const roomID = await createRoom.ensureRoom(message.channel_id) let senderMxid = null - if (!dUtils.isWebhookMessage(message)) { + if (dUtils.isWebhookMessage(message)) { + const useWebhookProfile = select("guild_space", "webhook_profile", {guild_id: guild.id}) ?? 0 + if (row && row.speedbump_webhook_id === message.webhook_id) { + // Handle the PluralKit public instance + if (row.speedbump_id === "466378653216014359") { + senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, true) + } + } else if (useWebhookProfile) { + senderMxid = await registerWebhookUser.syncUser(message.author, roomID, true) + } + } else { + // not a webhook if (message.author.id === discord.application.id) { // no need to sync the bot's own user } else { senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID) } - } else if (row && row.speedbump_webhook_id === message.webhook_id) { - // Handle the PluralKit public instance - if (row.speedbump_id === "466378653216014359") { - senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, true) - } } const events = await messageToEvent.messageToEvent(message, guild, {}, {api, snow: discord.snow}) diff --git a/src/d2m/converters/user-to-mxid.js b/src/d2m/converters/user-to-mxid.js index e0ab1377..c011b928 100644 --- a/src/d2m/converters/user-to-mxid.js +++ b/src/d2m/converters/user-to-mxid.js @@ -2,6 +2,7 @@ const assert = require("assert") const {reg} = require("../../matrix/read-registration") +const Ty = require("../../types") const passthrough = require("../../passthrough") const {select} = passthrough @@ -13,7 +14,7 @@ const SPECIAL_USER_MAPPINGS = new Map([ /** * Downcased and stripped username. Can only include a basic set of characters. * https://spec.matrix.org/v1.6/appendices/#user-identifiers - * @param {import("discord-api-types/v10").APIUser} user + * @param {import("discord-api-types/v10").APIUser | Ty.WebhookAuthor} user * @returns {string} localpart */ function downcaseUsername(user) { @@ -85,4 +86,49 @@ function userToSimName(user) { throw new Error(`Ran out of suggestions when generating sim name. downcased: "${downcased}"`) } +/** + * Webhooks have an ID specific to that webhook, but a single webhook can send messages with any user name. + * The point of this feature (gated by guild_space webhook_profile) is to create persistent Matrix accounts for individual webhook "users". + * This is convenient when using a bridge to a platform that does not assign persistent user IDs (e.g. IRC, Minecraft). + * In this case, webhook "users" are disambiguated by their username (downcased). + * @param {Ty.WebhookAuthor} author + * @returns {string} + */ +function webhookAuthorToFakeUserID(author) { + const downcased = downcaseUsername(author) + return `webhook_${downcased}` +} + +/** + * @param {Ty.WebhookAuthor} author + * @returns {string} + */ +function webhookAuthorToSimName(author) { + if (SPECIAL_USER_MAPPINGS.has(author.id)) { + const error = new Error("Special users should have followed the other code path.") + // @ts-ignore + error.author = author + throw error + } + + // 1. Is sim user already registered? + const fakeUserID = webhookAuthorToFakeUserID(author) + const existing = select("sim", "user_id", {user_id: fakeUserID}).pluck().get() + assert.equal(existing, null, "Shouldn't try to create a new name for an existing sim") + + // 2. Register based on username (could be new or old format) + const downcased = "webhook_" + downcaseUsername(author) + + // Check for conflicts with already registered sims + const matches = select("sim", "sim_name", {}, "WHERE sim_name LIKE ? ESCAPE '@'").pluck().all(downcased + "%") + // Keep generating until we get a suggestion that doesn't conflict + for (const suggestion of generateLocalpartAlternatives([downcased])) { + if (!matches.includes(suggestion)) return suggestion + } + /* c8 ignore next */ + throw new Error(`Ran out of suggestions when generating sim name. downcased: "${downcased}"`) +} + module.exports.userToSimName = userToSimName +module.exports.webhookAuthorToFakeUserID = webhookAuthorToFakeUserID +module.exports.webhookAuthorToSimName = webhookAuthorToSimName diff --git a/src/db/migrations/0025-add-webhook-profile-to-guild-space.sql b/src/db/migrations/0025-add-webhook-profile-to-guild-space.sql new file mode 100644 index 00000000..e629fb81 --- /dev/null +++ b/src/db/migrations/0025-add-webhook-profile-to-guild-space.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +ALTER TABLE guild_space ADD COLUMN webhook_profile INTEGER NOT NULL DEFAULT 0; + +COMMIT; diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index 79fd5010..b0b74a5a 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -52,6 +52,7 @@ export type Models = { privacy_level: number presence: 0 | 1 url_preview: 0 | 1 + webhook_profile: 0 | 1 } guild_active: { diff --git a/src/types.d.ts b/src/types.d.ts index c7cb006f..71358674 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -72,6 +72,13 @@ export type WebhookCreds = { token: string } +/** Discord API message->author. A webhook as an author. */ +export type WebhookAuthor = { + username: string + avatar: string | null + id: string +} + export type PkSystem = { id: string uuid: string diff --git a/src/web/pug/guild.pug b/src/web/pug/guild.pug index 92ffa1b6..c09ca7e6 100644 --- a/src/web/pug/guild.pug +++ b/src/web/pug/guild.pug @@ -124,6 +124,15 @@ block body | Show online statuses on Matrix p.s-description This might cause lag on really big Discord servers. + form.d-flex.ai-center.g16 + #webhook-profile-loading.p8 + - value = !!select("guild_space", "webhook_profile", {guild_id}).pluck().get() + input(type="hidden" name="guild_id" value=guild_id) + input.s-toggle-switch#webhook-profile(name="webhook_profile" type="checkbox" hx-post=rel("/api/webhook-profile") hx-indicator="#webhook-profile-loading" hx-disabled-elt="this" checked=value autocomplete="off") + label.s-label(for="webhook-profile") + | Create persistent Matrix sims for webhooks + p.s-description Useful when using other Discord bridges. Otherwise, not ideal, as sims will clutter the Matrix user list and will never be cleaned up. + if space_id h2.mt48.fs-headline1 Channel setup diff --git a/src/web/routes/guild-settings.js b/src/web/routes/guild-settings.js index b640d364..63dd3ec5 100644 --- a/src/web/routes/guild-settings.js +++ b/src/web/routes/guild-settings.js @@ -74,6 +74,8 @@ as.router.post("/api/autocreate", defineToggle("autocreate", { as.router.post("/api/url-preview", defineToggle("url_preview")) +as.router.post("/api/webhook-profile", defineToggle("webhook_profile")) + as.router.post("/api/presence", defineToggle("presence", { after() { setPresence.guildPresenceSetting.update() From 0f24994af975580485dade42725c5a4c53e7c3ad Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 19 Nov 2025 16:40:40 +1300 Subject: [PATCH 025/153] The database works better if you query it. --- src/d2m/actions/send-message.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index dcbd5767..d1be138c 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -31,7 +31,7 @@ async function sendMessage(message, channel, guild, row) { let senderMxid = null if (dUtils.isWebhookMessage(message)) { - const useWebhookProfile = select("guild_space", "webhook_profile", {guild_id: guild.id}) ?? 0 + const useWebhookProfile = select("guild_space", "webhook_profile", {guild_id: guild.id}).pluck().get() ?? 0 if (row && row.speedbump_webhook_id === message.webhook_id) { // Handle the PluralKit public instance if (row.speedbump_id === "466378653216014359") { From e6c30f80b5de455022fbe0829560f2e7d884c642 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 19 Nov 2025 16:47:35 +1300 Subject: [PATCH 026/153] Configure whether to receive presences --- src/d2m/discord-client.js | 13 ++++++++----- src/matrix/read-registration.js | 3 ++- src/types.d.ts | 2 ++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/d2m/discord-client.js b/src/d2m/discord-client.js index b05d48fb..c84b466d 100644 --- a/src/d2m/discord-client.js +++ b/src/d2m/discord-client.js @@ -20,17 +20,20 @@ class DiscordClient { * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate */ constructor(discordToken, listen = "full") { + /** @type {import("cloudstorm").IClientOptions["intents"]} */ + const intents = [ + "DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING", + "GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS", + "MESSAGE_CONTENT" + ] + if (reg.ooye.receive_presences !== false) intents.push("GUILD_PRESENCES") this.discordToken = discordToken this.snow = new SnowTransfer(discordToken) this.cloud = new CloudStorm(discordToken, { shards: [0], reconnect: true, snowtransferInstance: this.snow, - intents: [ - "DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING", - "GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS", - "MESSAGE_CONTENT", "GUILD_PRESENCES" - ], + intents, ws: { compress: false, encoding: "json" diff --git a/src/matrix/read-registration.js b/src/matrix/read-registration.js index d126851c..9316158d 100644 --- a/src/matrix/read-registration.js +++ b/src/matrix/read-registration.js @@ -57,7 +57,8 @@ function getTemplateRegistration(serverName) { max_file_size: 5000000, content_length_workaround: false, include_user_id_in_mxid: false, - invite: [] + invite: [], + receive_presences: true } } } diff --git a/src/types.d.ts b/src/types.d.ts index 71358674..cafd9bea 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -32,6 +32,7 @@ export type AppServiceRegistrationConfig = { discord_cdn_origin?: string, web_password: string time_zone?: string + receive_presences: boolean } old_bridge?: { as_token: string @@ -64,6 +65,7 @@ export type InitialAppServiceRegistrationConfig = { content_length_workaround: boolean invite: string[] include_user_id_in_mxid: boolean + receive_presences: boolean } } From a441c476f854169b3095e0eea2546b9b8e180b91 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 26 Nov 2025 15:43:48 +1300 Subject: [PATCH 027/153] Support multiple versions of rooms --- src/d2m/actions/announce-thread.js | 2 +- src/d2m/actions/create-room.js | 9 +- src/d2m/actions/delete-message.js | 24 +- src/d2m/actions/edit-message.js | 10 +- src/d2m/actions/register-user.js | 11 +- src/d2m/actions/remove-reaction.js | 17 +- src/d2m/actions/send-message.js | 4 +- src/d2m/converters/message-to-event.js | 30 +- src/d2m/converters/message-to-event.test.js | 4 +- src/d2m/converters/user-to-mxid.js | 5 + src/d2m/discord-packets.js | 8 - src/d2m/event-dispatcher.js | 2 +- src/db/migrations/0017-analyze.sql | 225 --------------- .../migrations/0026-make-rooms-historical.sql | 62 +++++ src/db/migrations/0027-analyze.sql | 256 ++++++++++++++++++ src/db/orm-defs.d.ts | 12 +- src/db/orm.js | 15 + src/db/orm.test.js | 4 +- src/discord/interactions/matrix-info.js | 19 +- src/discord/interactions/permissions.js | 9 +- src/discord/interactions/reactions.js | 2 +- src/m2d/actions/add-reaction.js | 2 +- src/m2d/actions/redact.js | 12 +- src/m2d/actions/send-event.js | 14 +- src/m2d/actions/update-pins.js | 7 +- src/m2d/converters/event-to-message.js | 22 +- src/web/routes/info.js | 18 +- src/web/routes/info.test.js | 56 ++-- test/ooye-test-data.sql | 8 +- 29 files changed, 520 insertions(+), 349 deletions(-) delete mode 100644 src/db/migrations/0017-analyze.sql create mode 100644 src/db/migrations/0026-make-rooms-historical.sql create mode 100644 src/db/migrations/0027-analyze.sql diff --git a/src/d2m/actions/announce-thread.js b/src/d2m/actions/announce-thread.js index 324c7a5b..c8cbf9de 100644 --- a/src/d2m/actions/announce-thread.js +++ b/src/d2m/actions/announce-thread.js @@ -1,6 +1,6 @@ // @ts-check -const assert = require("assert") +const assert = require("assert").strict const passthrough = require("../../passthrough") const {discord, sync, db, select} = passthrough diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 61e79f30..009e31e1 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -228,7 +228,10 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { ...spaceCreationContent }) - db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent) + db.transaction(() => { + db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent) + db.prepare("INSERT INTO historical_channel_room (channel_id, room_id) VALUES (?, ?)").run(channel.id, roomID) + })() return roomID }) @@ -400,7 +403,7 @@ async function _syncRoom(channelID, shouldActuallySync) { } const roomDiff = ks.diffKState(roomKState, channelKState) const roomApply = ks.applyKStateDiffToRoom(roomID, roomDiff) - db.prepare("UPDATE channel_room SET name = ? WHERE room_id = ?").run(channel.name, roomID) + db.prepare("UPDATE channel_room SET name = ? WHERE channel_id = ?").run(channel.name, channel.id) // sync room as space member const spaceApply = _syncSpaceMember(channel, spaceID, roomID, guild.id) @@ -464,7 +467,7 @@ async function unbridgeDeletedChannel(channel, guildID) { // delete room from database db.prepare("DELETE FROM member_cache WHERE room_id = ?").run(roomID) - db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channel.id) // cascades to most other tables, like messages + db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channel.id) // cascades to most other tables, like messages and historical rooms if (!botInRoom) return diff --git a/src/d2m/actions/delete-message.js b/src/d2m/actions/delete-message.js index e9e0b088..39b9fc84 100644 --- a/src/d2m/actions/delete-message.js +++ b/src/d2m/actions/delete-message.js @@ -14,11 +14,13 @@ async function deleteMessage(data) { const row = select("channel_room", ["room_id", "speedbump_checked", "thread_parent"], {channel_id: data.channel_id}).get() if (!row) return - const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all() - db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id) - for (const eventID of eventsToRedact) { + // Assume we can redact from tombstoned rooms. + const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("event_id", "room_id").where({message_id: data.id}).all() + db.prepare("DELETE FROM message_room WHERE message_id = ?").run(data.id) + for (const {event_id, room_id} of eventsToRedact) { // Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs - await api.redactEvent(row.room_id, eventID) + await api.redactEvent(room_id, event_id) } await speedbump.updateCache(row.thread_parent || data.channel_id, row.speedbump_checked) @@ -28,15 +30,17 @@ async function deleteMessage(data) { * @param {import("discord-api-types/v10").GatewayMessageDeleteBulkDispatchData} data */ async function deleteMessageBulk(data) { - const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() - if (!roomID) return + const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get() + if (!row) return const sids = JSON.stringify(data.ids) - const eventsToRedact = from("event_message").pluck("event_id").and("WHERE message_id IN (SELECT value FROM json_each(?))").all(sids) - db.prepare("DELETE FROM message_channel WHERE message_id IN (SELECT value FROM json_each(?))").run(sids) - for (const eventID of eventsToRedact) { + // Assume we can redact from tombstoned rooms. + const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("event_id", "room_id").and("WHERE message_id IN (SELECT value FROM json_each(?))").all(sids) + db.prepare("DELETE FROM message_room WHERE message_id IN (SELECT value FROM json_each(?))").run(sids) + for (const {event_id, room_id} of eventsToRedact) { // Awaiting will make it go slower, but since this could be a long-running operation either way, we want to leave rate limit capacity for other operations - await api.redactEvent(roomID, eventID) + await api.redactEvent(room_id, event_id) } } diff --git a/src/d2m/actions/edit-message.js b/src/d2m/actions/edit-message.js index 7f1bff7d..5970b59c 100644 --- a/src/d2m/actions/edit-message.js +++ b/src/d2m/actions/edit-message.js @@ -3,7 +3,7 @@ const assert = require("assert").strict const passthrough = require("../../passthrough") -const {sync, db, select} = passthrough +const {sync, db, select, from} = passthrough /** @type {import("../converters/edit-to-changes")} */ const editToChanges = sync.require("../converters/edit-to-changes") /** @type {import("./register-pk-user")} */ @@ -19,6 +19,12 @@ const mreq = sync.require("../../matrix/mreq") * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel */ async function editMessage(message, guild, row) { + const historicalRoomOfMessage = from("message_room").join("historical_channel_room", "historical_room_index").where({message_id: message.id}).select("room_id").get() + const currentRoom = from("channel_room").join("historical_channel_room", "room_id").where({channel_id: message.channel_id}).select("room_id", "historical_room_index").get() + assert(currentRoom) + + if (historicalRoomOfMessage && historicalRoomOfMessage.room_id !== currentRoom.room_id) return // tombstoned rooms should not have new events (including edits) sent to them + let {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api) if (row && row.speedbump_webhook_id === message.webhook_id) { @@ -61,7 +67,7 @@ async function editMessage(message, guild, row) { // 4. Send all the things. if (eventsToSend.length) { - db.prepare("INSERT OR IGNORE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id) + db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(message.id, currentRoom.historical_room_index) } for (const content of eventsToSend) { const eventType = content.$type diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index 674853a2..4bb8e982 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -4,10 +4,9 @@ const assert = require("assert").strict const {reg} = require("../../matrix/read-registration") const DiscordTypes = require("discord-api-types/v10") const Ty = require("../../types") -const mixin = require("@cloudrac3r/mixin-deep") const passthrough = require("../../passthrough") -const {discord, sync, db, select} = passthrough +const {discord, sync, db, from, select} = passthrough /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** @type {import("../../matrix/file")} */ @@ -222,7 +221,8 @@ async function syncUser(user, member, channel, guild, roomID) { * @param {string} roomID */ async function syncAllUsersInRoom(roomID) { - const mxids = select("sim_member", "mxid", {room_id: roomID}).pluck().all() + const users = from("sim_member").join("sim", "mxid") + .where({room_id: roomID}).and("and user_id not like '%-%' and user_id not like '%\\_%' escape '\\'").pluck("user_id").all() const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() assert.ok(typeof channelID === "string") @@ -234,10 +234,7 @@ async function syncAllUsersInRoom(roomID) { /** @ts-ignore @type {DiscordTypes.APIGuild} */ const guild = discord.guilds.get(guildID) - for (const mxid of mxids) { - const userID = select("sim", "user_id", {mxid}).pluck().get() - assert.ok(typeof userID === "string") - + for (const userID of users) { /** @ts-ignore @type {Required} */ const member = await discord.snow.guild.getGuildMember(guildID, userID) /** @ts-ignore @type {Required} user */ diff --git a/src/d2m/actions/remove-reaction.js b/src/d2m/actions/remove-reaction.js index 06c4b59d..0f7eec9c 100644 --- a/src/d2m/actions/remove-reaction.js +++ b/src/d2m/actions/remove-reaction.js @@ -4,7 +4,7 @@ const Ty = require("../../types") const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") -const {discord, sync, db, select} = passthrough +const {discord, sync, db, from, select} = passthrough /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** @type {import("../converters/emoji-to-key")} */ @@ -18,12 +18,15 @@ const converter = sync.require("../converters/remove-reaction") * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData | DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData | DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data */ async function removeSomeReactions(data) { - const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() - if (!roomID) return - const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get() - if (!eventIDForMessage) return + const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get() + if (!row) return - const reactions = await api.getFullRelations(roomID, eventIDForMessage, "m.annotation") + const eventReactedTo = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .where({message_id: data.message_id, reaction_part: 0}).select("event_id", "room_id").get() + if (!eventReactedTo) return + + // Due to server restrictions, all relations (i.e. reactions) have to be in the same room as the original event. + const reactions = await api.getFullRelations(eventReactedTo.room_id, eventReactedTo.event_id, "m.annotation") // Run the proper strategy and any strategy-specific database changes const removals = await @@ -33,7 +36,7 @@ async function removeSomeReactions(data) { // Redact the events and delete individual stored events in the database for (const removal of removals) { - await api.redactEvent(roomID, removal.eventID, removal.mxid) + await api.redactEvent(eventReactedTo.room_id, removal.eventID, removal.mxid) if (removal.hash) db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(removal.hash) } } diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index d1be138c..e9c5105a 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -28,6 +28,8 @@ const dUtils = sync.require("../../discord/utils") */ async function sendMessage(message, channel, guild, row) { const roomID = await createRoom.ensureRoom(message.channel_id) + const historicalRoomIndex = select("historical_channel_room", "historical_room_index", {room_id: roomID}).pluck().get() + assert(historicalRoomIndex) let senderMxid = null if (dUtils.isWebhookMessage(message)) { @@ -52,7 +54,7 @@ async function sendMessage(message, channel, guild, row) { const events = await messageToEvent.messageToEvent(message, guild, {}, {api, snow: discord.snow}) const eventIDs = [] if (events.length) { - db.prepare("INSERT OR IGNORE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id) + db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(message.id, historicalRoomIndex) if (senderMxid) api.sendTyping(roomID, false, senderMxid).catch(() => {}) } for (const event of events) { diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 9a975d28..bafe0187 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -265,7 +265,7 @@ async function messageToEvent(message, guild, options = {}, di) { - So make sure we don't do anything in this case. */ const mentions = {} - /** @type {{event_id: string, room_id: string, source: number}?} */ + /** @type {{event_id: string, room_id: string, source: number, channel_id: string}?} */ let repliedToEventRow = null let repliedToUnknownEvent = false let repliedToEventSenderMxid = null @@ -280,9 +280,9 @@ async function messageToEvent(message, guild, options = {}, di) { // Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions // (Still need to do scenarios 1 and 2 part B, and scenario 3.) if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) { - const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id) + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id) if (row) { - repliedToEventRow = row + repliedToEventRow = Object.assign(row, {channel_id: row.reference_channel_id}) } else if (message.referenced_message) { repliedToUnknownEvent = true } @@ -294,7 +294,7 @@ async function messageToEvent(message, guild, options = {}, di) { assert(message.embeds[0].description) const match = message.embeds[0].description.match(/\/channels\/[0-9]*\/[0-9]*\/([0-9]{2,})/) if (match) { - const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1]) + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1]) if (row) { /* we generate a partial referenced_message based on what PK provided. we don't need everything, since this will only be used for further message-to-event converting. @@ -313,7 +313,7 @@ async function messageToEvent(message, guild, options = {}, di) { } } message.embeds.shift() - repliedToEventRow = row + repliedToEventRow = Object.assign(row, {channel_id: row.reference_channel_id}) } } } @@ -485,8 +485,12 @@ async function messageToEvent(message, guild, options = {}, di) { const {body: repliedToBody, html: repliedToHtml} = await transformContent(repliedToContent) if (repliedToEventRow) { // Generate a reply pointing to the Matrix event we found - html = `
In reply to ${repliedToUserHtml}` - + `
${repliedToHtml}
` + const latestRoomID = select("channel_room", "room_id", {channel_id: repliedToEventRow.channel_id}).pluck().get() // native replies don't work across room upgrades, so make sure the old and new message are in the same room + html = + (latestRoomID === repliedToEventRow.room_id ? "" : "") + + `
In reply to ${repliedToUserHtml}` + + `
${repliedToHtml}
` + + (latestRoomID === repliedToEventRow.room_id ? "" : "") + html body = (`${repliedToDisplayName}: ` // scenario 1 part B for mentions + repliedToBody).split("\n").map(line => "> " + line).join("\n") @@ -544,21 +548,23 @@ async function messageToEvent(message, guild, options = {}, di) { // Forwarded content appears first if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_snapshots?.length) { // Forwarded notice - const eventID = select("event_message", "event_id", {message_id: message.message_reference.message_id}).pluck().get() + const event = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("event_id", "room_id").where({message_id: message.message_reference.message_id}).get() const room = select("channel_room", ["room_id", "name", "nick"], {channel_id: message.message_reference.channel_id}).get() const forwardedNotice = new mxUtils.MatrixStringBuilder() if (room) { const roomName = room && (room.nick || room.name) - const via = await getViaServersMemo(room.room_id) - if (eventID) { + if (event) { + const via = await getViaServersMemo(event.room_id) forwardedNotice.addLine( `[🔀 Forwarded from #${roomName}]`, - tag`🔀 Forwarded from ${roomName}` + tag`🔀 Forwarded from ${roomName} [jump to event]` ) } else { + const via = await getViaServersMemo(room.room_id) forwardedNotice.addLine( `[🔀 Forwarded from #${roomName}]`, - tag`🔀 Forwarded from ${roomName}` + tag`🔀 Forwarded from ${roomName} [jump to room]` ) } } else { diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index ee4ec037..cae88b3e 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -1152,7 +1152,7 @@ test("message2event: constructed forwarded message", async t => { body: "[🔀 Forwarded from #wonderland]" + "\n» What's cooking, good looking? :hipposcope:", format: "org.matrix.custom.html", - formatted_body: `🔀 Forwarded from wonderland` + formatted_body: `🔀 Forwarded from wonderland [jump to event]` + `
What's cooking, good looking? :hipposcope:
`, "m.mentions": {}, msgtype: "m.notice", @@ -1210,7 +1210,7 @@ test("message2event: constructed forwarded text", async t => { body: "[🔀 Forwarded from #amanda-spam]" + "\n» What's cooking, good looking?", format: "org.matrix.custom.html", - formatted_body: `🔀 Forwarded from amanda-spam` + formatted_body: `🔀 Forwarded from amanda-spam [jump to room]` + `
What's cooking, good looking?
`, "m.mentions": {}, msgtype: "m.notice", diff --git a/src/d2m/converters/user-to-mxid.js b/src/d2m/converters/user-to-mxid.js index c011b928..f7a49b10 100644 --- a/src/d2m/converters/user-to-mxid.js +++ b/src/d2m/converters/user-to-mxid.js @@ -99,6 +99,10 @@ function webhookAuthorToFakeUserID(author) { return `webhook_${downcased}` } +function isWebhookUserID(userID) { + return userID.match(/^webhook_[a-z90-9._=/-]+$/) +} + /** * @param {Ty.WebhookAuthor} author * @returns {string} @@ -132,3 +136,4 @@ function webhookAuthorToSimName(author) { module.exports.userToSimName = userToSimName module.exports.webhookAuthorToFakeUserID = webhookAuthorToFakeUserID module.exports.webhookAuthorToSimName = webhookAuthorToSimName +module.exports.isWebhookUserID = isWebhookUserID diff --git a/src/d2m/discord-packets.js b/src/d2m/discord-packets.js index 017d50ee..ed45324e 100644 --- a/src/d2m/discord-packets.js +++ b/src/d2m/discord-packets.js @@ -6,10 +6,6 @@ const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../passthrough") const {sync, db} = passthrough -function populateGuildID(guildID, channelID) { - db.prepare("UPDATE channel_room SET guild_id = ? WHERE channel_id = ?").run(guildID, channelID) -} - const utils = { /** * @param {import("./discord-client")} client @@ -41,14 +37,12 @@ const utils = { channel.guild_id = message.d.id arr.push(channel.id) client.channels.set(channel.id, channel) - populateGuildID(message.d.id, channel.id) } for (const thread of message.d.threads || []) { // @ts-ignore thread.guild_id = message.d.id arr.push(thread.id) client.channels.set(thread.id, thread) - populateGuildID(message.d.id, thread.id) } if (listen === "full") { @@ -112,7 +106,6 @@ const utils = { } else if (message.t === "THREAD_CREATE") { client.channels.set(message.d.id, message.d) if (message.d["guild_id"]) { - populateGuildID(message.d["guild_id"], message.d.id) const channels = client.guildChannelMap.get(message.d["guild_id"]) if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) } @@ -140,7 +133,6 @@ const utils = { } else if (message.t === "CHANNEL_CREATE") { client.channels.set(message.d.id, message.d) if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have - populateGuildID(message.d["guild_id"], message.d.id) const channels = client.guildChannelMap.get(message.d["guild_id"]) if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) } diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 49352d79..bf7efb9a 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -67,7 +67,7 @@ module.exports = { async checkMissedMessages(client, guild) { if (guild.unavailable) return const bridgedChannels = select("channel_room", "channel_id").pluck().all() - const preparedExists = db.prepare("SELECT channel_id FROM message_channel WHERE channel_id = ? LIMIT 1") + const preparedExists = from("message_room").join("historical_channel_room", "historical_room_index").pluck("message_id").and("WHERE reference_channel_id = ? LIMIT 1").prepare() const preparedGet = select("event_message", "event_id", {}, "WHERE message_id = ?").pluck() /** @type {(DiscordTypes.APIChannel & {type: DiscordTypes.GuildChannelType})[]} */ let channels = [] diff --git a/src/db/migrations/0017-analyze.sql b/src/db/migrations/0017-analyze.sql deleted file mode 100644 index 802fca20..00000000 --- a/src/db/migrations/0017-analyze.sql +++ /dev/null @@ -1,225 +0,0 @@ --- https://www.sqlite.org/lang_analyze.html - -BEGIN TRANSACTION; - -ANALYZE sqlite_schema; - -DELETE FROM "sqlite_stat1"; -INSERT INTO "sqlite_stat1" ("tbl","idx","stat") VALUES ('sim','sim','625 1'), - ('reaction','reaction','3242 1'), - ('channel_room','channel_room','389 1'), - ('channel_room','sqlite_autoindex_channel_room_1','389 1'), - ('media_proxy','media_proxy','5068 1'), - ('sim_proxy','sim_proxy','36 1'), - ('webhook','webhook','155 1'), - ('member_cache','member_cache','784 3 1'), - ('member_power','member_power','1 1 1'), - ('file','file','21862 1'), - ('message_channel','message_channel','366884 1'), - ('lottie','lottie','19 1'), - ('event_message','event_message','382920 1 1'), - ('migration',NULL,'1'), - ('sim_member','sim_member','2871 7 1'), - ('guild_space','guild_space','32 1'), - ('guild_active','guild_active','34 1'), - ('emoji','emoji','2563 1'), - ('auto_emoji','auto_emoji','3 1'); - -DELETE FROM "sqlite_stat4"; -INSERT INTO "sqlite_stat4" ("tbl","idx","neq","nlt","ndlt","sample") VALUES ('sim','sim','1','69','69',X'0231313137363631373038303932333039353039'), - ('sim','sim','1','139','139',X'0231313530383936363934333439373931323332'), - ('sim','sim','1','209','209',X'0231323231383737363334373737323139303732'), - ('sim','sim','1','279','279',X'0231333039313431353735353334313136383636'), - ('sim','sim','1','349','349',X'0231333935343433383235363034313635363434'), - ('sim','sim','1','419','419',X'0231353335363239373830383338353134373030'), - ('sim','sim','1','489','489',X'0231363930333339333730353930363636383034'), - ('sim','sim','1','559','559',X'0231383535353736303637393137323137383133'), - ('reaction','reaction','1','360','360',X'020699d5faceefb5fb4f'), - ('reaction','reaction','1','721','721',X'0206b61095e98b6b2fb1'), - ('reaction','reaction','1','1082','1082',X'0206d1dcb418603a5eaa'), - ('reaction','reaction','1','1443','1443',X'0206ef9fc42b9df746ad'), - ('reaction','reaction','1','1804','1804',X'02060f38c1f98f130605'), - ('reaction','reaction','1','2165','2165',X'02062b53df6dab7b1067'), - ('reaction','reaction','1','2526','2526',X'020645dd7e7f60c4aac7'), - ('reaction','reaction','1','2887','2887',X'0206658d2fe735805979'), - ('channel_room','channel_room','1','43','43',X'023331313434393131333330393139333231363330'), - ('channel_room','channel_room','1','87','87',X'023331313835343033343830303934303335393738'), - ('channel_room','channel_room','1','131','131',X'023331323139353036353836343139303638393839'), - ('channel_room','channel_room','1','175','175',X'023331323336353538333034323331303334393630'), - ('channel_room','channel_room','1','219','219',X'023331323933373932323135333930323234343235'), - ('channel_room','channel_room','1','263','263',X'023331333333323139363936393333323038303937'), - ('channel_room','channel_room','1','307','307',X'0231343835363635393733363433333738363938'), - ('channel_room','channel_room','1','351','351',X'0231373039303432313039353632323234363731'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','6 6','6 6',X'034b3321416a6c4c49464e6248646474424a6d4d73503a636164656e63652e6d6f6531313531333434383735363139343833373233'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','34 34','34 34',X'034b3321474b4a63424a6b527a47634e4855686c50613a636164656e63652e6d6f6531303237393433323532323237323630343637'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','43 43','43 43',X'034b3121484b50534d62736d694673506d6268414f513a636164656e63652e6d6f65313931343837343839393433343034353434'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','58 58','58 58',X'034b33214a4479425a685545706874784f6e6f6569513a636164656e63652e6d6f6531323937323836373434353633313236333532'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','87 87','87 87',X'034b33214e544d724e686e715271695755654d494d523a636164656e63652e6d6f6531323235323434353738393939373031353536'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','108 108','108 108',X'034b332151444e44796656674e7657565345656876713a636164656e63652e6d6f6531313432333134303935353535363435343830'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','131 131','131 131',X'034b3121544171536b575752654b43506f584c6a75483a636164656e63652e6d6f65383737303730363531343733363631393532'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','175 175','175 175',X'034b3321594249486864714e697255585941587845563a636164656e63652e6d6f6531323335303831373939353936373639333730'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','177 177','177 177',X'034b3321594b46454e79716667696951686956496b533a636164656e63652e6d6f6531323934363237303431343530333933373034'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','186 186','186 186',X'034b3321596f54644f55766a53765349767266716c653a636164656e63652e6d6f6531323734313936373733383435333430323933'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','202 202','202 202',X'034b3121625877616673695372655647676470535a463a636164656e63652e6d6f65373339303137363739373936343336393932'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','208 208','208 208',X'034b3321634a4b6843764943795377717a47634551423a636164656e63652e6d6f6531323732363632303331323238373331343834'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','219 219','219 219',X'034b3121656455786a56647a6755765844554951434b3a636164656e63652e6d6f65343937313631333530393334353630373738'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','242 242','242 242',X'034b31216a4d746e6e6f51414e4278466a486458494d3a636164656e63652e6d6f65373634353135323932303539323731313939'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','263 263','263 263',X'034b31216c7a776870666a5a6e59797468656a7453483a636164656e63652e6d6f65383838343831373132383438343030343534'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','264 264','264 264',X'034b33216d454c5846716a426958726d7558796943723a636164656e63652e6d6f6531313936393134373631303430393234373432'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','268 268','268 268',X'034b33216d557765577571546761574a767769576a653a636164656e63652e6d6f6531323936373131393236333032333830303834'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','291 291','291 291',X'034b3321704761494e45534643587a634e42497a724e3a636164656e63652e6d6f6531303237343531333333333533313532353232'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','306 306','306 306',X'034b3321717646656248564f4b6876454e54494563763a636164656e63652e6d6f6531323737373238383139323232303230313436'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','307 307','307 307',X'034b332171767370666d716f476449634a66794c506c3a636164656e63652e6d6f6531323936393138333638393539343633343735'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','351 351','351 351',X'034b3321774e7a7741724a47796f4c5168426f544e4b3a636164656e63652e6d6f6531303238303436373930333435333739383930'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','368 368','368 368',X'034b3321794e6d504c7765654a69756570725a677a733a636164656e63652e6d6f6531323531393631373233373731313632363234'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','376 376','376 376',X'034b3121796a4879795772466f704c66646878564e423a636164656e63652e6d6f65333336313537353037303734353233313336'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','379 379','379 379',X'034b31217a4c4f6b62766b44587551465948594555673a636164656e63652e6d6f65393933383838313433373030393330363331'), - ('media_proxy','media_proxy','1','563','563',X'02069e6054680b610946'), - ('media_proxy','media_proxy','1','1127','1127',X'0206bb489b717c9320e4'), - ('media_proxy','media_proxy','1','1691','1691',X'0206d75f602775b7a27c'), - ('media_proxy','media_proxy','1','2255','2255',X'0206f2c705ddca4e2b14'), - ('media_proxy','media_proxy','1','2819','2819',X'02061060db7a5151967b'), - ('media_proxy','media_proxy','1','3383','3383',X'02062cc47366f7550d22'), - ('media_proxy','media_proxy','1','3947','3947',X'020647d275ec0d781fc7'), - ('media_proxy','media_proxy','1','4511','4511',X'02066402024a7ea38249'), - ('sim_proxy','sim_proxy','1','4','4',X'025531316564343731342d636635652d346333372d393331382d376136353266383732636634'), - ('sim_proxy','sim_proxy','1','9','9',X'025533346636333932642d323263372d346337382d393063372d326536323734313535613266'), - ('sim_proxy','sim_proxy','1','14','14',X'025535396662363131392d626133392d346565382d393738612d386432376366303631393633'), - ('sim_proxy','sim_proxy','1','19','19',X'025539373066366536332d646234632d346531342d383063362d336639343938643961363665'), - ('sim_proxy','sim_proxy','1','24','24',X'025561636231613335642d313336662d343362332d626365622d326566646634616265306436'), - ('sim_proxy','sim_proxy','1','29','29',X'025563316635623735392d336136342d343633342d623634632d643461656436316539656632'), - ('sim_proxy','sim_proxy','1','34','34',X'025566323230373135632d633436332d343532622d626233612d373662646662306365353537'), - ('webhook','webhook','1','17','17',X'023331313532383834313435343038373230393936'), - ('webhook','webhook','1','35','35',X'023331313939303936333434333830343631313138'), - ('webhook','webhook','1','53','53',X'023331323331383036353337373032353736313938'), - ('webhook','webhook','1','71','71',X'023331323933373836383939343238383036363536'), - ('webhook','webhook','1','89','89',X'023331333132363031353130363535353537373132'), - ('webhook','webhook','1','107','107',X'0231323937323734313733303636333133373339'), - ('webhook','webhook','1','125','125',X'0231353239313736313536333938363832313137'), - ('webhook','webhook','1','143','143',X'0231363837303238373334333232313437333434'), - ('member_cache','member_cache','4 1','73 74','48 74',X'034b3921496f4866536e67625a6762747061747a494e3a636164656e63652e6d6f65406875636b6c65746f6e3a636164656e63652e6d6f65'), - ('member_cache','member_cache','2 1','86 87','57 87',X'034b2d214b5169714663546e764f6f4f424475746a7a3a636164656e63652e6d6f6540726e6c3a636164656e63652e6d6f65'), - ('member_cache','member_cache','4 1','101 104','68 104',X'034b43214e446249714e704a795076664b526e4e63723a636164656e63652e6d6f6540776f756e6465645f77617272696f723a6d61747269782e6f7267'), - ('member_cache','member_cache','4 1','110 113','73 113',X'034b3b214f485844457370624d485348716c4445614f3a636164656e63652e6d6f6540717561647261646963616c3a6d61747269782e6f7267'), - ('member_cache','member_cache','5 1','171 175','111 175',X'034b3b215450616f6a5454444446444847776c7276743a636164656e63652e6d6f6540766962656973766572796f3a6d61747269782e6f7267'), - ('member_cache','member_cache','39 1','180 208','116 208',X'034b4d2154716c79516d69667847556767456d64424e3a636164656e63652e6d6f6540726f626c6b796f6772653a6372616674696e67636f6d72616465732e6e6574'), - ('member_cache','member_cache','4 1','231 231','126 231',X'034b3b2156624f77675559777146614e4c5345644e413a636164656e63652e6d6f654061666c6f7765723a73796e646963617465642e676179'), - ('member_cache','member_cache','9 1','262 263','141 263',X'034b3b21594b46454e79716667696951686956496b533a636164656e63652e6d6f654062656e6d61633a636861742e62656e6d61632e78797a'), - ('member_cache','member_cache','3 1','283 283','149 283',X'034b35215a615a4d78456f52724d6d4e49554d79446c3a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), - ('member_cache','member_cache','88 1','307 351','166 351',X'034b3b2163427874565278446c5a765356684a58564b3a636164656e63652e6d6f65406a61736b617274683a736c656570696e672e746f776e'), - ('member_cache','member_cache','11 1','408 415','177 415',X'034b5121654856655270706e6c6f57587177704a6e553a636164656e63652e6d6f65406a61636b736f6e6368656e3636363a6a61636b736f6e6368656e3636362e636f6d'), - ('member_cache','member_cache','7 1','423 424','181 424',X'034b4b2165724f7079584e465a486a48724568784e583a636164656e63652e6d6f6540616d796973636f6f6c7a3a6d61747269782e6174697573616d792e636f6d'), - ('member_cache','member_cache','96 1','436 439','187 439',X'034b4b21676865544b5a7451666c444e7070684c49673a636164656e63652e6d6f6540616c65783a73706163652e67616d65727374617665726e2e6f6e6c696e65'), - ('member_cache','member_cache','96 1','436 527','187 527',X'034b3121676865544b5a7451666c444e7070684c49673a636164656e63652e6d6f654078796c6f626f6c3a616d6265722e74656c'), - ('member_cache','member_cache','10 1','546 555','197 555',X'0351312169537958674e7851634575586f587073536e3a707573737468656361742e6f726740797562697175653a6e6f70652e63686174'), - ('member_cache','member_cache','13 1','594 601','224 601',X'034b2b216c7570486a715444537a774f744d59476d493a636164656e63652e6d6f6540656c6c69753a68617368692e7265'), - ('member_cache','member_cache','2 1','614 615','229 615',X'034b2f216d584978494644676c4861734e53427371773a636164656e63652e6d6f654077696e673a666561746865722e6f6e6c'), - ('member_cache','member_cache','4 1','616 619','230 619',X'034b2f216d616767455367755a427147425a74536e723a636164656e63652e6d6f654077696e673a666561746865722e6f6e6c'), - ('member_cache','member_cache','4 1','659 660','259 660',X'034b332172454f73706e5971644f414c4149466e69563a636164656e63652e6d6f6540656c797369613a636164656e63652e6d6f65'), - ('member_cache','member_cache','4 1','699 701','284 701',X'034b3521766e717a56767678534a586c5a504f5276533a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), - ('member_cache','member_cache','1 1','703 703','285 703',X'034b3521767165714c474851616842464a56566779483a636164656e63652e6d6f654063696465723a6361746769726c2e636c6f7564'), - ('member_cache','member_cache','4 1','705 705','287 705',X'034b35217750454472596b77497a6f744e66706e57503a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), - ('member_cache','member_cache','35 1','709 709','288 709',X'034b2d2177574f667376757356486f4e4e567242585a3a636164656e63652e6d6f654061613a6361747669626572732e6d65'), - ('member_cache','member_cache','14 1','747 749','291 749',X'034b3721776c534544496a44676c486d42474b7254703a636164656e63652e6d6f654062616461746e616d65733a62616461742e646576'), - ('member_power','member_power','1 1','0 0','0 0',X'03350f40636164656e63653a636164656e63652e6d6f652a'), - ('file','file','1','2429','2429',X'03815f68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f313039393033313838373530303033343038382f313333313336303134333238333036303833372f50584c5f32303235303132315f3230323934323137372e6a7067'), - ('file','file','1','4859','4859',X'03817568747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f313134353832313533383832323637323436362f313330323232393131303834373936373331352f53637265656e73686f745f32303234313130325f3034313332365f5265646469742e6a7067'), - ('file','file','1','7289','7289',X'03815968747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f313231393439383932363436363636323433302f313239373634363930353038353636313234362f494d475f32303234313032305f3135323230302e6a7067'), - ('file','file','1','9719','9719',X'03814168747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3135393136353731343139343735393638302f313236383537333933363531343337313635392f494d475f353433362e6a7067'), - ('file','file','1','12149','12149',X'03813b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3236363736373539303634313233383032372f313237323430333931313939383730313630392f696d6167652e706e67'), - ('file','file','1','14579','14579',X'03816b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3539383730363933323736303434343936392f313237373532343532343330383632373437372f45585445524e414c5f454449545f323032345f4d5f64726166745f322e646f6378'), - ('file','file','1','17009','17009',X'03815768747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3635353231363137333639363238363734362f313333333630333132383634313036303938342f323032352d30312d32375f31372e30312e31352e706e67'), - ('file','file','1','19439','19439',X'027f68747470733a2f2f63646e2e646973636f72646170702e636f6d2f656d6f6a69732f313230323936303730343936313931323836322e706e67'), - ('message_channel','message_channel','1','40764','40764',X'023331313630333434353733303030383232383735'), - ('message_channel','message_channel','1','81529','81529',X'023331313830323437393130343238393837343333'), - ('message_channel','message_channel','1','122294','122294',X'023331313938303533383732383337363131363230'), - ('message_channel','message_channel','1','163059','163059',X'023331323237373739373330333839303738303537'), - ('message_channel','message_channel','1','203824','203824',X'023331323437303438333039303031313538373137'), - ('message_channel','message_channel','1','244589','244589',X'023331323635363939353734333034303830303034'), - ('message_channel','message_channel','1','285354','285354',X'023331323835343637363238323434313732383234'), - ('message_channel','message_channel','1','326119','326119',X'023331333038373932333935313031333736353732'), - ('lottie','lottie','1','2','2',X'0231373439303534363630373639323138363331'), - ('lottie','lottie','1','5','5',X'0231373534313037353339323030363731373635'), - ('lottie','lottie','1','8','8',X'0231373936313430363338303933343433303932'), - ('lottie','lottie','1','11','11',X'0231373936313431373032363935343835353030'), - ('lottie','lottie','1','14','14',X'0231383136303837373932323931323832393434'), - ('lottie','lottie','1','17','17',X'0231383233393736313032393736323930383636'), - ('event_message','event_message','11 1','14788 14796','14356 14796',X'03336531313532303033373639343137303839303434246d44714a474b3530424a715170394273684e534d64365f3768494354776e70362d793130786d6669766563'), - ('event_message','event_message','11 1','33806 33815','32914 33815',X'033365313135373630333638373035373836363736322459526f7139484b376e55677a397668796f6e3053424a49497978497a5750734e4f6e39756f765644664d45'), - ('event_message','event_message','1 1','42546 42546','41335 42546',X'03336531313630363236383737353835363938393237245033436e4f6d6a35462d6939454e4e79586b70796f5679306b74324a654464764276326e746d33566a4355'), - ('event_message','event_message','2 1','85093 85093','81999 85093',X'0333653131383039393939363531323930343831303424496748666562784533746e623047412d534f7176594a354e4e55385735706c68523159676854636a554734'), - ('event_message','event_message','11 1','116157 116165','111525 116165',X'03336531313933363837333730363137333237363536245a32305734766c737079566e387a6e2d526a6f64694a51745f5a7851644f5f33744e415169453755356477'), - ('event_message','event_message','1 1','127640 127640','122328 127640',X'0333653132303132353637313733373633303332323624702d626f415672476a4b45327a7158664f3738387a5a597a376a42624648717431334d386464705467476f'), - ('event_message','event_message','16 1','140270 140281','134379 140281',X'03336531323039333735363534373138343830343235246c374a4f7543526b7756306d627a69356e5843496b4538416a6951374f67473455456c2d7053445a516649'), - ('event_message','event_message','11 1','162065 162071','154933 162071',X'0333653132323434383135393033313937373537373424674c77513179796e4b6d5859496b5a597a4a55627a66557a55552d714c4b5f524f454e4250325f6e44766b'), - ('event_message','event_message','1 1','170187 170187','162530 170187',X'0333653132323937373533363839343532373038333424656c6c76416a544a5847627936767249767470677a555572787231716f5a75536e50525f474b4e35455945'), - ('event_message','event_message','11 1','178736 178736','170762 178736',X'03336531323333353238303533323338333337353537242d39304668552d36455373594b6435484d7237666d6a414a5f6a576149616e356c4776384e655436564959'), - ('event_message','event_message','10 1','180317 180325','172228 180325',X'03336531323334363237303030393333383130323637246a4679416449665a4f54432d2d735971715472473735374c7a50504f34386439657963485477686d797751'), - ('event_message','event_message','1 1','212734 212734','203278 212734',X'03336531323438303131373930303633373637363734246557493950456f4d576b614b4d33416a7030782d6d455f6c4b4a6c495f6d6d44686f6379464b5170534f59'), - ('event_message','event_message','11 1','228100 228103','217856 228103',X'033365313235333734353736363733373132313336322447326a7a746e5977716a3676304a7a4168576e30725950596470667a5f496f76426771574a4876434c516b'), - ('event_message','event_message','11 1','240172 240181','229123 240181',X'033365313235393239383538323233303630313735382472635679454c5a76647453547a37366f3669434a4f614a316a756f683835535778494d546c5072517a676f'), - ('event_message','event_message','10 1','240259 240264','229132 240264',X'0333653132353933303030343035333535373235323024535849376a465f696e71424d714c4f564b347659746852644b31724747502d435a5f355a354f6b774e3751'), - ('event_message','event_message','2 1','255280 255281','243230 255281',X'03336531323636373837343338353137343234313738246e6e4e576f526a54495757723441463770625454746b7a73784c4d336b312d6d6645665031496b43586e59'), - ('event_message','event_message','1 1','297828 297828','283436 297828',X'0333653132383637303937353334363834323032313024544342465970356f39767a2d4f6b2d33654f4e433772426d354966615934476b48536e58445257474f4767'), - ('event_message','event_message','1 1','340375 340375','323489 340375',X'033365313330393336363936343530353934303030392431664536386d2d50546e786d5474345458584c35754847594139353779396c76582d50797150496d395f30'), - ('event_message','event_message','11 1','343791 343799','326730 343799',X'03336531333131353731333337393331363537323737246731767241315269725951592d3933304f6973587142372d6961686e34684e492d6462374952714176616b'), - ('event_message','event_message','10 1','363207 363216','344868 363216',X'0333653133323434393732323633393438393834393524784f78636e4749364269545941734d7a4f7557316f526e68356b675378544e30466442386a3037695f4f67'), - ('event_message','event_message','10 1','363219 363219','344871 363219',X'033365313332343439373532363733323039393732352432586f7938567937643843375a47767472657966326b4c4f4159397546386b4755364c3642435f446b6d77'), - ('event_message','event_message','10 1','363340 363343','344918 363343',X'03336531333234353037313732333634373530393830244f46645731396649534176706d34646c36367a2d5543337236432d436354506e34752d63444f7036733345'), - ('event_message','event_message','11 1','369452 369456','350712 369456',X'0333653133323736353831313733343439323336383024574d774330644574417277375f4562554a534465546532577174506d3747584347774570646c4f79326d30'), - ('event_message','event_message','10 1','372353 372356','353425 372356',X'0333653133323933313737353039313234353035373224366259364d313472667163486d5854476349716d4a4366467471796839794472375a6a487463715a6d4f34'), - ('sim_member','sim_member','225 1','0 12','0 12',X'034b4721414956694e775a64636b4652764c4f4567433a636164656e63652e6d6f65405f6f6f79655f616b6972615f6e6965723a636164656e63652e6d6f65'), - ('sim_member','sim_member','2 1','319 319','14 319',X'034b43214456706f6e54524d56456570486378744c423a636164656e63652e6d6f65405f6f6f79655f656e746f6c6f6d613a636164656e63652e6d6f65'), - ('sim_member','sim_member','68 1','391 440','26 440',X'034b4921457a54624a496c496d45534f746b4e644e4a3a636164656e63652e6d6f65405f6f6f79655f73617475726461797465643a636164656e63652e6d6f65'), - ('sim_member','sim_member','8 1','638 639','59 639',X'034b3f21497a4f675169446e757346516977796d614c3a636164656e63652e6d6f65405f6f6f79655f636f6f6b69653a636164656e63652e6d6f65'), - ('sim_member','sim_member','31 1','743 771','86 771',X'034b49214d5071594e414a62576b72474f544a7461703a636164656e63652e6d6f65405f6f6f79655f746865666f6f6c323239343a636164656e63652e6d6f65'), - ('sim_member','sim_member','26 1','774 787','87 787',X'034b49214d687950614b4250506f496c7365794d6d743a636164656e63652e6d6f65405f6f6f79655f6b79757567727970686f6e3a636164656e63652e6d6f65'), - ('sim_member','sim_member','27 1','877 881','104 881',X'034b45215063734371724f466a48476f41424270414c3a636164656e63652e6d6f65405f6f6f79655f62696c6c795f626f623a636164656e63652e6d6f65'), - ('sim_member','sim_member','16 1','956 959','117 959',X'034b43215158526f4a777a63506d5047546d454b454d3a636164656e63652e6d6f65405f6f6f79655f626f7472616334723a636164656e63652e6d6f65'), - ('sim_member','sim_member','32 1','997 1012','121 1012',X'034b3f215170676c734e587a4c7751594d4c6c734f503a636164656e63652e6d6f65405f6f6f79655f6a75746f6d693a636164656e63652e6d6f65'), - ('sim_member','sim_member','7 1','1274 1279','157 1279',X'034b4121554d6f6e68556765644d47585a78466658753a636164656e63652e6d6f65405f6f6f79655f6d696e696d75733a636164656e63652e6d6f65'), - ('sim_member','sim_member','27 1','1415 1439','188 1439',X'034b3d21595868717249786d586e47736961796a59783a636164656e63652e6d6f65405f6f6f79655f73746161663a636164656e63652e6d6f65'), - ('sim_member','sim_member','16 1','1597 1599','217 1599',X'034b512163466a4479477274466d48796d794c6652453a636164656e63652e6d6f65405f6f6f79655f626f6a61636b5f686f7273656d616e3a636164656e63652e6d6f65'), - ('sim_member','sim_member','27 1','1758 1761','248 1761',X'034b3d2168665a74624d656f5355564e424850736a743a636164656e63652e6d6f65405f6f6f79655f617a7572653a636164656e63652e6d6f65'), - ('sim_member','sim_member','25 1','1865 1886','270 1886',X'034b3f216b4c52714b4b555158636962494d744f706c3a636164656e63652e6d6f65405f6f6f79655f7361796f72693a636164656e63652e6d6f65'), - ('sim_member','sim_member','19 1','1918 1919','276 1919',X'034b3d216b68497350756c465369736d43646c596e493a636164656e63652e6d6f65405f6f6f79655f617a7572653a636164656e63652e6d6f65'), - ('sim_member','sim_member','33 1','1986 2015','286 2015',X'034b3d216d5451744d736a534c4f646c576f7265594d3a636164656e63652e6d6f65405f6f6f79655f73746161663a636164656e63652e6d6f65'), - ('sim_member','sim_member','37 1','2027 2028','289 2028',X'034b4d216d616767455367755a427147425a74536e723a636164656e63652e6d6f65405f6f6f79655f2e7265616c2e706572736f6e2e3a636164656e63652e6d6f65'), - ('sim_member','sim_member','28 1','2117 2130','297 2130',X'034b3f216e4e595a794b6f4e70797859417a50466f733a636164656e63652e6d6f65405f6f6f79655f6a75746f6d693a636164656e63652e6d6f65'), - ('sim_member','sim_member','20 1','2230 2239','310 2239',X'034b41217046504c7270594879487a784e4c69594b413a636164656e63652e6d6f65405f6f6f79655f686578676f61743a636164656e63652e6d6f65'), - ('sim_member','sim_member','30 1','2381 2398','332 2398',X'034b3b21717a44626c4b6c69444c577a52524f6e465a3a636164656e63652e6d6f65405f6f6f79655f6d6e696b3a636164656e63652e6d6f65'), - ('sim_member','sim_member','38 1','2490 2518','344 2518',X'034b3d2173445250714549546e4f4e57474176496b423a636164656e63652e6d6f65405f6f6f79655f727974686d3a636164656e63652e6d6f65'), - ('sim_member','sim_member','11 1','2555 2559','358 2559',X'034b4321746751436d526b426e6474516362687150583a636164656e63652e6d6f65405f6f6f79655f6a6f7365707065793a636164656e63652e6d6f65'), - ('sim_member','sim_member','47 1','2633 2666','377 2666',X'034b4b217750454472596b77497a6f744e66706e57503a636164656e63652e6d6f65405f6f6f79655f6e61706f6c656f6e333038393a636164656e63652e6d6f65'), - ('sim_member','sim_member','52 1','2817 2837','414 2837',X'034b43217a66654e574d744b4f764f48766f727979563a636164656e63652e6d6f65405f6f6f79655f696e736f676e69613a636164656e63652e6d6f65'), - ('guild_space','guild_space','1','3','3',X'0231313132373630363639313738323431303234'), - ('guild_space','guild_space','1','7','7',X'023331313534383638343234373234343633363837'), - ('guild_space','guild_space','1','11','11',X'023331323139303338323637383430393235383138'), - ('guild_space','guild_space','1','15','15',X'023331323839353939363232353930383930313335'), - ('guild_space','guild_space','1','19','19',X'0231323733383737363437323234393935383431'), - ('guild_space','guild_space','1','23','23',X'0231353239313736313536333938363832313135'), - ('guild_space','guild_space','1','27','27',X'0231373535303134333534373334313533383138'), - ('guild_space','guild_space','1','31','31',X'0231393933383838313432343535323130303834'), - ('guild_active','guild_active','1','3','3',X'0231313132373630363639313738323431303234'), - ('guild_active','guild_active','1','7','7',X'023331313534383638343234373234343633363837'), - ('guild_active','guild_active','1','11','11',X'023331323139303338323637383430393235383138'), - ('guild_active','guild_active','1','15','15',X'023331323839353939363232353930383930313335'), - ('guild_active','guild_active','1','19','19',X'023331333333323139363936393333323038303934'), - ('guild_active','guild_active','1','23','23',X'0231343735353939303338353336373434393630'), - ('guild_active','guild_active','1','27','27',X'022f3636313932393535373737343836383438'), - ('guild_active','guild_active','1','31','31',X'0231383737303635303431393930353136373637'), - ('emoji','emoji','1','284','284',X'023331313132323031303430303637303339333332'), - ('emoji','emoji','1','569','569',X'023331323334393032303131393436383630363638'), - ('emoji','emoji','1','854','854',X'0231323735313734373438353034313935303732'), - ('emoji','emoji','1','1139','1139',X'0231333837343730383630303134393737303235'), - ('emoji','emoji','1','1424','1424',X'0231353435363639393734303130383838313932'), - ('emoji','emoji','1','1709','1709',X'0231363339313034333030393139393437323634'), - ('emoji','emoji','1','1994','1994',X'0231373532363932363230303638373832313231'), - ('emoji','emoji','1','2279','2279',X'0231383935343737353331303434363138323430'), - ('auto_emoji','auto_emoji','1','0','0',X'02114c31'), - ('auto_emoji','auto_emoji','1','1','1',X'02114c32'), - ('auto_emoji','auto_emoji','1','2','2',X'020f5f'); - -ANALYZE sqlite_schema; - -COMMIT; diff --git a/src/db/migrations/0026-make-rooms-historical.sql b/src/db/migrations/0026-make-rooms-historical.sql new file mode 100644 index 00000000..f9f4b483 --- /dev/null +++ b/src/db/migrations/0026-make-rooms-historical.sql @@ -0,0 +1,62 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; + +-- *** historical_channel_room *** + +CREATE TABLE "historical_channel_room" ( + "historical_room_index" INTEGER NOT NULL, + "reference_channel_id" TEXT NOT NULL, + "room_id" TEXT NOT NULL UNIQUE, + PRIMARY KEY("historical_room_index" AUTOINCREMENT), + FOREIGN KEY("reference_channel_id") REFERENCES "channel_room"("channel_id") ON DELETE CASCADE +); + +INSERT INTO historical_channel_room (reference_channel_id, room_id) SELECT channel_id, room_id FROM channel_room; + +-- *** message_channel -> message_room *** + +CREATE TABLE "message_room" ( + "message_id" TEXT NOT NULL, + "historical_room_index" INTEGER NOT NULL, + PRIMARY KEY("message_id"), + FOREIGN KEY("historical_room_index") REFERENCES "historical_channel_room"("historical_room_index") ON DELETE CASCADE +) WITHOUT ROWID; +INSERT INTO message_room (message_id, historical_room_index) SELECT message_id, max(historical_room_index) as historical_room_index FROM message_channel INNER JOIN historical_channel_room ON historical_channel_room.reference_channel_id = message_channel.channel_id GROUP BY message_id; + +-- *** event_message *** + +CREATE TABLE "new_event_message" ( + "event_id" TEXT NOT NULL, + "event_type" TEXT, + "event_subtype" TEXT, + "message_id" TEXT NOT NULL, + "part" INTEGER NOT NULL, + "reaction_part" INTEGER NOT NULL, + "source" INTEGER NOT NULL, + PRIMARY KEY("message_id","event_id"), + FOREIGN KEY("message_id") REFERENCES "message_room"("message_id") ON DELETE CASCADE +) WITHOUT ROWID; +INSERT INTO new_event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) SELECT event_id, event_type, event_subtype, message_id, part, reaction_part, source from event_message; +DROP TABLE event_message; +ALTER TABLE new_event_message RENAME TO event_message; + +-- *** reaction *** + +CREATE TABLE "new_reaction" ( + "hashed_event_id" INTEGER NOT NULL, + "message_id" TEXT NOT NULL, + "encoded_emoji" TEXT NOT NULL, original_encoding TEXT, + PRIMARY KEY("hashed_event_id"), + FOREIGN KEY("message_id") REFERENCES "message_room"("message_id") ON DELETE CASCADE +) WITHOUT ROWID; +INSERT INTO new_reaction (hashed_event_id, message_id, encoded_emoji) SELECT hashed_event_id, message_id, encoded_emoji FROM reaction; +DROP TABLE reaction; +ALTER TABLE new_reaction RENAME TO reaction; + +-- *** + +DROP TABLE message_channel; +PRAGMA foreign_key_check; + +COMMIT; +PRAGMA foreign_keys=ON; diff --git a/src/db/migrations/0027-analyze.sql b/src/db/migrations/0027-analyze.sql new file mode 100644 index 00000000..03573939 --- /dev/null +++ b/src/db/migrations/0027-analyze.sql @@ -0,0 +1,256 @@ +-- https://www.sqlite.org/lang_analyze.html + +BEGIN TRANSACTION; + +ANALYZE sqlite_schema; + +DELETE FROM "sqlite_stat1"; +INSERT INTO "sqlite_stat1" ("tbl","idx","stat") VALUES ('reaction','reaction','4706 1'), +('event_message','event_message','535744 1 1'), +('historical_channel_room','sqlite_autoindex_historical_channel_room_1','996 1'), +('message_room','message_room','508753 1'), +('invite','invite','3 2 1'), +('auto_emoji','auto_emoji','2 1'), +('sim','sim','1072 1'), +('webhook','webhook','205 1'), +('channel_room','channel_room','996 1'), +('channel_room','sqlite_autoindex_channel_room_1','996 1'), +('guild_active','guild_active','45 1'), +('media_proxy','media_proxy','19581 1'), +('sim_member','sim_member','5501 6 1'), +('emoji','emoji','3470 1'), +('guild_space','guild_space','42 1'), +('member_power','member_power','1 1 1'), +('sim_proxy','sim_proxy','213 1'), +('migration',NULL,'1'), +('member_cache','member_cache','1099 3 1'), +('direct','direct','1 1'), +('file','file','36309 1'), +('lottie','lottie','22 1'); + +DELETE FROM "sqlite_stat4"; +INSERT INTO "sqlite_stat4" ("tbl","idx","neq","nlt","ndlt","sample") VALUES ('reaction','reaction','1','522','522',X'02069c21bd28f26ae025'), + ('reaction','reaction','1','1045','1045',X'0206b8b64d2a67851518'), + ('reaction','reaction','1','1568','1568',X'0206d45580ee5b75848e'), + ('reaction','reaction','1','2091','2091',X'0206f103da3779e1cf70'), + ('reaction','reaction','1','2614','2614',X'02061028b6f24ae55f1a'), + ('reaction','reaction','1','3137','3137',X'02062bd1ffb636826ad6'), + ('reaction','reaction','1','3660','3660',X'020647c632b246bf22ae'), + ('reaction','reaction','1','4183','4183',X'020664ebb265471ad34e'), + ('event_message','event_message','11 1','14790 14800','14356 14800',X'033365313135323030333736393431373038393034342471375f4b666676763631582d30794639334b7532776459377a56553068745a744c2d705272367142473851'), + ('event_message','event_message','11 1','33809 33817','32914 33817',X'03336531313537363033363837303537383636373632245646504363595144413134586c664d4f5041786c58596b4432595a503562656d6f63336b39787735596141'), + ('event_message','event_message','1 1','59527 59527','57719 59527',X'03336531313636383130343235323637303737323031244e343074365465464f3841585255766a353568312d31345f3533476c686b586a43364b6142534442647677'), + ('event_message','event_message','10 1','104218 104225','100236 104225',X'03336531313838373238363331323637313736343538246c6f35326a6332723637734b4b5f4b767a4361702d62627573506d423931454f76776f5f614a6a5a6e6f4d'), + ('event_message','event_message','11 1','116172 116174','111525 116174',X'03336531313933363837333730363137333237363536244546764673764c6c62316a446a4f616b4838584f68636d556161316f6d7651494143374575723146686567'), + ('event_message','event_message','10 1','116223 116230','111530 116230',X'03336531313933363837363335343238393231343636246a7243703367546f676a4930525a315561655931577978724d67737874704431494b3769454d6972754e73'), + ('event_message','event_message','1 1','119055 119055','114207 119055',X'0333653131393530353035383636333130363537353124335561353079304c7044386248725f7142574f44306d6569794f37524a4c746238755755725a716c665477'), + ('event_message','event_message','16 1','140286 140286','134379 140286',X'033365313230393337353635343731383438303432352430347938482d444f49634270597873576b31756d417a4e424e3061774634487457636e3177426476755255'), + ('event_message','event_message','11 1','162080 162090','154932 162090',X'0333653132323434383135393033313937373537373424784753744170626668647074526f4a755a2d4d416c557446424e524939344f7661506e4a55694952366138'), + ('event_message','event_message','1 1','178583 178583','170598 178583',X'033365313233333436343838383836393931363736352444643279534c3857704f666268576b626d684758716c49746f6e7237384b6656456956334b664a43702d38'), + ('event_message','event_message','11 1','178659 178662','170672 178662',X'03336531323333353238303533323338333337353537244e482d334a42617864424f53724c3259766b715534696c6e54724d485f57454973524b556c684b744f4977'), + ('event_message','event_message','11 1','215266 215274','205302 215274',X'03336531323533373435373636373337313231333632246a4b63784e434f784b627558315746346e59696538783472753559724a2d756c38455756715731796a4349'), + ('event_message','event_message','11 1','224498 224504','213831 224504',X'03336531323539323938353832323330363031373538246646483237794b4f75554a5943723671762d676b324b6f636f784663574b766c654b3167636d6445377177'), + ('event_message','event_message','10 1','224585 224593','213840 224593',X'03336531323539333030303430353335353732353230247164484754527353717138524376756735435349457a6d73717633396c4335746e334942487a4b6366646b'), + ('event_message','event_message','1 1','238111 238111','226382 238111',X'0333653132363833333233363330323132383735333724766f6261774b476a5742357770795970364c4a3931615136524a6b7330597a7a717a6e715953344d325467'), + ('event_message','event_message','1 1','297639 297639','282242 297639',X'0333653133303634313236363735393832373837333624502d47394f59667430735143334d6a475a532d445242376a586b4e62366147734c6a6b39544c7561434763'), + ('event_message','event_message','11 1','304605 304606','288785 304606',X'03336531333131353731333337393331363537323737243031717a505a477a35764a475276464f716533653570513930705a6a43644c326a33485635415571504559'), + ('event_message','event_message','10 1','322247 322256','305183 322256',X'03336531333234353034393632373530383737373937247a58477647646d614b6f6e4a497232767246354c523235796e4e4b476b5a357a774a696a44645433694f49'), + ('event_message','event_message','11 1','327028 327032','309699 327032',X'0333653133323736353831313733343439323336383024574d774330644574417277375f4562554a534465546532577174506d3747584347774570646c4f79326d30'), + ('event_message','event_message','1 1','357167 357167','338131 357167',X'033365313334353735363437313239393933363237362436755847477632536e727a63346353616e3937686a6562356a6a77516d6d4c66464f4c305f74316d627130'), + ('event_message','event_message','10 1','365779 365781','346205 365781',X'03336531333439393933323736353839323135373835244f484162596f6b67435748704a36556a6e74545f74774452484743624e52656e617a5a3658776f464a4e55'), + ('event_message','event_message','1 1','416695 416695','394011 416695',X'0333653133363737343039323037343035383134363624486e346e6d784571634758767153577a7343656a586733696c517930554869705f34547073504e71684f55'), + ('event_message','event_message','11 1','422263 422266','399248 422266',X'0333653133363938333439303532363130343738313824436a34597373765a46544877415a5236724c77472d635250716c426b7a3749626d473261766653562d4e55'), + ('event_message','event_message','1 1','476223 476223','449789 476223',X'033365313430313837343437373832313732363834312455414d51397a6c2d775a39572d327231687866584162746f447143314734646239554b384573596e657359'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','8 8','8 8',X'034b022141534d746248706d6f4b4e736765574274733a636164656e63652e6d6f65025d'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','18 18','18 18',X'034b02214173786a53777176484e444f4665587a676b3a636164656e63652e6d6f65014c'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','94 94','94 94',X'034b0221464d5346425a536d59596964656f4a58594b3a636164656e63652e6d6f65017e'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','100 100','100 100',X'034b0221466875676b616e45716943627448734644483a636164656e63652e6d6f6502d2'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','103 103','103 103',X'034b0221466a7167556258447a474a75796a464746613a636164656e63652e6d6f6503c6'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','110 110','110 110',X'034b0221474650514f614e7a517465534c4b485374543a636164656e63652e6d6f65013e'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','221 221','221 221',X'034b02214c506d4c664f6b796d63646d725644624a463a636164656e63652e6d6f6501ee'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','311 311','311 311',X'034b02215157735a4a7042716c716548686c616962443a636164656e63652e6d6f65029a'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','332 332','332 332',X'034b022152547356547675424f4555506d52687347633a636164656e63652e6d6f650255'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','408 408','408 408',X'034b022155635059696f48454f426761664f576b694f3a636164656e63652e6d6f6502ea'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','443 443','443 443',X'034b01215754625a53764a66524a72736348574b78563a636164656e63652e6d6f6529'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','450 450','450 450',X'034b022157695842616d5676586279476565657a50413a636164656e63652e6d6f650157'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','468 468','468 468',X'034b022158717470725744744f5761667a72486f477a3a636164656e63652e6d6f65018b'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','526 526','526 526',X'034b022162574b794f596c6468484b7a784b757667623a636164656e63652e6d6f650127'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','554 554','554 554',X'034b0221637779454c6c6b55714a565942646a4250543a636164656e63652e6d6f6501b0'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','597 597','597 597',X'034b02216668587668437279724e525661777572636f3a636164656e63652e6d6f6503bd'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','608 608','608 608',X'034b022167636d734472716442706f7463586b4545703a636164656e63652e6d6f6501d5'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','665 665','665 665',X'034b02216a6c4e64496a62687654486b5a74745166773a636164656e63652e6d6f650131'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','708 708','708 708',X'034b01216d454c5846716a426958726d7558796943723a636164656e63652e6d6f656b'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','768 768','768 768',X'034b02216f7a6f494e55494261685177775a6f586b7a3a636164656e63652e6d6f6501d8'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','776 776','776 776',X'034b022170506f63657a415046506f584c6a6a5750443a636164656e63652e6d6f6503aa'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','877 877','877 877',X'034b02217474644e4b6b4b4f49757879566869794b6a3a636164656e63652e6d6f6502e1'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','887 887','887 887',X'034b02217568496f63597a4e41714244694b5645506d3a636164656e63652e6d6f650080'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','943 943','943 943',X'034b02217863655a4641624250425a6d7842677a51523a636164656e63652e6d6f65036b'), + ('message_room','message_room','1','56528','56528',X'023331313635393437353536363733363334333635'), + ('message_room','message_room','1','113057','113057',X'023331313932353536323035343730363538363132'), + ('message_room','message_room','1','169586','169586',X'023331323331373432353832313033373430343937'), + ('message_room','message_room','1','226115','226115',X'023331323635333430353738363135333934333034'), + ('message_room','message_room','1','282644','282644',X'023331333033373933303435323632353635343637'), + ('message_room','message_room','1','339173','339173',X'023331333434343332333234333137363735363733'), + ('message_room','message_room','1','395702','395702',X'023331333637303635303537373939393636383631'), + ('message_room','message_room','1','452231','452231',X'023331343030353931393936323333373131373238'), + ('invite','invite','1 1','0 0','0 0',X'033549406d616c6b696572693a6d61747269782e6f7267216877535462776967674c584a6f6756506b763a6d61747269782e6f7267'), + ('invite','invite','2 1','1 1','1 1',X'03334b406d65636879613a636164656e63652e6d6f652172486b466c6d724e6e664157455943494a703a636164656e63652e6d6f65'), + ('invite','invite','2 1','1 2','1 2',X'03334b406d65636879613a636164656e63652e6d6f65217549514e424f4f68735976514c645368726b3a636164656e63652e6d6f65'), + ('auto_emoji','auto_emoji','1','0','0',X'02114c31'), + ('auto_emoji','auto_emoji','1','1','1',X'02114c32'), + ('sim','sim','1','119','119',X'025531316564343731342d636635652d346333372d393331382d376136353266383732636634'), + ('sim','sim','1','239','239',X'0231313530373435393839383336333038343830'), + ('sim','sim','1','359','359',X'0231323235393737393435323133313038323234'), + ('sim','sim','1','479','479',X'0231333038323937303732373738343132303332'), + ('sim','sim','1','599','599',X'0231343135373135363735313031373234363735'), + ('sim','sim','1','719','719',X'0231353732363938363739363138353638313933'), + ('sim','sim','1','839','839',X'0231373332373338333838303431343030343231'), + ('sim','sim','1','959','959',X'0231393437373336313831393732353636303636'), + ('webhook','webhook','1','22','22',X'023331313630383933333337303239353836393536'), + ('webhook','webhook','1','45','45',X'023331323139343938393236343636363632343330'), + ('webhook','webhook','1','68','68',X'023331323432383939363632343734373131303630'), + ('webhook','webhook','1','91','91',X'023331323937323836383730393534323833313533'), + ('webhook','webhook','1','114','114',X'023331333430353438363133363931393332373133'), + ('webhook','webhook','1','137','137',X'023331343034313334383236303530383436393331'), + ('webhook','webhook','1','160','160',X'0231333639373535303430343638303431373238'), + ('webhook','webhook','1','183','183',X'0231363035353930343336333230333738383930'), + ('channel_room','channel_room','1','110','110',X'023331313939353030313137393834363733393133'), + ('channel_room','channel_room','1','221','221',X'023331323734313935333432323131393430353434'), + ('channel_room','channel_room','1','332','332',X'023331333437303939353439343430363735383732'), + ('channel_room','channel_room','1','443','443',X'023331343035323432323838343138303632333636'), + ('channel_room','channel_room','1','554','554',X'023331343036373736363630393936333935323830'), + ('channel_room','channel_room','1','665','665',X'023331343039363536363537383835323635393830'), + ('channel_room','channel_room','1','776','776',X'023331343139353132333134363234383638343632'), + ('channel_room','channel_room','1','887','887',X'0231333735373135343638383937353530333631'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','8 8','8 8',X'034b332141534d746248706d6f4b4e736765574274733a636164656e63652e6d6f6531343037333239313938393039303330353135'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','18 18','18 18',X'034b33214173786a53777176484e444f4665587a676b3a636164656e63652e6d6f6531333437303036333637393639343433383430'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','94 94','94 94',X'034b3321464d5346425a536d59596964656f4a58594b3a636164656e63652e6d6f6531343034313333323339303736393530313631'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','100 100','100 100',X'034b3321466875676b616e45716943627448734644483a636164656e63652e6d6f6531343132383936323333373839353837353036'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','103 103','103 103',X'034b3121466a7167556258447a474a75796a464746613a636164656e63652e6d6f65383034363236313139333938393136313037'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','110 110','110 110',X'034b3321474650514f614e7a517465534c4b485374543a636164656e63652e6d6f6531333435363431343537323035323532313737'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','221 221','221 221',X'034b33214c506d4c664f6b796d63646d725644624a463a636164656e63652e6d6f6531343035393733313637323535353834383830'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','311 311','311 311',X'034b33215157735a4a7042716c716548686c616962443a636164656e63652e6d6f6531343039363536363537383835323635393830'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','332 332','332 332',X'034b332152547356547675424f4555506d52687347633a636164656e63652e6d6f6531343037323235393932313935333432343237'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','408 408','408 408',X'034b332155635059696f48454f426761664f576b694f3a636164656e63652e6d6f6531343134303334313437323032303434303134'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','443 443','443 443',X'034b33215754625a53764a66524a72736348574b78563a636164656e63652e6d6f6531313433313231353134393235393238353431'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','450 450','450 450',X'034b332157695842616d5676586279476565657a50413a636164656e63652e6d6f6531333536353037353335313132323738303839'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','468 468','468 468',X'034b332158717470725744744f5761667a72486f477a3a636164656e63652e6d6f6531343034353137353530393634303830363530'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','526 526','526 526',X'034b332162574b794f596c6468484b7a784b757667623a636164656e63652e6d6f6531333239353237383038343232333138313030'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','554 554','554 554',X'034b3321637779454c6c6b55714a565942646a4250543a636164656e63652e6d6f6531343034393538363332363830303939393931'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','597 597','597 597',X'034b31216668587668437279724e525661777572636f3a636164656e63652e6d6f65373535373235353231333731303034393739'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','608 608','608 608',X'034b332167636d734472716442706f7463586b4545703a636164656e63652e6d6f6531343035353833303939383532363139393137'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','665 665','665 665',X'034b33216a6c4e64496a62687654486b5a74745166773a636164656e63652e6d6f6531333339343132353232373538393633323030'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','708 708','708 708',X'034b33216d454c5846716a426958726d7558796943723a636164656e63652e6d6f6531313936393134373631303430393234373432'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','768 768','768 768',X'034b33216f7a6f494e55494261685177775a6f586b7a3a636164656e63652e6d6f6531343035363036393133303638313039383735'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','776 776','776 776',X'034b312170506f63657a415046506f584c6a6a5750443a636164656e63652e6d6f65363832333334343939393136333439353231'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','877 877','877 877',X'034b33217474644e4b6b4b4f49757879566869794b6a3a636164656e63652e6d6f6531343133323134313538383631373632363430'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','887 887','887 887',X'034b33217568496f63597a4e41714244694b5645506d3a636164656e63652e6d6f6531323137383839303233393831323536373534'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','943 943','943 943',X'034b31217863655a4641624250425a6d7842677a51523a636164656e63652e6d6f65323937323734343934303737333730333638'), + ('guild_active','guild_active','1','5','5',X'023331313433333336323438373631363437313534'), + ('guild_active','guild_active','1','11','11',X'023331313630383933333336333234393331353834'), + ('guild_active','guild_active','1','17','17',X'023331323839353936343835343631323137333430'), + ('guild_active','guild_active','1','23','23',X'023331333338363530383035363233393834333030'), + ('guild_active','guild_active','1','29','29',X'023331343338363132393630393137353836313233'), + ('guild_active','guild_active','1','35','35',X'0231343937313539373236343535343535373534'), + ('guild_active','guild_active','1','41','41',X'0231383730313138363530373638363730373530'), + ('media_proxy','media_proxy','1','2175','2175',X'02069ccd283fa260cb90'), + ('media_proxy','media_proxy','1','4351','4351',X'0206b95c95635f40b3d8'), + ('media_proxy','media_proxy','1','6527','6527',X'0206d546a2d00310b6cc'), + ('media_proxy','media_proxy','1','8703','8703',X'0206f0b1fb70331afcb3'), + ('media_proxy','media_proxy','1','10879','10879',X'02060e48cd55947d2372'), + ('media_proxy','media_proxy','1','13055','13055',X'02062ac70844c1762329'), + ('media_proxy','media_proxy','1','15231','15231',X'0206470a46ab937760c8'), + ('media_proxy','media_proxy','1','17407','17407',X'0206632a367225280573'), + ('sim_member','sim_member','225 1','14 104','4 104',X'034b4521414956694e775a64636b4652764c4f4567433a636164656e63652e6d6f65405f6f6f79655f6a6a6a6a6a363634343a636164656e63652e6d6f65'), + ('sim_member','sim_member','124 1','598 611','85 611',X'034b4921457a54624a496c496d45534f746b4e644e4a3a636164656e63652e6d6f65405f6f6f79655f61726a756e3034323236393a636164656e63652e6d6f65'), + ('sim_member','sim_member','35 1','817 845','107 845',X'034b532147486e4d476978756867527255546d4b77683a636164656e63652e6d6f65405f6f6f79655f6e65637461726f66616d62726f7369613a636164656e63652e6d6f65'), + ('sim_member','sim_member','63 1','943 948','140 948',X'034b452148725979716b6f7942485a4a4e634455564f3a636164656e63652e6d6f65405f6f6f79655f5f706b5f64717a76613a636164656e63652e6d6f65'), + ('sim_member','sim_member','48 1','1022 1037','148 1037',X'034b47214943566475566c646e6774484b64567070503a636164656e63652e6d6f65405f6f6f79655f5f706b5f7078677679783a636164656e63652e6d6f65'), + ('sim_member','sim_member','39 1','1203 1223','174 1223',X'034b45214a48614a714258706d6b49654963615562513a636164656e63652e6d6f65405f6f6f79655f6d69647473756d61723a636164656e63652e6d6f65'), + ('sim_member','sim_member','48 1','1736 1781','289 1781',X'034b472150747969527851614879636777626c636f743a636164656e63652e6d6f65405f6f6f79655f76616e746164656c69613a636164656e63652e6d6f65'), + ('sim_member','sim_member','5 1','1834 1835','299 1835',X'034b472151544372636e695373616f626957444c516f3a636164656e63652e6d6f65405f6f6f79655f68756d616e67616d65723a636164656e63652e6d6f65'), + ('sim_member','sim_member','64 1','2099 2137','353 2137',X'034b4721536c7664497a734f6f534469434e6a6e77733a636164656e63652e6d6f65405f6f6f79655f5f706b5f777768667a6f3a636164656e63652e6d6f65'), + ('sim_member','sim_member','81 1','2215 2276','361 2276',X'034b4321544f61794476734c735a566d5779745166483a636164656e63652e6d6f65405f6f6f79655f6171756173316d703a636164656e63652e6d6f65'), + ('sim_member','sim_member','42 1','2370 2389','373 2389',X'034b47215468436b4b5857434a77657451496d747a573a636164656e63652e6d6f65405f6f6f79655f5f706b5f7465736361723a636164656e63652e6d6f65'), + ('sim_member','sim_member','36 1','2424 2447','380 2447',X'034b472154716c79516d69667847556767456d64424e3a636164656e63652e6d6f65405f6f6f79655f657665727970697a7a613a636164656e63652e6d6f65'), + ('sim_member','sim_member','65 1','2691 2723','438 2723',X'034b472157755a5549494e7457456a64656658414c473a636164656e63652e6d6f65405f6f6f79655f5f706b5f77797a63686a3a636164656e63652e6d6f65'), + ('sim_member','sim_member','2 1','3059 3059','497 3059',X'034b3f21616f764c6d776a674d6c44414c6e666c426e3a636164656e63652e6d6f65405f6f6f79655f6f6363696d793a636164656e63652e6d6f65'), + ('sim_member','sim_member','32 1','3127 3147','520 3147',X'034b392163427874565278446c5a765356684a58564b3a636164656e63652e6d6f65405f6f6f79655f626f743a636164656e63652e6d6f65'), + ('sim_member','sim_member','8 1','3666 3671','630 3671',X'034b39216966636d75794e6e544861464163636575543a636164656e63652e6d6f65405f6f6f79655f726e6c3a636164656e63652e6d6f65'), + ('sim_member','sim_member','43 1','3849 3855','668 3855',X'034b47216b6b4b714249664c45596a4b534b626b4a633a636164656e63652e6d6f65405f6f6f79655f5f706b5f6f77617a76663a636164656e63652e6d6f65'), + ('sim_member','sim_member','8 1','4280 4283','746 4283',X'034b3f216f705748554e6b46646247796b7579564b6a3a636164656e63652e6d6f65405f6f6f79655f636f6f6b69653a636164656e63652e6d6f65'), + ('sim_member','sim_member','158 1','4424 4526','770 4526',X'034b472170757146464b59487750677073554e6d6e443a636164656e63652e6d6f65405f6f6f79655f5f706b5f77797a63686a3a636164656e63652e6d6f65'), + ('sim_member','sim_member','44 1','4807 4843','824 4843',X'034b4521734b4c6f784a4e62547a6c72436d4a796f533a636164656e63652e6d6f65405f6f6f79655f7370696e6e657265743a636164656e63652e6d6f65'), + ('sim_member','sim_member','11 1','4889 4895','841 4895',X'034b45217443744769524448676e4a62505a4f5479573a636164656e63652e6d6f65405f6f6f79655f6a656d74616e756b693a636164656e63652e6d6f65'), + ('sim_member','sim_member','73 1','5069 5107','888 5107',X'034b4721766576446275617472435946704e71426d583a636164656e63652e6d6f65405f6f6f79655f5f706b5f75616e6766633a636164656e63652e6d6f65'), + ('sim_member','sim_member','59 1','5179 5207','903 5207',X'034b5f217750454472596b77497a6f744e66706e57503a636164656e63652e6d6f65405f6f6f79655f66756a69776172615f6e6f5f6d6f6b6f755f66756d6f3a636164656e63652e6d6f65'), + ('sim_member','sim_member','52 1','5438 5453','968 5453',X'034b41217a66654e574d744b4f764f48766f727979563a636164656e63652e6d6f65405f6f6f79655f666f67656c38323a636164656e63652e6d6f65'), + ('emoji','emoji','1','385','385',X'023331313035373039393137313237353737363733'), + ('emoji','emoji','1','771','771',X'023331323230353735323436303531303533353638'), + ('emoji','emoji','1','1157','1157',X'023331333530383339313335363836313033303730'), + ('emoji','emoji','1','1543','1543',X'0231333530373039313634383434323533313934'), + ('emoji','emoji','1','1929','1929',X'0231343934393031383434343036303432363337'), + ('emoji','emoji','1','2315','2315',X'0231363432383535303135363835323932303438'), + ('emoji','emoji','1','2701','2701',X'0231373738353731303433363339333934333134'), + ('emoji','emoji','1','3087','3087',X'0231393031313133373739343135333134343332'), + ('guild_space','guild_space','1','4','4',X'023331313333333135333632353636343535333336'), + ('guild_space','guild_space','1','9','9',X'023331313534383638343234373234343633363837'), + ('guild_space','guild_space','1','14','14',X'023331323139303338323637383430393235383138'), + ('guild_space','guild_space','1','19','19',X'023331323839363030383537323437303535383733'), + ('guild_space','guild_space','1','24','24',X'023331333536353037353335313132323738303836'), + ('guild_space','guild_space','1','29','29',X'0231323937323732313833373136303532393933'), + ('guild_space','guild_space','1','34','34',X'0231363437393532363337373630363334383831'), + ('guild_space','guild_space','1','39','39',X'0231383737303635303431393930353136373637'), + ('member_power','member_power','1 1','0 0','0 0',X'03350f40636164656e63653a636164656e63652e6d6f652a'), + ('sim_proxy','sim_proxy','1','23','23',X'025531363733363165392d656137652d343530392d623533302d356531613863613735336237'), + ('sim_proxy','sim_proxy','1','47','47',X'025532653561626332312d326332622d346133352d386237642d366432383162363036653932'), + ('sim_proxy','sim_proxy','1','71','71',X'025534383131393165322d393462302d346534632d623934352d336330323932623135356238'), + ('sim_proxy','sim_proxy','1','95','95',X'025536346331346631642d663834342d346535622d386665332d336162336163363239616230'), + ('sim_proxy','sim_proxy','1','119','119',X'025538376562363463322d363763352d346432352d383161642d666664333235663266303639'), + ('sim_proxy','sim_proxy','1','143','143',X'025561616630313539652d623165312d343231342d396266652d313334613536303738323231'), + ('sim_proxy','sim_proxy','1','167','167',X'025563396534393633372d663061352d343566352d383234382d366436393565643861316434'), + ('sim_proxy','sim_proxy','1','191','191',X'025565333734613634362d386231332d343365392d393635392d653233326366653866626265'), + ('member_cache','member_cache','1 1','122 122','82 122',X'034b2d214d5071594e414a62576b72474f544a7461703a636164656e63652e6d6f6540726e6c3a636164656e63652e6d6f65'), + ('member_cache','member_cache','5 1','126 130','85 130',X'034b43214e446249714e704a795076664b526e4e63723a636164656e63652e6d6f6540776f756e6465645f77617272696f723a6d61747269782e6f7267'), + ('member_cache','member_cache','4 1','136 137','90 137',X'034b3d214f485844457370624d485348716c4445614f3a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), + ('member_cache','member_cache','5 1','204 206','135 206',X'034b51215450616f6a5454444446444847776c7276743a636164656e63652e6d6f65406a61636b736f6e6368656e3636363a6a61636b736f6e6368656e3636362e636f6d'), + ('member_cache','member_cache','76 1','213 245','140 245',X'034b412154716c79516d69667847556767456d64424e3a636164656e63652e6d6f65406869726f616e7461676f6e6973743a6d61747269782e6f7267'), + ('member_cache','member_cache','4 1','301 303','150 303',X'034b352156624f77675559777146614e4c5345644e413a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), + ('member_cache','member_cache','4 1','338 340','170 340',X'034b3321586f4c466b65786a45466c57447959544d453a636164656e63652e6d6f65406d65636879613a636164656e63652e6d6f65'), + ('member_cache','member_cache','10 1','344 344','173 344',X'034b3d21594b46454e79716667696951686956496b533a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), + ('member_cache','member_cache','2 1','368 368','183 368',X'034b35215a51714f4757704f52676e544f7254474f583a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), + ('member_cache','member_cache','152 1','396 491','203 491',X'034b412163427874565278446c5a765356684a58564b3a636164656e63652e6d6f65406d61727368616d616c6c6f773a616c74686165612e7a6f6e65'), + ('member_cache','member_cache','4 1','553 555','207 555',X'034b33216356514d45455158494d5047554b43467a763a636164656e63652e6d6f65406d65636879613a636164656e63652e6d6f65'), + ('member_cache','member_cache','8 1','571 576','219 576',X'034b2b21654856655270706e6c6f57587177704a6e553a636164656e63652e6d6f6540656c6c69753a68617368692e7265'), + ('member_cache','member_cache','7 1','583 584','223 584',X'034b4b2165724f7079584e465a486a48724568784e583a636164656e63652e6d6f6540616d796973636f6f6c7a3a6d61747269782e6174697573616d792e636f6d'), + ('member_cache','member_cache','165 1','599 614','230 614',X'034b3321676865544b5a7451666c444e7070684c49673a636164656e63652e6d6f65406172636f6e79783a6d61747269782e6f7267'), + ('member_cache','member_cache','165 1','599 737','230 737',X'034b3921676865544b5a7451666c444e7070684c49673a636164656e63652e6d6f654073756368616372636b723a6d61747269782e6f7267'), + ('member_cache','member_cache','6 1','764 764','231 764',X'034b3d21676b6b686f756d42506643415055664554783a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), + ('member_cache','member_cache','10 1','772 775','234 775',X'034b412168424a766e654e4f63646b77694e464c67483a636164656e63652e6d6f65406265617264616e64726f7365733a636164656e63652e6d6f65'), + ('member_cache','member_cache','13 1','795 800','244 800',X'03513d2169537958674e7851634575586f587073536e3a707573737468656361742e6f72674063726f6e793a63726f6e79616b617473756b692e78797a'), + ('member_cache','member_cache','12 1','851 860','274 860',X'034b35216c7570486a715444537a774f744d59476d493a636164656e63652e6d6f654068656c6c63703a6f70656e737573652e6f7267'), + ('member_cache','member_cache','4 1','872 874','280 874',X'034b39216d616767455367755a427147425a74536e723a636164656e63652e6d6f65406875636b6c65746f6e3a636164656e63652e6d6f65'), + ('member_cache','member_cache','4 1','924 924','315 924',X'034b352172454f73706e5971644f414c4149466e69563a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), + ('member_cache','member_cache','4 1','966 966','342 966',X'034b3d21766e717a56767678534a586c5a504f5276533a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), + ('member_cache','member_cache','71 1','981 983','350 983',X'034b3d2177574f667376757356486f4e4e567242585a3a636164656e63652e6d6f6540616c65783a6d61747269782e7370656564626f792e6368'), + ('member_cache','member_cache','14 1','1056 1062','354 1062',X'034b3d21776c534544496a44676c486d42474b7254703a636164656e63652e6d6f65406b61746c796e3a69732d686172646c792e6f6e6c696e65'), + ('direct','direct','1','0','0',X'024d4070726f666573736f725f706f6f7469735f7068643a6d61747269782e6f7267'), + ('file','file','1','4034','4034',X'03821368747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3131323736303636393137383234313032342f313135303430343234303037393036393237362f636d64726e656f6e2d313632383534343535313536373635303831362d32303233303232325f3134353733322d766964312e6d7034'), + ('file','file','1','8069','8069',X'03814968747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3131393239333936393731353639313532322f313230393030323833373230383630343638322f636869702d77696c736f6e2e77656270'), + ('file','file','1','12104','12104',X'03813b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3133343037373735333438353033333437322f313139383038373532313531393830303436312f696d6167652e706e67'), + ('file','file','1','16139','16139',X'03814168747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f313337363732343737393830353034383838322f313339323632373139373133383730323336362f707265766965772e706e67'), + ('file','file','1','20174','20174',X'03817568747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3135393136353731343139343735393638302f313236383537313138333330363131373230302f53637265656e73686f745f32303234303830315f3130303834315f446973636f72642e6a7067'), + ('file','file','1','24209','24209',X'03816568747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3238383838323935333331343839333832352f313135343539393530343936303537333533322f53637265656e73686f745f32303233303631302d3130333631332e706e67'), + ('file','file','1','28244','28244',X'03813b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3635353231363137333639363238363734362f313330383839373634363434353938393930392f696d6167652e706e67'), + ('file','file','1','32279','32279',X'027f68747470733a2f2f63646e2e646973636f72646170702e636f6d2f656d6f6a69732f313034393232363234323836313332323238312e706e67'), + ('lottie','lottie','1','2','2',X'0231373439303532393434363832353832303336'), + ('lottie','lottie','1','5','5',X'0231373531363036333739333430333635383634'), + ('lottie','lottie','1','8','8',X'0231373534313038373731383532323232353634'), + ('lottie','lottie','1','11','11',X'0231373936313430363338303933343433303932'), + ('lottie','lottie','1','14','14',X'0231373936313431373032363935343835353030'), + ('lottie','lottie','1','17','17',X'0231383136303837373932323931323832393434'), + ('lottie','lottie','1','20','20',X'0231383233393736313032393736323930383636'); + +ANALYZE sqlite_schema; + +COMMIT; diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index b0b74a5a..a3fcf0dd 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -60,6 +60,12 @@ export type Models = { autocreate: 0 | 1 } + historical_channel_room: { + historical_room_index: number + reference_channel_id: string + room_id: string + } + invite: { mxid: string room_id: string @@ -91,15 +97,15 @@ export type Models = { power_level: number } - message_channel: { + message_room: { message_id: string - channel_id: string + historical_room_index: number } sim: { user_id: string + username: string sim_name: string - localpart: string mxid: string } diff --git a/src/db/orm.js b/src/db/orm.js index 4d9b6f1a..d14e8ae2 100644 --- a/src/db/orm.js +++ b/src/db/orm.js @@ -104,6 +104,21 @@ class From { return r } + /** + * @template {Col} Select + * @param {string} what + * @param {Select} col + */ + pluckAs(what, col) { + /** @type {Pluck} */ + // @ts-ignore + const r = this + r.cols = [`${what} AS ${col}`] + this.makeColsSafe = false + r.isPluck = true + return r + } + /** * @param {string} sql */ diff --git a/src/db/orm.test.js b/src/db/orm.test.js index 4549b9e0..56a3257d 100644 --- a/src/db/orm.test.js +++ b/src/db/orm.test.js @@ -21,8 +21,8 @@ test("orm: select: get pluck works", t => { }) test("orm: select: get, where and pluck works", t => { - const channelID = select("message_channel", "channel_id", {message_id: "1128118177155526666"}).pluck().get() - t.equal(channelID, "112760669178241024") + const emojiName = select("emoji", "name", {emoji_id: "230201364309868544"}).pluck().get() + t.equal(emojiName, "hippo") }) test("orm: select: all, where and pluck works on multiple columns", t => { diff --git a/src/discord/interactions/matrix-info.js b/src/discord/interactions/matrix-info.js index b7551f10..e35cde1d 100644 --- a/src/discord/interactions/matrix-info.js +++ b/src/discord/interactions/matrix-info.js @@ -1,7 +1,8 @@ // @ts-check const DiscordTypes = require("discord-api-types/v10") -const {discord, sync, from} = require("../../passthrough") +const {discord, sync, select, from} = require("../../passthrough") +const assert = require("assert").strict /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") @@ -11,9 +12,9 @@ const api = sync.require("../../matrix/api") * @param {{api: typeof api}} di * @returns {Promise} */ -async function _interact({guild_id, data}, {api}) { - const message = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id") - .select("name", "nick", "source", "channel_id", "room_id", "event_id").where({message_id: data.target_id, part: 0}).get() +async function _interact({guild_id, channel, data}, {api}) { + const message = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("source", "reference_channel_id", "room_id", "event_id").where({message_id: data.target_id, part: 0}).get() if (!message) { return { @@ -25,15 +26,19 @@ async function _interact({guild_id, data}, {api}) { } } + const channel_id = message.reference_channel_id + const room = select("channel_room", ["name", "nick"], {channel_id}).get() + assert(room) + const idInfo = `\n-# Room ID: \`${message.room_id}\`\n-# Event ID: \`${message.event_id}\`` - const roomName = message.nick || message.name + const roomName = room.nick || room.name if (message.source === 1) { // from Discord const userID = data.resolved.messages[data.target_id].author.id return { type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, data: { - content: `Bridged <@${userID}> https://discord.com/channels/${guild_id}/${message.channel_id}/${data.target_id} on Discord to [${roomName}]() on Matrix.` + content: `Bridged <@${userID}> https://discord.com/channels/${guild_id}/${channel_id}/${data.target_id} on Discord to [${roomName}]() on Matrix.` + idInfo, flags: DiscordTypes.MessageFlags.Ephemeral } @@ -45,7 +50,7 @@ async function _interact({guild_id, data}, {api}) { return { type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, data: { - content: `Bridged [${event.sender}]()'s message in [${roomName}]() on Matrix to https://discord.com/channels/${guild_id}/${message.channel_id}/${data.target_id} on Discord.` + content: `Bridged [${event.sender}]()'s message in [${roomName}]() on Matrix to https://discord.com/channels/${guild_id}/${channel_id}/${data.target_id} on Discord.` + idInfo, flags: DiscordTypes.MessageFlags.Ephemeral } diff --git a/src/discord/interactions/permissions.js b/src/discord/interactions/permissions.js index 0a573e34..07324acc 100644 --- a/src/discord/interactions/permissions.js +++ b/src/discord/interactions/permissions.js @@ -18,8 +18,8 @@ const api = sync.require("../../matrix/api") async function* _interact({data, guild_id}, {api}) { // Get message info const row = from("event_message") - .join("message_channel", "message_id") - .select("event_id", "source", "channel_id") + .join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("event_id", "source", "room_id", "reference_channel_id") .where({message_id: data.target_id}) .get() @@ -35,10 +35,9 @@ async function* _interact({data, guild_id}, {api}) { } // Get the message sender, the person that will be inspected/edited - const eventID = row.event_id - const roomID = select("channel_room", "room_id", {channel_id: row.channel_id}).pluck().get() + const roomID = select("channel_room", "room_id", {channel_id: row.reference_channel_id}).pluck().get() assert(roomID) - const event = await api.getEvent(roomID, eventID) + const event = await api.getEvent(row.room_id, row.event_id) const sender = event.sender // Get the space, where the power levels will be inspected/edited diff --git a/src/discord/interactions/reactions.js b/src/discord/interactions/reactions.js index 13dbd69d..59bf065a 100644 --- a/src/discord/interactions/reactions.js +++ b/src/discord/interactions/reactions.js @@ -16,7 +16,7 @@ const utils = sync.require("../../m2d/converters/utils") * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} */ async function* _interact({data}, {api}) { - const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id") + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") .select("event_id", "room_id").where({message_id: data.target_id}).get() if (!row) { return yield {createInteractionResponse: { diff --git a/src/m2d/actions/add-reaction.js b/src/m2d/actions/add-reaction.js index 277c475a..268888c3 100644 --- a/src/m2d/actions/add-reaction.js +++ b/src/m2d/actions/add-reaction.js @@ -14,7 +14,7 @@ const emoji = sync.require("../converters/emoji") * @param {Ty.Event.Outer} event */ async function addReaction(event) { - const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get() + const channelID = select("historical_channel_room", "reference_channel_id", {room_id: event.room_id}).pluck().get() if (!channelID) return // We just assume the bridge has already been created const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id}, "ORDER BY reaction_part").pluck().get() if (!messageID) return // Nothing can be done if the parent message was never bridged. diff --git a/src/m2d/actions/redact.js b/src/m2d/actions/redact.js index 1d3aa67f..f980a540 100644 --- a/src/m2d/actions/redact.js +++ b/src/m2d/actions/redact.js @@ -12,13 +12,14 @@ const utils = sync.require("../converters/utils") * @param {Ty.Event.Outer_M_Room_Redaction} event */ async function deleteMessage(event) { - const rows = from("event_message").join("message_channel", "message_id").select("channel_id", "message_id").where({event_id: event.redacts}).all() + const rows = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("reference_channel_id", "message_id").where({event_id: event.redacts}).all() if (!rows.length) return for (const row of rows) { - await discord.snow.channel.deleteMessage(row.channel_id, row.message_id, event.content.reason) + await discord.snow.channel.deleteMessage(row.reference_channel_id, row.message_id, event.content.reason) db.prepare("DELETE FROM event_message WHERE message_id = ?").run(row.message_id) } - db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(rows[0].message_id) + db.prepare("DELETE FROM message_room WHERE message_id = ?").run(rows[0].message_id) } /** @@ -26,9 +27,10 @@ async function deleteMessage(event) { */ async function removeReaction(event) { const hash = utils.getEventIDHash(event.redacts) - const row = from("reaction").join("message_channel", "message_id").select("channel_id", "message_id", "encoded_emoji").where({hashed_event_id: hash}).get() + const row = from("reaction").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("reference_channel_id", "message_id", "encoded_emoji").where({hashed_event_id: hash}).get() if (!row) return - await discord.snow.channel.deleteReactionSelf(row.channel_id, row.message_id, row.encoded_emoji) + await discord.snow.channel.deleteReactionSelf(row.reference_channel_id, row.message_id, row.encoded_emoji) db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(hash) } diff --git a/src/m2d/actions/send-event.js b/src/m2d/actions/send-event.js index 6b4eb262..7ed70c72 100644 --- a/src/m2d/actions/send-event.js +++ b/src/m2d/actions/send-event.js @@ -6,7 +6,7 @@ const stream = require("stream") const assert = require("assert").strict const crypto = require("crypto") const passthrough = require("../../passthrough") -const {sync, discord, db, select} = passthrough +const {sync, discord, db, from, select} = passthrough /** @type {import("./channel-webhook")} */ const channelWebhook = sync.require("./channel-webhook") @@ -61,7 +61,7 @@ async function resolvePendingFiles(message) { /** @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker} event */ async function sendEvent(event) { - const row = select("channel_room", ["channel_id", "thread_parent"], {room_id: event.room_id}).get() + const row = from("channel_room").where({room_id: event.room_id}).select("channel_id", "thread_parent").get() if (!row) return [] // allow the bot to exist in unbridged rooms, just don't do anything with it let channelID = row.channel_id let threadID = undefined @@ -73,6 +73,8 @@ async function sendEvent(event) { const guildID = discord.channels.get(channelID).guild_id const guild = discord.guilds.get(guildID) assert(guild) + const historicalRoomIndex = select("historical_channel_room", "historical_room_index", {room_id: event.room_id}).pluck().get() + assert(historicalRoomIndex) // no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it @@ -98,15 +100,17 @@ async function sendEvent(event) { } for (const id of messagesToDelete) { - db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(id) + db.prepare("DELETE FROM message_room WHERE message_id = ?").run(id) await channelWebhook.deleteMessageWithWebhook(channelID, id, threadID) } for (const message of messagesToSend) { const reactionPart = messagesToEdit.length === 0 && message === messagesToSend[messagesToSend.length - 1] ? 0 : 1 const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID) - db.prepare("INSERT INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, threadID || channelID) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart, reactionPart) // source 0 = matrix + db.transaction(() => { + db.prepare("INSERT INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(messageResponse.id, historicalRoomIndex) + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart, reactionPart) // source 0 = matrix + })() eventPart = 1 messageResponses.push(messageResponse) diff --git a/src/m2d/actions/update-pins.js b/src/m2d/actions/update-pins.js index 46074c6a..d06f6e82 100644 --- a/src/m2d/actions/update-pins.js +++ b/src/m2d/actions/update-pins.js @@ -12,12 +12,13 @@ const diffPins = sync.require("../converters/diff-pins") async function updatePins(pins, prev) { const diff = diffPins.diffPins(pins, prev) for (const [event_id, added] of diff) { - const row = from("event_message").join("message_channel", "message_id").where({event_id, part: 0}).select("channel_id", "message_id").get() + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("reference_channel_id", "message_id").get() if (!row) continue if (added) { - discord.snow.channel.addChannelPinnedMessage(row.channel_id, row.message_id, "Message pinned on Matrix") + discord.snow.channel.addChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message pinned on Matrix") } else { - discord.snow.channel.removeChannelPinnedMessage(row.channel_id, row.message_id, "Message unpinned on Matrix") + discord.snow.channel.removeChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message unpinned on Matrix") } } } diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index fd9289dd..a9415096 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -24,6 +24,8 @@ const file = sync.require("../../matrix/file") const emojiSheet = sync.require("./emoji-sheet") /** @type {import("../actions/setup-emojis")} */ const setupEmojis = sync.require("../actions/setup-emojis") +/** @type {import("../../d2m/converters/user-to-mxid")} */ +const userToMxid = sync.require("../../d2m/converters/user-to-mxid") /** @type {[RegExp, string][]} */ const markdownEscapes = [ @@ -332,10 +334,11 @@ function splitDisplayName(displayName) { * Convert a Matrix user ID into a Discord user ID for mentioning, where if the user is a PK proxy, it will mention the proxy owner. * @param {string} mxid */ -function getUserOrProxyOwnerID(mxid) { - const row = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({mxid}).get() +function getUserOrProxyOwnerMention(mxid) { + const row = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "username", "proxy_owner_id").where({mxid}).get() if (!row) return null - return row.proxy_owner_id || row.user_id + if (userToMxid.isWebhookUserID(row.user_id)) return `**@${row.username}**` + return `<@${row.proxy_owner_id || row.user_id}>` } /** @@ -398,7 +401,7 @@ async function handleRoomOrMessageLinks(input, di) { } } - const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() + const channelID = select("historical_channel_room", "reference_channel_id", {room_id: roomID}).pluck().get() if (!channelID) continue if (!eventID) { // 1: It's a room link, so <#link> to the channel @@ -681,9 +684,10 @@ async function eventToMessage(event, guild, di) { } replyLine = await getL1L2ReplyLine() - const row = from("event_message").join("message_channel", "message_id").select("channel_id", "message_id").where({event_id: repliedToEventId}).and("ORDER BY part").get() + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("reference_channel_id", "message_id").where({event_id: repliedToEventId}).and("ORDER BY part").get() if (row) { - replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} ` + replyLine += `https://discord.com/channels/${guild.id}/${row.reference_channel_id}/${row.message_id} ` } // If the event has been edited, the homeserver will include the relation in `unsigned`. if (repliedToEvent.unsigned?.["m.relations"]?.["m.replace"]?.content?.["m.new_content"]) { @@ -727,9 +731,9 @@ async function eventToMessage(event, guild, di) { } } const sender = repliedToEvent.sender - const authorID = getUserOrProxyOwnerID(sender) - if (authorID) { - replyLine += `<@${authorID}>` + const authorMention = getUserOrProxyOwnerMention(sender) + if (authorMention) { + replyLine += authorMention } else { let senderName = select("member_cache", "displayname", {mxid: sender}).pluck().get() if (!senderName) senderName = sender.match(/@([^:]*)/)?.[1] diff --git a/src/web/routes/info.js b/src/web/routes/info.js index 0ccdecad..9c202fa8 100644 --- a/src/web/routes/info.js +++ b/src/web/routes/info.js @@ -26,16 +26,28 @@ as.router.get("/api/message", defineEventHandler(async event => { const api = getAPI(event) const {message_id} = await getValidatedQuery(event, schema.message.parse) - const metadatas = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").where({message_id}) - .select("event_id", "event_type", "event_subtype", "part", "reaction_part", "room_id", "source").and("ORDER BY part ASC, reaction_part DESC").all() + const metadatas = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").where({message_id}) + .select("event_id", "event_type", "event_subtype", "part", "reaction_part", "reference_channel_id", "room_id", "source").and("ORDER BY part ASC, reaction_part DESC").all() if (metadatas.length === 0) { return new Response("Message not found", {status: 404, statusText: "Not Found"}) } + const current_room_id = select("channel_room", "room_id", {channel_id: metadatas[0].reference_channel_id}).pluck().get() const events = await Promise.all(metadatas.map(metadata => api.getEvent(metadata.room_id, metadata.event_id).then(raw => ({ - metadata: Object.assign({sender: raw.sender}, metadata), + metadata: { + event_id: metadata.event_id, + event_type: metadata.event_type, + event_subtype: metadata.event_subtype, + part: metadata.part, + reaction_part: metadata.reaction_part, + channel_id: metadata.reference_channel_id, + room_id: metadata.room_id, + source: metadata.source, + sender: raw.sender, + current_room_id: current_room_id + }, raw })) )) diff --git a/src/web/routes/info.test.js b/src/web/routes/info.test.js index 28dac3bb..39b2c00d 100644 --- a/src/web/routes/info.test.js +++ b/src/web/routes/info.test.js @@ -57,14 +57,16 @@ test("web info: returns data for a matrix message and profile", async t => { }, events: [{ metadata: { - event_id: "$51gH61p_eJc2RylOdE2lAr4-ogP7dS0WJI62lCFzBvk", - event_subtype: "m.text", - event_type: "m.room.message", - part: 0, - reaction_part: 0, - room_id: "!qzDBLKlildpzrrOnFZ:cadence.moe", - sender: "@cadence:cadence.moe", - source: 0 + event_id: "$51gH61p_eJc2RylOdE2lAr4-ogP7dS0WJI62lCFzBvk", + event_subtype: "m.text", + event_type: "m.room.message", + part: 0, + reaction_part: 0, + room_id: "!qzDBLKlildpzrrOnFZ:cadence.moe", + channel_id: "176333891320283136", + current_room_id: "!qzDBLKlildpzrrOnFZ:cadence.moe", + sender: "@cadence:cadence.moe", + source: 0 }, raw }] @@ -113,14 +115,16 @@ test("web info: returns data for a matrix message without profile", async t => { }, events: [{ metadata: { - event_id: "$51gH61p_eJc2RylOdE2lAr4-ogP7dS0WJI62lCFzBvk", - event_subtype: "m.text", - event_type: "m.room.message", - part: 0, - reaction_part: 0, - room_id: "!qzDBLKlildpzrrOnFZ:cadence.moe", - sender: "@cadence:cadence.moe", - source: 0 + event_id: "$51gH61p_eJc2RylOdE2lAr4-ogP7dS0WJI62lCFzBvk", + event_subtype: "m.text", + event_type: "m.room.message", + part: 0, + reaction_part: 0, + room_id: "!qzDBLKlildpzrrOnFZ:cadence.moe", + channel_id: "176333891320283136", + current_room_id: "!qzDBLKlildpzrrOnFZ:cadence.moe", + sender: "@cadence:cadence.moe", + source: 0 }, raw }] @@ -191,14 +195,16 @@ test("web info: returns data for a discord message", async t => { matrix_author: undefined, events: [{ metadata: { - event_id: "$AfrB8hzXkDMvuoWjSZkDdFYomjInWH7jMBPkwQMN8AI", - event_subtype: "m.text", - event_type: "m.room.message", - part: 0, - reaction_part: 1, - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", - sender: "@_ooye_accavish:cadence.moe", - source: 1 + event_id: "$AfrB8hzXkDMvuoWjSZkDdFYomjInWH7jMBPkwQMN8AI", + event_subtype: "m.text", + event_type: "m.room.message", + part: 0, + reaction_part: 1, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + channel_id: "112760669178241024", + current_room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@_ooye_accavish:cadence.moe", + source: 1 }, raw: raw1 }, { @@ -209,6 +215,8 @@ test("web info: returns data for a discord message", async t => { part: 1, reaction_part: 0, room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + channel_id: "112760669178241024", + current_room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", sender: "@_ooye_accavish:cadence.moe", source: 1 }, diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 65719548..78ab2b45 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -23,6 +23,8 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom ('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL), ('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL); +INSERT INTO historical_channel_room (reference_channel_id, room_id) SELECT channel_id, room_id FROM channel_room; + INSERT INTO sim (user_id, username, sim_name, mxid) VALUES ('0', 'Matrix Bridge', 'bot', '@_ooye_bot:cadence.moe'), ('820865262526005258', 'Crunch God', 'crunch_god', '@_ooye_crunch_god:cadence.moe'), @@ -42,7 +44,8 @@ INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES INSERT INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES ('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '196188877885538304', 'Azalea &flwr; 🌺'); -INSERT INTO message_channel (message_id, channel_id) VALUES +INSERT INTO message_room (message_id, historical_room_index) +WITH a (message_id, channel_id) AS (VALUES ('1106366167788044450', '122155380120748034'), ('1106366167788044451', '122155380120748034'), ('1106366167788044452', '122155380120748034'), @@ -75,7 +78,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES ('1339000288144658482', '176333891320283136'), ('1381212840957972480', '112760669178241024'), ('1401760355339862066', '112760669178241024'), -('1439351590262800565', '1438284564815548418'); +('1439351590262800565', '1438284564815548418')) +SELECT message_id, max(historical_room_index) as historical_room_index FROM a INNER JOIN historical_channel_room ON historical_channel_room.reference_channel_id = a.channel_id GROUP BY message_id; INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1), From a0fc7a7136ae007c31a1d5b6a336598cc323af02 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 26 Nov 2025 17:21:15 +1300 Subject: [PATCH 028/153] m->d: make image-replies work --- src/m2d/converters/event-to-message.js | 294 ++++++++++---------- src/m2d/converters/event-to-message.test.js | 93 +++++++ 2 files changed, 241 insertions(+), 146 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index a9415096..7002c385 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -608,7 +608,7 @@ async function eventToMessage(event, guild, di) { } attachments.push({id: "0", filename}) pendingFiles.push({name: filename, mxc: event.content.url}) - } else if (shouldProcessTextEvent) { + } else { // Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events. // this event ---is an edit of--> original event ---is a reply to--> past event await (async () => { @@ -742,157 +742,159 @@ async function eventToMessage(event, guild, di) { replyLine = `-# > ${replyLine}${contentPreview}\n` })() - if (event.content.format === "org.matrix.custom.html" && event.content.formatted_body) { - let input = event.content.formatted_body - if (event.content.msgtype === "m.emote") { - input = `* ${displayName} ${input}` - } - - // Handling mentions of Discord users - input = input.replace(/("https:\/\/matrix.to\/#\/((?:@|%40)[^"]+)")>/g, (whole, attributeValue, mxid) => { - mxid = decodeURIComponent(mxid) - if (mxUtils.eventSenderIsFromDiscord(mxid)) { - // Handle mention of an OOYE sim user by their mxid - const id = select("sim", "user_id", {mxid}).pluck().get() - if (!id) return whole - return `${attributeValue} data-user-id="${id}">` - } else { - // Handle mention of a Matrix user by their mxid - // Check if this Matrix user is actually the sim user from another old bridge in the room? - const match = mxid.match(/[^:]*discord[^:]*_([0-9]{6,}):/) // try to match @_discord_123456, @_discordpuppet_123456, etc. - if (match) return `${attributeValue} data-user-id="${match[1]}">` - // Nope, just a real Matrix user. - return whole + if (shouldProcessTextEvent) { + if (event.content.format === "org.matrix.custom.html" && event.content.formatted_body) { + let input = event.content.formatted_body + if (event.content.msgtype === "m.emote") { + input = `* ${displayName} ${input}` } - }) - // Handling mentions of rooms and room-messages - input = await handleRoomOrMessageLinks(input, di) - - // Stripping colons after mentions - input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1") - input = input.replace(/("https:\/\/matrix.to.*?<\/a>):?/g, "$1") - - // Element adds a bunch of
before but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those. - input = input.replace(/(?:\n|
\s*)*<\/blockquote>/g, "") - - // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. - // But I should not count it if it's between block elements. - input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => { - // console.error(beforeContext, beforeTag, afterContext, afterTag) - if (typeof beforeTag !== "string" && typeof afterTag !== "string") { - return "
" - } - beforeContext = beforeContext || "" - beforeTag = beforeTag || "" - afterContext = afterContext || "" - afterTag = afterTag || "" - if (!mxUtils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !mxUtils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) { - return beforeContext + "
" + afterContext - } else { - return whole - } - }) - - // Note: Element's renderers on Web and Android currently collapse whitespace, like the browser does. Turndown also collapses whitespace which is good for me. - // If later I'm using a client that doesn't collapse whitespace and I want turndown to follow suit, uncomment the following line of code, and it Just Works: - // input = input.replace(/ /g, " ") - // There is also a corresponding test to uncomment, named "event2message: whitespace is retained" - - // Handling written @mentions: we need to look for candidate Discord members to join to the room - // This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here. - // We're using the domino parser because Turndown uses the same and can reuse this tree. - const doc = domino.createDocument( - // DOM parsers arrange elements in the and . Wrapping in a custom element ensures elements are reliably arranged in a single element. - '' + input + '' - ); - const root = doc.getElementById("turndown-root"); - async function forEachNode(node) { - for (; node; node = node.nextSibling) { - // Check written mentions - if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) { - const result = await checkWrittenMentions(node.nodeValue, event.sender, event.room_id, guild, di) - if (result) { - node.nodeValue = result.content - ensureJoined.push(...result.ensureJoined) - allowedMentionsParse.push(...result.allowedMentionsParse) - } + // Handling mentions of Discord users + input = input.replace(/("https:\/\/matrix.to\/#\/((?:@|%40)[^"]+)")>/g, (whole, attributeValue, mxid) => { + mxid = decodeURIComponent(mxid) + if (mxUtils.eventSenderIsFromDiscord(mxid)) { + // Handle mention of an OOYE sim user by their mxid + const id = select("sim", "user_id", {mxid}).pluck().get() + if (!id) return whole + return `${attributeValue} data-user-id="${id}">` + } else { + // Handle mention of a Matrix user by their mxid + // Check if this Matrix user is actually the sim user from another old bridge in the room? + const match = mxid.match(/[^:]*discord[^:]*_([0-9]{6,}):/) // try to match @_discord_123456, @_discordpuppet_123456, etc. + if (match) return `${attributeValue} data-user-id="${match[1]}">` + // Nope, just a real Matrix user. + return whole } - // Check for incompatible backticks in code blocks - let preNode - if (node.nodeType === 3 && node.nodeValue.includes("```") && (preNode = nodeIsChildOf(node, ["PRE"]))) { - if (preNode.firstChild?.nodeName === "CODE") { - const ext = preNode.firstChild.className.match(/language-(\S+)/)?.[1] || "txt" - const filename = `inline_code.${ext}` - // Build the replacement node - const replacementCode = doc.createElement("code") - replacementCode.textContent = `[${filename}]` - // Build its containing node - const replacement = doc.createElement("span") - replacement.appendChild(doc.createTextNode(" ")) - replacement.appendChild(replacementCode) - replacement.appendChild(doc.createTextNode(" ")) - // Replace the code block with the - preNode.replaceWith(replacement) - // Upload the code as an attachment - const content = getCodeContent(preNode.firstChild) - attachments.push({id: String(attachments.length), filename}) - pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")}) - } + }) + + // Handling mentions of rooms and room-messages + input = await handleRoomOrMessageLinks(input, di) + + // Stripping colons after mentions + input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1") + input = input.replace(/("https:\/\/matrix.to.*?<\/a>):?/g, "$1") + + // Element adds a bunch of
before but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those. + input = input.replace(/(?:\n|
\s*)*<\/blockquote>/g, "") + + // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. + // But I should not count it if it's between block elements. + input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => { + // console.error(beforeContext, beforeTag, afterContext, afterTag) + if (typeof beforeTag !== "string" && typeof afterTag !== "string") { + return "
" + } + beforeContext = beforeContext || "" + beforeTag = beforeTag || "" + afterContext = afterContext || "" + afterTag = afterTag || "" + if (!mxUtils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !mxUtils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) { + return beforeContext + "
" + afterContext + } else { + return whole + } + }) + + // Note: Element's renderers on Web and Android currently collapse whitespace, like the browser does. Turndown also collapses whitespace which is good for me. + // If later I'm using a client that doesn't collapse whitespace and I want turndown to follow suit, uncomment the following line of code, and it Just Works: + // input = input.replace(/ /g, " ") + // There is also a corresponding test to uncomment, named "event2message: whitespace is retained" + + // Handling written @mentions: we need to look for candidate Discord members to join to the room + // This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here. + // We're using the domino parser because Turndown uses the same and can reuse this tree. + const doc = domino.createDocument( + // DOM parsers arrange elements in the and . Wrapping in a custom element ensures elements are reliably arranged in a single element. + '' + input + '' + ); + const root = doc.getElementById("turndown-root"); + async function forEachNode(node) { + for (; node; node = node.nextSibling) { + // Check written mentions + if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) { + const result = await checkWrittenMentions(node.nodeValue, event.sender, event.room_id, guild, di) + if (result) { + node.nodeValue = result.content + ensureJoined.push(...result.ensureJoined) + allowedMentionsParse.push(...result.allowedMentionsParse) + } + } + // Check for incompatible backticks in code blocks + let preNode + if (node.nodeType === 3 && node.nodeValue.includes("```") && (preNode = nodeIsChildOf(node, ["PRE"]))) { + if (preNode.firstChild?.nodeName === "CODE") { + const ext = preNode.firstChild.className.match(/language-(\S+)/)?.[1] || "txt" + const filename = `inline_code.${ext}` + // Build the replacement node + const replacementCode = doc.createElement("code") + replacementCode.textContent = `[${filename}]` + // Build its containing node + const replacement = doc.createElement("span") + replacement.appendChild(doc.createTextNode(" ")) + replacement.appendChild(replacementCode) + replacement.appendChild(doc.createTextNode(" ")) + // Replace the code block with the + preNode.replaceWith(replacement) + // Upload the code as an attachment + const content = getCodeContent(preNode.firstChild) + attachments.push({id: String(attachments.length), filename}) + pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")}) + } + } + await forEachNode(node.firstChild) } - await forEachNode(node.firstChild) } + await forEachNode(root) + + // SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet. + // First we need to determine which emojis are at the end. + endOfMessageEmojis = [] + let match + let last = input.length + while ((match = input.slice(0, last).match(/]*>\s*$/))) { + if (!match[0].includes("data-mx-emoticon")) break + const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/) + if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1]) + assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec") + last = match.index + } + + // @ts-ignore bad type from turndown + content = turndownService.turndown(root) + + // Put < > around any surviving matrix.to links to hide the URL previews + content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/g, "<$&>") + + // It's designed for commonmark, we need to replace the space-space-newline with just newline + content = content.replace(/ \n/g, "\n") + + // If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated + if (replyLine && content.startsWith("> ")) content = "\n" + content + + // SPRITE SHEET EMOJIS FEATURE: + content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader) + } else { + // Looks like we're using the plaintext body! + content = event.content.body + + if (event.content.msgtype === "m.emote") { + content = `* ${displayName} ${content}` + } + + content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible + content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews + + const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di) + if (result) { + content = result.content + ensureJoined.push(...result.ensureJoined) + allowedMentionsParse.push(...result.allowedMentionsParse) + } + + // Markdown needs to be escaped, though take care not to escape the middle of links + // @ts-ignore bad type from turndown + content = turndownService.escape(content) } - await forEachNode(root) - - // SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet. - // First we need to determine which emojis are at the end. - endOfMessageEmojis = [] - let match - let last = input.length - while ((match = input.slice(0, last).match(/]*>\s*$/))) { - if (!match[0].includes("data-mx-emoticon")) break - const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/) - if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1]) - assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec") - last = match.index - } - - // @ts-ignore bad type from turndown - content = turndownService.turndown(root) - - // Put < > around any surviving matrix.to links to hide the URL previews - content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/g, "<$&>") - - // It's designed for commonmark, we need to replace the space-space-newline with just newline - content = content.replace(/ \n/g, "\n") - - // If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated - if (replyLine && content.startsWith("> ")) content = "\n" + content - - // SPRITE SHEET EMOJIS FEATURE: - content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader) - } else { - // Looks like we're using the plaintext body! - content = event.content.body - - if (event.content.msgtype === "m.emote") { - content = `* ${displayName} ${content}` - } - - content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible - content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews - - const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di) - if (result) { - content = result.content - ensureJoined.push(...result.ensureJoined) - allowedMentionsParse.push(...result.allowedMentionsParse) - } - - // Markdown needs to be escaped, though take care not to escape the middle of links - // @ts-ignore bad type from turndown - content = turndownService.escape(content) } } diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 73ca4e96..2e347f54 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -2671,6 +2671,99 @@ test("event2message: rich reply to a state event with no body", async t => { ) }) +test("event2message: rich reply with an image", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + body: "image.png", + info: { + size: 470379, + mimetype: "image/png", + thumbnail_info: { + w: 800, + h: 450, + mimetype: "image/png", + size: 183014 + }, + w: 1920, + h: 1080, + "xyz.amorgan.blurhash": "L24_wtVt00xuxvR%NFX74Toz?waL", + thumbnail_url: "mxc://cadence.moe/lPtnjlleowWCXGOHKVDyoXGn" + }, + msgtype: "m.image", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4" + } + }, + url: "mxc://cadence.moe/yxMobQMbSqNHpajxgSHtaooG" + }, + origin_server_ts: 1764127662631, + unsigned: { + membership: "join", + age: 97, + transaction_id: "m1764127662540.2" + }, + event_id: "$QOxkw7u8vjTrrdKxEUO13JWSixV7UXAZU1freT1SkHc", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }, data.guild.general, { + api: { + getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + t.equal(eventID, "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4") + return { + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "you have to check every diff above insane on this set https://osu.ppy.sh/beatmapsets/2263303#osu/4826296" + }, + origin_server_ts: 1763639396419, + unsigned: { + membership: "join", + age: 486586696, + transaction_id: "m1763639396324.578" + }, + event_id: "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + } + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [ + { + content: "-# > <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/112760669178241024/1128118177155526666 **Ⓜcadence [they]**: you have to check every diff above insane on this...", + allowed_mentions: { + parse: ["users", "roles"] + }, + attachments: [ + { + filename: "image.png", + id: "0", + }, + ], + avatar_url: undefined, + pendingFiles: [ + { + mxc: "mxc://cadence.moe/yxMobQMbSqNHpajxgSHtaooG", + name: "image.png", + }, + ], + username: "cadence [they]", + }, + ] + } + ) +}) + test("event2message: raw mentioning discord users in plaintext body works", async t => { t.deepEqual( await eventToMessage({ From 42beb6d249b754c4b20ea9a0456ed65513214739 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 27 Nov 2025 21:48:49 +1300 Subject: [PATCH 029/153] Maybe accept invites more reliably --- src/d2m/actions/create-room.js | 3 +- src/m2d/event-dispatcher.js | 16 +++++-- src/matrix/api.js | 19 ++++++++ src/types.d.ts | 81 ++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 4 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 009e31e1..0d9f2657 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -230,7 +230,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { db.transaction(() => { db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent) - db.prepare("INSERT INTO historical_channel_room (channel_id, room_id) VALUES (?, ?)").run(channel.id, roomID) + db.prepare("INSERT INTO historical_channel_room (reference_channel_id, room_id) VALUES (?, ?)").run(channel.id, roomID) })() return roomID @@ -571,6 +571,7 @@ module.exports.createAllForGuild = createAllForGuild module.exports.channelToKState = channelToKState module.exports.postApplyPowerLevels = postApplyPowerLevels module.exports._convertNameAndTopic = convertNameAndTopic +module.exports._syncSpaceMember = _syncSpaceMember module.exports.unbridgeChannel = unbridgeChannel module.exports.unbridgeDeletedChannel = unbridgeDeletedChannel module.exports.existsOrAutocreatable = existsOrAutocreatable diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index ce3638c1..1f816db3 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -322,14 +322,25 @@ sync.addTemporaryListener(as, "type:m.room.member", guard("m.room.member", */ async event => { if (event.state_key[0] !== "@") return + const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` - if (event.content.membership === "invite" && event.state_key === `@${reg.sender_localpart}:${reg.ooye.server_name}`) { + if (event.content.membership === "invite" && event.state_key === bot) { // We were invited to a room. We should join, and register the invite details for future reference in web. + let attemptedApiMessage = "According to unsigned invite data." + let inviteRoomState = event.unsigned?.invite_room_state + if (!Array.isArray(inviteRoomState) || inviteRoomState.length === 0) { + try { + inviteRoomState = await api.getInviteState(event.room_id) + attemptedApiMessage = "According to SSS API." + } catch (e) { + attemptedApiMessage = "According to unsigned invite data. SSS API unavailable: " + e.toString() + } + } const name = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.name", "name") const topic = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.topic", "topic") const avatar = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.avatar", "url") const creationType = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.create", "type") - if (!name) return await api.leaveRoomWithReason(event.room_id, "Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite!") + if (!name) return await api.leaveRoomWithReason(event.room_id, `Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite! (${attemptedApiMessage})`) await api.joinRoom(event.room_id) db.prepare("INSERT OR IGNORE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)").run(event.sender, event.room_id, creationType, name, topic, avatar) if (avatar) utils.getPublicUrlForMxc(avatar) // make sure it's available in the media_proxy allowed URLs @@ -342,7 +353,6 @@ async event => { db.prepare("DELETE FROM member_cache WHERE room_id = ? and mxid = ?").run(event.room_id, event.state_key) // Unregister room's use as a direct chat if the bot itself left - const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` if (event.state_key === bot) { db.prepare("DELETE FROM direct WHERE room_id = ?").run(event.room_id) } diff --git a/src/matrix/api.js b/src/matrix/api.js index e529d0f8..d0892ff8 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -137,6 +137,24 @@ function getStateEvent(roomID, type, key) { return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state/${type}/${key}`) } +/** + * @param {string} roomID + * @returns {Promise} + */ +async function getInviteState(roomID) { + /** @type {Ty.R.SSS} */ + const root = await mreq.mreq("POST", "/client/unstable/org.matrix.simplified_msc3575/sync", { + room_subscriptions: { + [roomID]: { + timeline_limit: 0, + required_state: [] + } + } + }) + const roomResponse = root.rooms[roomID] + return "stripped_state" in roomResponse ? roomResponse.stripped_state : roomResponse.invite_state +} + /** * "Any of the AS's users must be in the room. This API is primarily for Application Services and should be faster to respond than /members as it can be implemented more efficiently on the server." * @param {string} roomID @@ -483,6 +501,7 @@ module.exports.getEvent = getEvent module.exports.getEventForTimestamp = getEventForTimestamp module.exports.getAllState = getAllState module.exports.getStateEvent = getStateEvent +module.exports.getInviteState = getInviteState module.exports.getJoinedMembers = getJoinedMembers module.exports.getMembers = getMembers module.exports.getHierarchy = getHierarchy diff --git a/src/types.d.ts b/src/types.d.ts index cafd9bea..f9488b97 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -166,6 +166,37 @@ export namespace Event { content: any } + export type InviteStrippedState = { + type: string + state_key: string + sender: string + content: Event.M_Room_Create | Event.M_Room_Name | Event.M_Room_Avatar | Event.M_Room_Topic | Event.M_Room_JoinRules | Event.M_Room_CanonicalAlias + } + + export type M_Room_Create = { + additional_creators: string[] + "m.federate"?: boolean + room_version: string + type?: string + predecessor?: { + room_id: string + event_id?: string + } + } + + export type M_Room_JoinRules = { + join_rule: "public" | "knock" | "invite" | "private" | "restricted" | "knock_restricted" + allow?: { + type: string + room_id: string + }[] + } + + export type M_Room_CanonicalAlias = { + alias?: string + alt_aliases?: string[] + } + export type M_Room_Message = { msgtype: "m.text" | "m.emote" body: string @@ -375,8 +406,58 @@ export namespace R { room_id: string servers: string[] } + + export type SSS = { + pos: string + lists: { + [list_key: string]: { + count: number + } + } + rooms: { + [room_id: string]: { + bump_stamp: number + /** Omitted if user not in room (peeking) */ + membership?: Membership + /** Names of lists that match this room */ + lists: string[] + } + // If user has been in the room - at least, that's what the spec says. Synapse returns some of these, such as `name` and `avatar`, for invites as well. Go nuts. + & { + name?: string + avatar?: string + heroes?: any[] + /** According to account data */ + is_dm?: boolean + /** If false, omitted fields are unchanged from their previous value. If true, omitted fields means the fields are not set. */ + initial?: boolean + expanded_timeline?: boolean + required_state?: Event.StateOuter[] + timeline_events?: Event.Outer[] + prev_batch?: string + limited?: boolean + num_live?: number + joined_count?: number + invited_count?: number + notification_count?: number + highlight_count?: number + } + // If user is invited or knocked + & ({ + /** @deprecated */ + invite_state: Event.InviteStrippedState[] + } | { + stripped_state: Event.InviteStrippedState[] + }) + } + extensions: { + [extension_key: string]: any + } + } } +export type Membership = "invite" | "knock" | "join" | "leave" | "ban" + export type Pagination = { chunk: T[] next_batch?: string From 6d0a98eb49f37ca7c4c1a87295cb3f14d88f1b99 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 28 Nov 2025 17:20:44 +1300 Subject: [PATCH 030/153] Fix unbridging procedure --- src/d2m/actions/create-room.js | 4 ++-- src/m2d/event-dispatcher.js | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 0d9f2657..d76b1a75 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -437,7 +437,7 @@ async function unbridgeChannel(channelID) { async function unbridgeDeletedChannel(channel, guildID) { const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get() assert.ok(roomID) - const row = from("guild_space").join("guild_active", "guild_id").select("space_id", "autocreate").get() + const row = from("guild_space").join("guild_active", "guild_id").select("space_id", "autocreate").where({guild_id: guildID}).get() assert.ok(row) let botInRoom = true @@ -461,7 +461,7 @@ async function unbridgeDeletedChannel(channel, guildID) { // delete webhook on discord const webhook = select("webhook", ["webhook_id", "webhook_token"], {channel_id: channel.id}).get() if (webhook) { - await discord.snow.webhook.deleteWebhook(webhook.webhook_id, webhook.webhook_token) + await discord.snow.webhook.deleteWebhook(webhook.webhook_id, webhook.webhook_token).catch(() => {}) db.prepare("DELETE FROM webhook WHERE channel_id = ?").run(channel.id) } diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 1f816db3..c102d018 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -7,6 +7,7 @@ const util = require("util") const Ty = require("../types") const {discord, db, sync, as, select} = require("../passthrough") +const {tag} = require("@cloudrac3r/html-template-tag") /** @type {import("./actions/send-event")} */ const sendEvent = sync.require("./actions/send-event") @@ -121,10 +122,10 @@ async function sendError(roomID, source, type, e, payload) { // Where const stack = stringifyErrorStack(e) - builder.addLine(`Error trace:\n${stack}`, `
Error trace
${stack}
`) + builder.addLine(`Error trace:\n${stack}`, tag`
Error trace
${stack}
`) // How - builder.addLine("", `
Original payload
${util.inspect(payload, false, 4, false)}
`) + builder.addLine("", tag`
Original payload
${util.inspect(payload, false, 4, false)}
`) } // Send From 20453aab4391d1bdf782f147fc83323bb009c986 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Dec 2025 16:03:27 +1300 Subject: [PATCH 031/153] res.json type fixes --- scripts/setup.js | 8 ++------ src/d2m/actions/register-pk-user.js | 1 + src/matrix/mreq.js | 1 + 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/setup.js b/scripts/setup.js index ecef03d8..07ded566 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -50,11 +50,6 @@ let {reg, getTemplateRegistration, writeRegistration, readRegistration, checkReg const {setupEmojis} = require("../src/m2d/actions/setup-emojis") -function die(message) { - console.error(message) - process.exit(1) -} - async function suggestWellKnown(serverUrlPrompt, url, otherwise) { try { var json = await fetch(`${url}/.well-known/matrix/client`).then(res => res.json()) @@ -80,6 +75,7 @@ async function validateHomeserverOrigin(serverUrlPrompt, url) { return e.message } try { + /** @type {any} */ var json = await res.json() if (!Array.isArray(json?.versions) || !json.versions.includes("v1.11")) { return `OOYE needs Matrix version v1.11, but ${url} doesn't support this` @@ -130,7 +126,7 @@ function defineEchoHandler() { initial: "6693" }) portResponse.socket = +portResponse.socket || portResponse.socket // convert to number if numeric - + const app = createApp() app.use(defineEchoHandler()) const server = createServer(toNodeListener(app)) diff --git a/src/d2m/actions/register-pk-user.js b/src/d2m/actions/register-pk-user.js index e17f0613..e73fa44c 100644 --- a/src/d2m/actions/register-pk-user.js +++ b/src/d2m/actions/register-pk-user.js @@ -22,6 +22,7 @@ async function fetchMessage(messageID) { throw new Error(`Failed to connect to PK API: ${networkError.toString()}`) } if (!res.ok) throw new Error(`PK API returned an error: ${await res.text()}`) + /** @type {any} */ const root = await res.json() if (!root.member) throw new Error(`PK API didn't return member data: ${JSON.stringify(root)}`) return root diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index 384950ed..bb359752 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -72,6 +72,7 @@ async function mreq(method, url, bodyIn, extra = {}) { }, extra) const res = await fetch(baseUrl + url, opts) + /** @type {any} */ const root = await res.json() if (!res.ok || root.errcode) { From 3fc51cef15eadfc9d354fd5f5e1834306929cafe Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Dec 2025 16:38:38 +1300 Subject: [PATCH 032/153] Add upgraded_timestamp to historical_channel_room --- src/d2m/actions/create-room.js | 2 +- src/d2m/converters/message-to-event.js | 34 +- .../migrations/0026-make-rooms-historical.sql | 3 +- src/db/migrations/0027-analyze.sql | 372 +++++++++--------- src/db/orm-defs.d.ts | 1 + test/ooye-test-data.sql | 2 +- 6 files changed, 207 insertions(+), 207 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index d76b1a75..5b7d3be1 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -230,7 +230,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { db.transaction(() => { db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent) - db.prepare("INSERT INTO historical_channel_room (reference_channel_id, room_id) VALUES (?, ?)").run(channel.id, roomID) + db.prepare("INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) VALUES (?, ?, 0)").run(channel.id, roomID) })() return roomID diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index bafe0187..fecd2d44 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -351,27 +351,31 @@ async function messageToEvent(message, guild, options = {}, di) { for (const match of [...content.matchAll(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/[0-9]+\/([0-9]+)\/([0-9]+)/g)]) { assert(typeof match.index === "number") const [_, channelID, messageID] = match - let result + const result = await (async () => { + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("event_id", "room_id").where({message_id: messageID}).get() + // const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() + if (row) { + const via = await getViaServersMemo(row.room_id) + return `https://matrix.to/#/${row.room_id}/${row.event_id}?${via}` + } - const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() - if (roomID) { - const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get() - const via = await getViaServersMemo(roomID) - if (eventID && roomID) { - result = `https://matrix.to/#/${roomID}/${eventID}?${via}` - } else { - const ts = dUtils.snowflakeToTimestampExact(messageID) + const ts = dUtils.snowflakeToTimestampExact(messageID) + const oldestRow = from("historical_channel_room").selectUnsafe("max(upgraded_timestamp)", "room_id") + .where({reference_channel_id: channelID}).and("and upgraded_timestamp < ?").get(ts) + if (oldestRow?.room_id) { + const via = await getViaServersMemo(oldestRow.room_id) try { - const {event_id} = await di.api.getEventForTimestamp(roomID, ts) - result = `https://matrix.to/#/${roomID}/${event_id}?${via}` + const {event_id} = await di.api.getEventForTimestamp(oldestRow.room_id, ts) + return `https://matrix.to/#/${oldestRow.room_id}/${event_id}?${via}` } catch (e) { // M_NOT_FOUND: Unable to find event from in direction Direction.FORWARDS - result = `[unknown event, timestamp resolution failed, in room: https://matrix.to/#/${roomID}?${via}]` + return `[unknown event, timestamp resolution failed, in room: https://matrix.to/#/${oldestRow.room_id}?${via}]` } } - } else { - result = `${match[0]} [event is from another server]` - } + + return `${match[0]} [event is from another server]` + })() content = content.slice(0, match.index + offset) + result + content.slice(match.index + match[0].length + offset) offset += result.length - match[0].length diff --git a/src/db/migrations/0026-make-rooms-historical.sql b/src/db/migrations/0026-make-rooms-historical.sql index f9f4b483..ba4775ef 100644 --- a/src/db/migrations/0026-make-rooms-historical.sql +++ b/src/db/migrations/0026-make-rooms-historical.sql @@ -7,11 +7,12 @@ CREATE TABLE "historical_channel_room" ( "historical_room_index" INTEGER NOT NULL, "reference_channel_id" TEXT NOT NULL, "room_id" TEXT NOT NULL UNIQUE, + "upgraded_timestamp" INTEGER NOT NULL, PRIMARY KEY("historical_room_index" AUTOINCREMENT), FOREIGN KEY("reference_channel_id") REFERENCES "channel_room"("channel_id") ON DELETE CASCADE ); -INSERT INTO historical_channel_room (reference_channel_id, room_id) SELECT channel_id, room_id FROM channel_room; +INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) SELECT channel_id, room_id, 0 FROM channel_room; -- *** message_channel -> message_room *** diff --git a/src/db/migrations/0027-analyze.sql b/src/db/migrations/0027-analyze.sql index 03573939..f66e0c11 100644 --- a/src/db/migrations/0027-analyze.sql +++ b/src/db/migrations/0027-analyze.sql @@ -5,107 +5,102 @@ BEGIN TRANSACTION; ANALYZE sqlite_schema; DELETE FROM "sqlite_stat1"; -INSERT INTO "sqlite_stat1" ("tbl","idx","stat") VALUES ('reaction','reaction','4706 1'), -('event_message','event_message','535744 1 1'), -('historical_channel_room','sqlite_autoindex_historical_channel_room_1','996 1'), -('message_room','message_room','508753 1'), -('invite','invite','3 2 1'), +INSERT INTO "sqlite_stat1" ("tbl","idx","stat") VALUES ('reaction','reaction','4741 1'), +('event_message','event_message','537386 1 1'), +('message_room','message_room','510262 1'), +('historical_channel_room','sqlite_autoindex_historical_channel_room_1','991 1'), ('auto_emoji','auto_emoji','2 1'), -('sim','sim','1072 1'), +('sim','sim','1075 1'), ('webhook','webhook','205 1'), -('channel_room','channel_room','996 1'), -('channel_room','sqlite_autoindex_channel_room_1','996 1'), +('channel_room','channel_room','992 1'), +('channel_room','sqlite_autoindex_channel_room_1','992 1'), ('guild_active','guild_active','45 1'), -('media_proxy','media_proxy','19581 1'), -('sim_member','sim_member','5501 6 1'), -('emoji','emoji','3470 1'), -('guild_space','guild_space','42 1'), +('media_proxy','media_proxy','19794 1'), +('sim_member','sim_member','5504 6 1'), +('emoji','emoji','3472 1'), +('guild_space','guild_space','43 1'), ('member_power','member_power','1 1 1'), ('sim_proxy','sim_proxy','213 1'), ('migration',NULL,'1'), -('member_cache','member_cache','1099 3 1'), -('direct','direct','1 1'), -('file','file','36309 1'), +('member_cache','member_cache','1117 3 1'), +('file','file','36489 1'), ('lottie','lottie','22 1'); DELETE FROM "sqlite_stat4"; -INSERT INTO "sqlite_stat4" ("tbl","idx","neq","nlt","ndlt","sample") VALUES ('reaction','reaction','1','522','522',X'02069c21bd28f26ae025'), - ('reaction','reaction','1','1045','1045',X'0206b8b64d2a67851518'), - ('reaction','reaction','1','1568','1568',X'0206d45580ee5b75848e'), - ('reaction','reaction','1','2091','2091',X'0206f103da3779e1cf70'), - ('reaction','reaction','1','2614','2614',X'02061028b6f24ae55f1a'), - ('reaction','reaction','1','3137','3137',X'02062bd1ffb636826ad6'), - ('reaction','reaction','1','3660','3660',X'020647c632b246bf22ae'), - ('reaction','reaction','1','4183','4183',X'020664ebb265471ad34e'), - ('event_message','event_message','11 1','14790 14800','14356 14800',X'033365313135323030333736393431373038393034342471375f4b666676763631582d30794639334b7532776459377a56553068745a744c2d705272367142473851'), - ('event_message','event_message','11 1','33809 33817','32914 33817',X'03336531313537363033363837303537383636373632245646504363595144413134586c664d4f5041786c58596b4432595a503562656d6f63336b39787735596141'), - ('event_message','event_message','1 1','59527 59527','57719 59527',X'03336531313636383130343235323637303737323031244e343074365465464f3841585255766a353568312d31345f3533476c686b586a43364b6142534442647677'), - ('event_message','event_message','10 1','104218 104225','100236 104225',X'03336531313838373238363331323637313736343538246c6f35326a6332723637734b4b5f4b767a4361702d62627573506d423931454f76776f5f614a6a5a6e6f4d'), - ('event_message','event_message','11 1','116172 116174','111525 116174',X'03336531313933363837333730363137333237363536244546764673764c6c62316a446a4f616b4838584f68636d556161316f6d7651494143374575723146686567'), - ('event_message','event_message','10 1','116223 116230','111530 116230',X'03336531313933363837363335343238393231343636246a7243703367546f676a4930525a315561655931577978724d67737874704431494b3769454d6972754e73'), - ('event_message','event_message','1 1','119055 119055','114207 119055',X'0333653131393530353035383636333130363537353124335561353079304c7044386248725f7142574f44306d6569794f37524a4c746238755755725a716c665477'), - ('event_message','event_message','16 1','140286 140286','134379 140286',X'033365313230393337353635343731383438303432352430347938482d444f49634270597873576b31756d417a4e424e3061774634487457636e3177426476755255'), - ('event_message','event_message','11 1','162080 162090','154932 162090',X'0333653132323434383135393033313937373537373424784753744170626668647074526f4a755a2d4d416c557446424e524939344f7661506e4a55694952366138'), - ('event_message','event_message','1 1','178583 178583','170598 178583',X'033365313233333436343838383836393931363736352444643279534c3857704f666268576b626d684758716c49746f6e7237384b6656456956334b664a43702d38'), - ('event_message','event_message','11 1','178659 178662','170672 178662',X'03336531323333353238303533323338333337353537244e482d334a42617864424f53724c3259766b715534696c6e54724d485f57454973524b556c684b744f4977'), - ('event_message','event_message','11 1','215266 215274','205302 215274',X'03336531323533373435373636373337313231333632246a4b63784e434f784b627558315746346e59696538783472753559724a2d756c38455756715731796a4349'), - ('event_message','event_message','11 1','224498 224504','213831 224504',X'03336531323539323938353832323330363031373538246646483237794b4f75554a5943723671762d676b324b6f636f784663574b766c654b3167636d6445377177'), - ('event_message','event_message','10 1','224585 224593','213840 224593',X'03336531323539333030303430353335353732353230247164484754527353717138524376756735435349457a6d73717633396c4335746e334942487a4b6366646b'), - ('event_message','event_message','1 1','238111 238111','226382 238111',X'0333653132363833333233363330323132383735333724766f6261774b476a5742357770795970364c4a3931615136524a6b7330597a7a717a6e715953344d325467'), - ('event_message','event_message','1 1','297639 297639','282242 297639',X'0333653133303634313236363735393832373837333624502d47394f59667430735143334d6a475a532d445242376a586b4e62366147734c6a6b39544c7561434763'), - ('event_message','event_message','11 1','304605 304606','288785 304606',X'03336531333131353731333337393331363537323737243031717a505a477a35764a475276464f716533653570513930705a6a43644c326a33485635415571504559'), - ('event_message','event_message','10 1','322247 322256','305183 322256',X'03336531333234353034393632373530383737373937247a58477647646d614b6f6e4a497232767246354c523235796e4e4b476b5a357a774a696a44645433694f49'), - ('event_message','event_message','11 1','327028 327032','309699 327032',X'0333653133323736353831313733343439323336383024574d774330644574417277375f4562554a534465546532577174506d3747584347774570646c4f79326d30'), - ('event_message','event_message','1 1','357167 357167','338131 357167',X'033365313334353735363437313239393933363237362436755847477632536e727a63346353616e3937686a6562356a6a77516d6d4c66464f4c305f74316d627130'), - ('event_message','event_message','10 1','365779 365781','346205 365781',X'03336531333439393933323736353839323135373835244f484162596f6b67435748704a36556a6e74545f74774452484743624e52656e617a5a3658776f464a4e55'), - ('event_message','event_message','1 1','416695 416695','394011 416695',X'0333653133363737343039323037343035383134363624486e346e6d784571634758767153577a7343656a586733696c517930554869705f34547073504e71684f55'), - ('event_message','event_message','11 1','422263 422266','399248 422266',X'0333653133363938333439303532363130343738313824436a34597373765a46544877415a5236724c77472d635250716c426b7a3749626d473261766653562d4e55'), - ('event_message','event_message','1 1','476223 476223','449789 476223',X'033365313430313837343437373832313732363834312455414d51397a6c2d775a39572d327231687866584162746f447143314734646239554b384573596e657359'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','8 8','8 8',X'034b022141534d746248706d6f4b4e736765574274733a636164656e63652e6d6f65025d'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','18 18','18 18',X'034b02214173786a53777176484e444f4665587a676b3a636164656e63652e6d6f65014c'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','94 94','94 94',X'034b0221464d5346425a536d59596964656f4a58594b3a636164656e63652e6d6f65017e'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','100 100','100 100',X'034b0221466875676b616e45716943627448734644483a636164656e63652e6d6f6502d2'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','103 103','103 103',X'034b0221466a7167556258447a474a75796a464746613a636164656e63652e6d6f6503c6'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','110 110','110 110',X'034b0221474650514f614e7a517465534c4b485374543a636164656e63652e6d6f65013e'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','221 221','221 221',X'034b02214c506d4c664f6b796d63646d725644624a463a636164656e63652e6d6f6501ee'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','311 311','311 311',X'034b02215157735a4a7042716c716548686c616962443a636164656e63652e6d6f65029a'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','332 332','332 332',X'034b022152547356547675424f4555506d52687347633a636164656e63652e6d6f650255'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','408 408','408 408',X'034b022155635059696f48454f426761664f576b694f3a636164656e63652e6d6f6502ea'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','443 443','443 443',X'034b01215754625a53764a66524a72736348574b78563a636164656e63652e6d6f6529'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','450 450','450 450',X'034b022157695842616d5676586279476565657a50413a636164656e63652e6d6f650157'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','468 468','468 468',X'034b022158717470725744744f5761667a72486f477a3a636164656e63652e6d6f65018b'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','526 526','526 526',X'034b022162574b794f596c6468484b7a784b757667623a636164656e63652e6d6f650127'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','554 554','554 554',X'034b0221637779454c6c6b55714a565942646a4250543a636164656e63652e6d6f6501b0'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','597 597','597 597',X'034b02216668587668437279724e525661777572636f3a636164656e63652e6d6f6503bd'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','608 608','608 608',X'034b022167636d734472716442706f7463586b4545703a636164656e63652e6d6f6501d5'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','665 665','665 665',X'034b02216a6c4e64496a62687654486b5a74745166773a636164656e63652e6d6f650131'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','708 708','708 708',X'034b01216d454c5846716a426958726d7558796943723a636164656e63652e6d6f656b'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','768 768','768 768',X'034b02216f7a6f494e55494261685177775a6f586b7a3a636164656e63652e6d6f6501d8'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','776 776','776 776',X'034b022170506f63657a415046506f584c6a6a5750443a636164656e63652e6d6f6503aa'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','877 877','877 877',X'034b02217474644e4b6b4b4f49757879566869794b6a3a636164656e63652e6d6f6502e1'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','887 887','887 887',X'034b02217568496f63597a4e41714244694b5645506d3a636164656e63652e6d6f650080'), - ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','943 943','943 943',X'034b02217863655a4641624250425a6d7842677a51523a636164656e63652e6d6f65036b'), - ('message_room','message_room','1','56528','56528',X'023331313635393437353536363733363334333635'), - ('message_room','message_room','1','113057','113057',X'023331313932353536323035343730363538363132'), - ('message_room','message_room','1','169586','169586',X'023331323331373432353832313033373430343937'), - ('message_room','message_room','1','226115','226115',X'023331323635333430353738363135333934333034'), - ('message_room','message_room','1','282644','282644',X'023331333033373933303435323632353635343637'), - ('message_room','message_room','1','339173','339173',X'023331333434343332333234333137363735363733'), - ('message_room','message_room','1','395702','395702',X'023331333637303635303537373939393636383631'), - ('message_room','message_room','1','452231','452231',X'023331343030353931393936323333373131373238'), - ('invite','invite','1 1','0 0','0 0',X'033549406d616c6b696572693a6d61747269782e6f7267216877535462776967674c584a6f6756506b763a6d61747269782e6f7267'), - ('invite','invite','2 1','1 1','1 1',X'03334b406d65636879613a636164656e63652e6d6f652172486b466c6d724e6e664157455943494a703a636164656e63652e6d6f65'), - ('invite','invite','2 1','1 2','1 2',X'03334b406d65636879613a636164656e63652e6d6f65217549514e424f4f68735976514c645368726b3a636164656e63652e6d6f65'), +INSERT INTO "sqlite_stat4" ("tbl","idx","neq","nlt","ndlt","sample") VALUES ('reaction','reaction','1','526','526',X'02069c21bd28f26ae025'), + ('reaction','reaction','1','1053','1053',X'0206b8866f4c30c2e1aa'), + ('reaction','reaction','1','1580','1580',X'0206d43fceca129b040e'), + ('reaction','reaction','1','2107','2107',X'0206f121f9a4fe54b557'), + ('reaction','reaction','1','2634','2634',X'020610299199abbd0e9c'), + ('reaction','reaction','1','3161','3161',X'02062be99961e7716037'), + ('reaction','reaction','1','3688','3688',X'020647b48fa5ee5a415c'), + ('reaction','reaction','1','4215','4215',X'020664fdc2d88c77dda3'), + ('event_message','event_message','11 1','14790 14792','14356 14792',X'03336531313532303033373639343137303839303434244a616d4c6d732d4b77454c6b47766866344d524f385576535536336a574a5a4c4474524c4c57664f775873'), + ('event_message','event_message','11 1','33809 33816','32914 33816',X'0333653131353736303336383730353738363637363224544b7141734f58566c6e67506f546f4a427565514e664444756d494a6d38384f486a76766f7949496e7130'), + ('event_message','event_message','1 1','59709 59709','57896 59709',X'033365313136363930353332323132303637353336392442794756564f6767326a416845624267463941755056486178377a34314459514e4459316e34435a4a4455'), + ('event_message','event_message','11 1','116172 116182','111525 116182',X'0333653131393336383733373036313733323736353624786a385f70696e784f624f4349666c70556832305542345973664a547642694b4164675f473168562d5334'), + ('event_message','event_message','1 1','119419 119419','114559 119419',X'03336531313935323132333038393839383730313732244f556670664d5054576c364774734943484d725459556d6464656c636232663374494a662d425769554355'), + ('event_message','event_message','16 1','140286 140287','134379 140287',X'0333653132303933373536353437313834383034323524346b61796e4d68422d336d6967417571347255745f726639353454636b6f657636664c5f3675394f455030'), + ('event_message','event_message','11 1','162080 162086','154932 162086',X'0333653132323434383135393033313937373537373424674c77513179796e4b6d5859496b5a597a4a55627a66557a55552d714c4b5f524f454e4250325f6e44766b'), + ('event_message','event_message','11 1','178659 178659','170672 178659',X'03336531323333353238303533323338333337353537242d39304668552d36455373594b6435484d7237666d6a414a5f6a576149616e356c4776384e655436564959'), + ('event_message','event_message','1 1','179129 179129','171083 179129',X'03336531323333393533373032373637383836343636245a446e5f42385a6b41674c645939495649767445516e47373369706a555a55447943634768697851673859'), + ('event_message','event_message','10 1','180049 180052','171954 180052',X'03336531323334363237303030393333383130323637245171504b7357795254734a49695449744646716a686e506d48764a6e5932584a6c595a506b424e372d766f'), + ('event_message','event_message','11 1','215266 215271','205302 215271',X'03336531323533373435373636373337313231333632244b4936672d57724f5a5757533463534c4c4f353950555176425066754b5f5446504b443233583130504759'), + ('event_message','event_message','11 1','224498 224499','213831 224499',X'0333653132353932393835383232333036303137353824356a573361764d37626d643661756c7367635650506f5257417552476e30503477324939786b5675326f6b'), + ('event_message','event_message','10 1','224519 224523','213833 224523',X'033365313235393239383739353234323635353839352452696c715a6862347a32526b594c596958504375445975546f6b6430544e365a784638737842745670346b'), + ('event_message','event_message','10 1','224615 224616','213843 224616',X'0333653132353933303036363636373831383139323024425a69396d4c73323034344c674a6e56673761557a614467484b4b5545787334587a467954474245585573'), + ('event_message','event_message','1 1','238839 238839','227061 238839',X'0333653132363839343934383836303535393336343224374d3633546d416c526947553847795f416164576f4d4f4e4a334b363441326235385f6e72385961652d51'), + ('event_message','event_message','1 1','298549 298549','283096 298549',X'03336531333037303536353132313931303337343532245830424a3954514e544d3041687554736c7258744b5836383376723749524355747a4b47524a4374493555'), + ('event_message','event_message','11 1','304605 304605','288785 304605',X'03336531333131353731333337393331363537323737242d674e75657465765a426169587949335859717437325743695438396549573269514761416266384f6455'), + ('event_message','event_message','11 1','327028 327037','309699 327037',X'0333653133323736353831313733343439323336383024715055786a61394c36694e756548683046335962304b524b67665730414356394769367a4147464b714973'), + ('event_message','event_message','10 1','329549 329550','312055 329550',X'033365313332393331373735303931323435303537322430325a4779526f33656133786e5356706b52487047325459415464373971684834536632506f4e7a614773'), + ('event_message','event_message','1 1','358259 358259','339179 358259',X'03336531333436303136333531313138313634303539243364757343667558596a506f715a3642774851755a48496e5163504f4e70766c64387476654a4d45685a38'), + ('event_message','event_message','1 1','417969 417969','395237 417969',X'0333653133363831333832343230383333393336363724537a7775656948304b696130376d67304e51322d58627751352d6a7653507649464e645053396464416655'), + ('event_message','event_message','11 1','422263 422270','399248 422270',X'033365313336393833343930353236313034373831382456754f5872464d593547734350377467425f6a763348486f426264666b3859464c4b4f6e48583732497677'), + ('event_message','event_message','10 1','424260 424266','401135 424266',X'03336531333730353132353138353938303939303637246c7268447950715458362d45497a3637552d616a75453839614655394c4151556f5a356d7363725072466f'), + ('event_message','event_message','1 1','477679 477679','451062 477679',X'0333653134303434353430323035313234383133333324524f454b6b5f726b3373344b7451337a75344552774c4b5069484964757676575f514d4b4e66306c385630'), + ('message_room','message_room','1','56695','56695',X'023331313636313031373337333834343630333739'), + ('message_room','message_room','1','113391','113391',X'023331313932353935303036363435363132353434'), + ('message_room','message_room','1','170087','170087',X'023331323331393439393133373937393535363335'), + ('message_room','message_room','1','226783','226783',X'023331323636313430343634333733383239373532'), + ('message_room','message_room','1','283479','283479',X'023331333034303933383132373833373130323539'), + ('message_room','message_room','1','340175','340175',X'023331333434383431363637333537393730343332'), + ('message_room','message_room','1','396871','396871',X'023331333637353035313132313333363638393134'), + ('message_room','message_room','1','453567','453567',X'023331343032363934353234333439373134343833'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','0 0','0 0',X'034b0221414355774c616c64303030303030303030303a636164656e63652e6d6f650288'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','24 24','24 24',X'034b02214255635a694c7a57303030303030303030303a636164656e63652e6d6f6501ab'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','110 110','110 110',X'034b022147486e4d47697875303030303030303030303a636164656e63652e6d6f6500c6'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','193 193','193 193',X'034b02214b4b535575717666303030303030303030303a636164656e63652e6d6f650350'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','221 221','221 221',X'034b02214c51715351594b73303030303030303030303a636164656e63652e6d6f6503af'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','319 319','319 319',X'034b02215170676c734e587a303030303030303030303a636164656e63652e6d6f650366'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','332 332','332 332',X'034b0221525a585a7064554f303030303030303030303a636164656e63652e6d6f65009f'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','351 351','351 351',X'034b0221534b6f6c6f636b77303030303030303030303a636164656e63652e6d6f65035f'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','443 443','443 443',X'034b0221576374435a494d73303030303030303030303a636164656e63652e6d6f650084'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','551 551','551 551',X'034b0221637779454c6c6b55303030303030303030303a636164656e63652e6d6f6501b0'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','554 554','554 554',X'034b0221644965496d615167303030303030303030303a636164656e63652e6d6f6503a0'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','560 560','560 560',X'034b0221645568456f756a71303030303030303030303a636164656e63652e6d6f650090'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','573 573','573 573',X'034b02216552517465644b67303030303030303030303a636164656e63652e6d6f650099'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','593 593','593 593',X'034b0221666764594e526d4e303030303030303030303a636164656e63652e6d6f65016b'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','624 624','624 624',X'034b0221687078416c4c6f71303030303030303030303a636164656e63652e6d6f650297'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','625 625','625 625',X'034b02216873414570464e47303030303030303030303a636164656e63652e6d6f6500be'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','665 665','665 665',X'034b01216a71484b51424476303030303030303030303a636164656e63652e6d6f653b'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','758 758','758 758',X'034b02216f6251554d424b75303030303030303030303a636164656e63652e6d6f6500f7'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','776 776','776 776',X'034b022170566e596b5a4f46303030303030303030303a636164656e63652e6d6f650232'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','781 781','781 781',X'034b01217065766e6542516e303030303030303030303a636164656e63652e6d6f6518'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','857 857','857 857',X'034b02217446564c65724b78303030303030303030303a636164656e63652e6d6f65024e'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','866 866','866 866',X'034b0221745a61474145557a303030303030303030303a636164656e63652e6d6f6501f8'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','887 887','887 887',X'034b022175727a464b754d61303030303030303030303a636164656e63652e6d6f65033b'), + ('historical_channel_room','sqlite_autoindex_historical_channel_room_1','1 1','921 921','921 921',X'034b022177574a5548445a74303030303030303030303a636164656e63652e6d6f65025c'), ('auto_emoji','auto_emoji','1','0','0',X'02114c31'), ('auto_emoji','auto_emoji','1','1','1',X'02114c32'), ('sim','sim','1','119','119',X'025531316564343731342d636635652d346333372d393331382d376136353266383732636634'), - ('sim','sim','1','239','239',X'0231313530373435393839383336333038343830'), - ('sim','sim','1','359','359',X'0231323235393737393435323133313038323234'), - ('sim','sim','1','479','479',X'0231333038323937303732373738343132303332'), - ('sim','sim','1','599','599',X'0231343135373135363735313031373234363735'), - ('sim','sim','1','719','719',X'0231353732363938363739363138353638313933'), - ('sim','sim','1','839','839',X'0231373332373338333838303431343030343231'), - ('sim','sim','1','959','959',X'0231393437373336313831393732353636303636'), + ('sim','sim','1','239','239',X'0231313439363932303632313634333230323536'), + ('sim','sim','1','359','359',X'025532323533323035312d633335332d346638662d383835362d653137383831323435303763'), + ('sim','sim','1','479','479',X'0231333036373839323436333237353836383136'), + ('sim','sim','1','599','599',X'0231343132383438323830343635313738363235'), + ('sim','sim','1','719','719',X'0231353638323430303837363238373735343234'), + ('sim','sim','1','839','839',X'0231373234383037393132373233313835373534'), + ('sim','sim','1','959','959',X'0231393431303333313033353936353835303630'), ('webhook','webhook','1','22','22',X'023331313630383933333337303239353836393536'), ('webhook','webhook','1','45','45',X'023331323139343938393236343636363632343330'), ('webhook','webhook','1','68','68',X'023331323432383939363632343734373131303630'), @@ -116,36 +111,36 @@ INSERT INTO "sqlite_stat4" ("tbl","idx","neq","nlt","ndlt","sample") VALUES ('re ('webhook','webhook','1','183','183',X'0231363035353930343336333230333738383930'), ('channel_room','channel_room','1','110','110',X'023331313939353030313137393834363733393133'), ('channel_room','channel_room','1','221','221',X'023331323734313935333432323131393430353434'), - ('channel_room','channel_room','1','332','332',X'023331333437303939353439343430363735383732'), + ('channel_room','channel_room','1','332','332',X'023331333437303036333637393639343433383430'), ('channel_room','channel_room','1','443','443',X'023331343035323432323838343138303632333636'), ('channel_room','channel_room','1','554','554',X'023331343036373736363630393936333935323830'), ('channel_room','channel_room','1','665','665',X'023331343039363536363537383835323635393830'), ('channel_room','channel_room','1','776','776',X'023331343139353132333134363234383638343632'), - ('channel_room','channel_room','1','887','887',X'0231333735373135343638383937353530333631'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','8 8','8 8',X'034b332141534d746248706d6f4b4e736765574274733a636164656e63652e6d6f6531343037333239313938393039303330353135'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','18 18','18 18',X'034b33214173786a53777176484e444f4665587a676b3a636164656e63652e6d6f6531333437303036333637393639343433383430'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','94 94','94 94',X'034b3321464d5346425a536d59596964656f4a58594b3a636164656e63652e6d6f6531343034313333323339303736393530313631'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','100 100','100 100',X'034b3321466875676b616e45716943627448734644483a636164656e63652e6d6f6531343132383936323333373839353837353036'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','103 103','103 103',X'034b3121466a7167556258447a474a75796a464746613a636164656e63652e6d6f65383034363236313139333938393136313037'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','110 110','110 110',X'034b3321474650514f614e7a517465534c4b485374543a636164656e63652e6d6f6531333435363431343537323035323532313737'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','221 221','221 221',X'034b33214c506d4c664f6b796d63646d725644624a463a636164656e63652e6d6f6531343035393733313637323535353834383830'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','311 311','311 311',X'034b33215157735a4a7042716c716548686c616962443a636164656e63652e6d6f6531343039363536363537383835323635393830'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','332 332','332 332',X'034b332152547356547675424f4555506d52687347633a636164656e63652e6d6f6531343037323235393932313935333432343237'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','408 408','408 408',X'034b332155635059696f48454f426761664f576b694f3a636164656e63652e6d6f6531343134303334313437323032303434303134'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','443 443','443 443',X'034b33215754625a53764a66524a72736348574b78563a636164656e63652e6d6f6531313433313231353134393235393238353431'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','450 450','450 450',X'034b332157695842616d5676586279476565657a50413a636164656e63652e6d6f6531333536353037353335313132323738303839'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','468 468','468 468',X'034b332158717470725744744f5761667a72486f477a3a636164656e63652e6d6f6531343034353137353530393634303830363530'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','526 526','526 526',X'034b332162574b794f596c6468484b7a784b757667623a636164656e63652e6d6f6531333239353237383038343232333138313030'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','554 554','554 554',X'034b3321637779454c6c6b55714a565942646a4250543a636164656e63652e6d6f6531343034393538363332363830303939393931'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','597 597','597 597',X'034b31216668587668437279724e525661777572636f3a636164656e63652e6d6f65373535373235353231333731303034393739'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','608 608','608 608',X'034b332167636d734472716442706f7463586b4545703a636164656e63652e6d6f6531343035353833303939383532363139393137'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','665 665','665 665',X'034b33216a6c4e64496a62687654486b5a74745166773a636164656e63652e6d6f6531333339343132353232373538393633323030'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','708 708','708 708',X'034b33216d454c5846716a426958726d7558796943723a636164656e63652e6d6f6531313936393134373631303430393234373432'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','768 768','768 768',X'034b33216f7a6f494e55494261685177775a6f586b7a3a636164656e63652e6d6f6531343035363036393133303638313039383735'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','776 776','776 776',X'034b312170506f63657a415046506f584c6a6a5750443a636164656e63652e6d6f65363832333334343939393136333439353231'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','877 877','877 877',X'034b33217474644e4b6b4b4f49757879566869794b6a3a636164656e63652e6d6f6531343133323134313538383631373632363430'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','887 887','887 887',X'034b33217568496f63597a4e41714244694b5645506d3a636164656e63652e6d6f6531323137383839303233393831323536373534'), - ('channel_room','sqlite_autoindex_channel_room_1','1 1','943 943','943 943',X'034b31217863655a4641624250425a6d7842677a51523a636164656e63652e6d6f65323937323734343934303737333730333638'), + ('channel_room','channel_room','1','887','887',X'0231333734383732393736313738343133353639'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','23 23','23 23',X'034b3121425167434a4d4c78303030303030303030303a636164656e63652e6d6f65393631373335333036303032393732373432'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','96 96','96 96',X'034b332146514f654f667747303030303030303030303a636164656e63652e6d6f6531323137393638383531303939313839323738'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','110 110','110 110',X'034b332147486e4d47697875303030303030303030303a636164656e63652e6d6f6531323432323436333730303938363739383838'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','138 138','138 138',X'034b3321484a79705a6b6863303030303030303030303a636164656e63652e6d6f6531303237323933303239313633333335373130'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','161 161','161 161',X'034b33214962646466626172303030303030303030303a636164656e63652e6d6f6531323937373538303931373331303039353536'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','221 221','221 221',X'034b31214c51715351594b73303030303030303030303a636164656e63652e6d6f65373039303431393733353332363838343235'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','240 240','240 240',X'034b33214d5071594e414a62303030303030303030303a636164656e63652e6d6f6531323139303338323638323835323539393037'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','250 250','250 250',X'034b33214e414f484c4e444c303030303030303030303a636164656e63652e6d6f6531343037323332343832313338333934363634'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','325 325','325 325',X'034b33215178576669464359303030303030303030303a636164656e63652e6d6f6531343034353739343736363837323934363434'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','332 332','332 332',X'034b33215254735654767542303030303030303030303a636164656e63652e6d6f6531343037323235393932313935333432343237'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','430 430','430 430',X'034b33215673656a6b6b5a71303030303030303030303a636164656e63652e6d6f6531323235323636343030363838333431303833'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','443 443','443 443',X'034b3321576241744a736c6b303030303030303030303a636164656e63652e6d6f6531343230323635333433363931313332393339'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','552 552','552 552',X'034b3321637779454c6c6b55303030303030303030303a636164656e63652e6d6f6531343034393538363332363830303939393931'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','554 554','554 554',X'034b332164484e5378484a47303030303030303030303a636164656e63652e6d6f6531343035363439333331343335393939323932'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','565 565','565 565',X'034b3321646c584f50766944303030303030303030303a636164656e63652e6d6f6531323735353037363433323231343039393033'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','579 579','579 579',X'034b332165656c6c7a6a5370303030303030303030303a636164656e63652e6d6f6531343036373736363630393936333935323830'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','619 619','619 619',X'034b332168525179596e6d4e303030303030303030303a636164656e63652e6d6f6531343237323832333338303035353136343233'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','664 664','664 664',X'034b33216a6c566e54585747303030303030303030303a636164656e63652e6d6f6531323139353034373636333137373536343238'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','665 665','665 665',X'034b33216a6c6c4479666d76303030303030303030303a636164656e63652e6d6f6531303835303935353736383731333837313936'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','776 776','776 776',X'034b332170555653686b7978303030303030303030303a636164656e63652e6d6f6531323139343939353736363137303738383735'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','813 813','813 813',X'034b33217179416246555961303030303030303030303a636164656e63652e6d6f6531343035393130313436363731393732333833'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','887 887','887 887',X'034b33217571697357484575303030303030303030303a636164656e63652e6d6f6531323331383036353337373032353736313938'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','924 924','924 924',X'034b332177585242634d4851303030303030303030303a636164656e63652e6d6f6531343233373338343430363833363232353332'), + ('channel_room','sqlite_autoindex_channel_room_1','1 1','953 953','953 953',X'034b33217954757a6749556f303030303030303030303a636164656e63652e6d6f6531333338353537373531373232323530333130'), ('guild_active','guild_active','1','5','5',X'023331313433333336323438373631363437313534'), ('guild_active','guild_active','1','11','11',X'023331313630383933333336333234393331353834'), ('guild_active','guild_active','1','17','17',X'023331323839353936343835343631323137333430'), @@ -153,54 +148,54 @@ INSERT INTO "sqlite_stat4" ("tbl","idx","neq","nlt","ndlt","sample") VALUES ('re ('guild_active','guild_active','1','29','29',X'023331343338363132393630393137353836313233'), ('guild_active','guild_active','1','35','35',X'0231343937313539373236343535343535373534'), ('guild_active','guild_active','1','41','41',X'0231383730313138363530373638363730373530'), - ('media_proxy','media_proxy','1','2175','2175',X'02069ccd283fa260cb90'), - ('media_proxy','media_proxy','1','4351','4351',X'0206b95c95635f40b3d8'), - ('media_proxy','media_proxy','1','6527','6527',X'0206d546a2d00310b6cc'), - ('media_proxy','media_proxy','1','8703','8703',X'0206f0b1fb70331afcb3'), - ('media_proxy','media_proxy','1','10879','10879',X'02060e48cd55947d2372'), - ('media_proxy','media_proxy','1','13055','13055',X'02062ac70844c1762329'), - ('media_proxy','media_proxy','1','15231','15231',X'0206470a46ab937760c8'), - ('media_proxy','media_proxy','1','17407','17407',X'0206632a367225280573'), - ('sim_member','sim_member','225 1','14 104','4 104',X'034b4521414956694e775a64636b4652764c4f4567433a636164656e63652e6d6f65405f6f6f79655f6a6a6a6a6a363634343a636164656e63652e6d6f65'), - ('sim_member','sim_member','124 1','598 611','85 611',X'034b4921457a54624a496c496d45534f746b4e644e4a3a636164656e63652e6d6f65405f6f6f79655f61726a756e3034323236393a636164656e63652e6d6f65'), - ('sim_member','sim_member','35 1','817 845','107 845',X'034b532147486e4d476978756867527255546d4b77683a636164656e63652e6d6f65405f6f6f79655f6e65637461726f66616d62726f7369613a636164656e63652e6d6f65'), - ('sim_member','sim_member','63 1','943 948','140 948',X'034b452148725979716b6f7942485a4a4e634455564f3a636164656e63652e6d6f65405f6f6f79655f5f706b5f64717a76613a636164656e63652e6d6f65'), - ('sim_member','sim_member','48 1','1022 1037','148 1037',X'034b47214943566475566c646e6774484b64567070503a636164656e63652e6d6f65405f6f6f79655f5f706b5f7078677679783a636164656e63652e6d6f65'), - ('sim_member','sim_member','39 1','1203 1223','174 1223',X'034b45214a48614a714258706d6b49654963615562513a636164656e63652e6d6f65405f6f6f79655f6d69647473756d61723a636164656e63652e6d6f65'), - ('sim_member','sim_member','48 1','1736 1781','289 1781',X'034b472150747969527851614879636777626c636f743a636164656e63652e6d6f65405f6f6f79655f76616e746164656c69613a636164656e63652e6d6f65'), - ('sim_member','sim_member','5 1','1834 1835','299 1835',X'034b472151544372636e695373616f626957444c516f3a636164656e63652e6d6f65405f6f6f79655f68756d616e67616d65723a636164656e63652e6d6f65'), - ('sim_member','sim_member','64 1','2099 2137','353 2137',X'034b4721536c7664497a734f6f534469434e6a6e77733a636164656e63652e6d6f65405f6f6f79655f5f706b5f777768667a6f3a636164656e63652e6d6f65'), - ('sim_member','sim_member','81 1','2215 2276','361 2276',X'034b4321544f61794476734c735a566d5779745166483a636164656e63652e6d6f65405f6f6f79655f6171756173316d703a636164656e63652e6d6f65'), - ('sim_member','sim_member','42 1','2370 2389','373 2389',X'034b47215468436b4b5857434a77657451496d747a573a636164656e63652e6d6f65405f6f6f79655f5f706b5f7465736361723a636164656e63652e6d6f65'), - ('sim_member','sim_member','36 1','2424 2447','380 2447',X'034b472154716c79516d69667847556767456d64424e3a636164656e63652e6d6f65405f6f6f79655f657665727970697a7a613a636164656e63652e6d6f65'), - ('sim_member','sim_member','65 1','2691 2723','438 2723',X'034b472157755a5549494e7457456a64656658414c473a636164656e63652e6d6f65405f6f6f79655f5f706b5f77797a63686a3a636164656e63652e6d6f65'), - ('sim_member','sim_member','2 1','3059 3059','497 3059',X'034b3f21616f764c6d776a674d6c44414c6e666c426e3a636164656e63652e6d6f65405f6f6f79655f6f6363696d793a636164656e63652e6d6f65'), - ('sim_member','sim_member','32 1','3127 3147','520 3147',X'034b392163427874565278446c5a765356684a58564b3a636164656e63652e6d6f65405f6f6f79655f626f743a636164656e63652e6d6f65'), - ('sim_member','sim_member','8 1','3666 3671','630 3671',X'034b39216966636d75794e6e544861464163636575543a636164656e63652e6d6f65405f6f6f79655f726e6c3a636164656e63652e6d6f65'), - ('sim_member','sim_member','43 1','3849 3855','668 3855',X'034b47216b6b4b714249664c45596a4b534b626b4a633a636164656e63652e6d6f65405f6f6f79655f5f706b5f6f77617a76663a636164656e63652e6d6f65'), - ('sim_member','sim_member','8 1','4280 4283','746 4283',X'034b3f216f705748554e6b46646247796b7579564b6a3a636164656e63652e6d6f65405f6f6f79655f636f6f6b69653a636164656e63652e6d6f65'), - ('sim_member','sim_member','158 1','4424 4526','770 4526',X'034b472170757146464b59487750677073554e6d6e443a636164656e63652e6d6f65405f6f6f79655f5f706b5f77797a63686a3a636164656e63652e6d6f65'), - ('sim_member','sim_member','44 1','4807 4843','824 4843',X'034b4521734b4c6f784a4e62547a6c72436d4a796f533a636164656e63652e6d6f65405f6f6f79655f7370696e6e657265743a636164656e63652e6d6f65'), - ('sim_member','sim_member','11 1','4889 4895','841 4895',X'034b45217443744769524448676e4a62505a4f5479573a636164656e63652e6d6f65405f6f6f79655f6a656d74616e756b693a636164656e63652e6d6f65'), - ('sim_member','sim_member','73 1','5069 5107','888 5107',X'034b4721766576446275617472435946704e71426d583a636164656e63652e6d6f65405f6f6f79655f5f706b5f75616e6766633a636164656e63652e6d6f65'), - ('sim_member','sim_member','59 1','5179 5207','903 5207',X'034b5f217750454472596b77497a6f744e66706e57503a636164656e63652e6d6f65405f6f6f79655f66756a69776172615f6e6f5f6d6f6b6f755f66756d6f3a636164656e63652e6d6f65'), - ('sim_member','sim_member','52 1','5438 5453','968 5453',X'034b41217a66654e574d744b4f764f48766f727979563a636164656e63652e6d6f65405f6f6f79655f666f67656c38323a636164656e63652e6d6f65'), + ('media_proxy','media_proxy','1','2199','2199',X'02069cb7709d83b92e22'), + ('media_proxy','media_proxy','1','4399','4399',X'0206b953cc685f0b68d2'), + ('media_proxy','media_proxy','1','6599','6599',X'0206d546a2d00310b6cc'), + ('media_proxy','media_proxy','1','8799','8799',X'0206f0d029ff71e1dae5'), + ('media_proxy','media_proxy','1','10999','10999',X'02060e4626697605710f'), + ('media_proxy','media_proxy','1','13199','13199',X'02062adc53c43825bc39'), + ('media_proxy','media_proxy','1','15399','15399',X'02064704c4b0f76fa5ff'), + ('media_proxy','media_proxy','1','17599','17599',X'02066338ce2423770613'), + ('sim_member','sim_member','225 1','14 80','4 80',X'034b4721414956694e775a64303030303030303030303a636164656e63652e6d6f65405f6f6f79655f66726f73745f313139323a636164656e63652e6d6f65'), + ('sim_member','sim_member','32 1','483 488','68 488',X'034b3b21455450534d664d69303030303030303030303a636164656e63652e6d6f65405f6f6f79655f653372613a636164656e63652e6d6f65'), + ('sim_member','sim_member','125 1','598 611','85 611',X'034b4921457a54624a496c49303030303030303030303a636164656e63652e6d6f65405f6f6f79655f61726a756e3034323236393a636164656e63652e6d6f65'), + ('sim_member','sim_member','35 1','818 851','107 851',X'034b472147486e4d47697875303030303030303030303a636164656e63652e6d6f65405f6f6f79655f76616e746164656c69613a636164656e63652e6d6f65'), + ('sim_member','sim_member','63 1','945 1005','141 1005',X'034b412148725979716b6f79303030303030303030303a636164656e63652e6d6f65405f6f6f79655f7669686f776c733a636164656e63652e6d6f65'), + ('sim_member','sim_member','48 1','1024 1025','149 1025',X'034b47214943566475566c64303030303030303030303a636164656e63652e6d6f65405f6f6f79655f5f706b5f6172656866723a636164656e63652e6d6f65'), + ('sim_member','sim_member','39 1','1205 1223','175 1223',X'034b41214a48614a71425870303030303030303030303a636164656e63652e6d6f65405f6f6f79655f6c6f6f6e656c613a636164656e63652e6d6f65'), + ('sim_member','sim_member','48 1','1734 1768','289 1768',X'034b47215074796952785161303030303030303030303a636164656e63652e6d6f65405f6f6f79655f6d6f6d6f7473756b692e3a636164656e63652e6d6f65'), + ('sim_member','sim_member','5 1','1832 1835','299 1835',X'034b4b2151544372636e6953303030303030303030303a636164656e63652e6d6f65405f6f6f79655f72616e646f6d6974796775793a636164656e63652e6d6f65'), + ('sim_member','sim_member','64 1','2097 2100','353 2100',X'034b4521536c7664497a734f303030303030303030303a636164656e63652e6d6f65405f6f6f79655f5f706b5f626369736c3a636164656e63652e6d6f65'), + ('sim_member','sim_member','81 1','2213 2240','361 2240',X'034b4721544f61794476734c303030303030303030303a636164656e63652e6d6f65405f6f6f79655f5f706b5f6f7a707a79633a636164656e63652e6d6f65'), + ('sim_member','sim_member','42 1','2368 2409','373 2409',X'034b49215468436b4b585743303030303030303030303a636164656e63652e6d6f65405f6f6f79655f776172736d6974686c69763a636164656e63652e6d6f65'), + ('sim_member','sim_member','36 1','2422 2447','380 2447',X'034b4b2154716c79516d6966303030303030303030303a636164656e63652e6d6f65405f6f6f79655f6a6f6b65726765726d616e793a636164656e63652e6d6f65'), + ('sim_member','sim_member','65 1','2689 2721','438 2721',X'034b472157755a5549494e74303030303030303030303a636164656e63652e6d6f65405f6f6f79655f5f706b5f77797a63686a3a636164656e63652e6d6f65'), + ('sim_member','sim_member','2 1','3058 3059','497 3059',X'034b3921616f764c6d776a67303030303030303030303a636164656e63652e6d6f65405f6f6f79655f726e6c3a636164656e63652e6d6f65'), + ('sim_member','sim_member','8 1','3666 3671','630 3671',X'034b39216966636d75794e6e303030303030303030303a636164656e63652e6d6f65405f6f6f79655f726e6c3a636164656e63652e6d6f65'), + ('sim_member','sim_member','43 1','3849 3874','668 3874',X'034b4f216b6b4b714249664c303030303030303030303a636164656e63652e6d6f65405f6f6f79655f656c656374726f6e6963353339313a636164656e63652e6d6f65'), + ('sim_member','sim_member','8 1','4280 4283','746 4283',X'034b3f216f705748554e6b46303030303030303030303a636164656e63652e6d6f65405f6f6f79655f636f6f6b69653a636164656e63652e6d6f65'), + ('sim_member','sim_member','158 1','4424 4465','770 4465',X'034b452170757146464b5948303030303030303030303a636164656e63652e6d6f65405f6f6f79655f5f706b5f6a666e747a3a636164656e63652e6d6f65'), + ('sim_member','sim_member','44 1','4810 4810','824 4810',X'034b4121734b4c6f784a4e62303030303030303030303a636164656e63652e6d6f65405f6f6f79655f313030626563733a636164656e63652e6d6f65'), + ('sim_member','sim_member','11 1','4892 4895','841 4895',X'034b45217443744769524448303030303030303030303a636164656e63652e6d6f65405f6f6f79655f646f6f74736b7972653a636164656e63652e6d6f65'), + ('sim_member','sim_member','73 1','5072 5089','888 5089',X'034b47217665764462756174303030303030303030303a636164656e63652e6d6f65405f6f6f79655f5f706b5f6b73706a75653a636164656e63652e6d6f65'), + ('sim_member','sim_member','59 1','5182 5236','903 5236',X'034b43217750454472596b77303030303030303030303a636164656e63652e6d6f65405f6f6f79655f74656368323334613a636164656e63652e6d6f65'), + ('sim_member','sim_member','52 1','5441 5469','968 5469',X'034b41217a66654e574d744b303030303030303030303a636164656e63652e6d6f65405f6f6f79655f6e6f766574746f3a636164656e63652e6d6f65'), ('emoji','emoji','1','385','385',X'023331313035373039393137313237353737363733'), ('emoji','emoji','1','771','771',X'023331323230353735323436303531303533353638'), ('emoji','emoji','1','1157','1157',X'023331333530383339313335363836313033303730'), - ('emoji','emoji','1','1543','1543',X'0231333530373039313634383434323533313934'), - ('emoji','emoji','1','1929','1929',X'0231343934393031383434343036303432363337'), - ('emoji','emoji','1','2315','2315',X'0231363432383535303135363835323932303438'), - ('emoji','emoji','1','2701','2701',X'0231373738353731303433363339333934333134'), - ('emoji','emoji','1','3087','3087',X'0231393031313133373739343135333134343332'), + ('emoji','emoji','1','1543','1543',X'0231333439373232393637383636393938373834'), + ('emoji','emoji','1','1929','1929',X'0231343933383437383237313138353535313436'), + ('emoji','emoji','1','2315','2315',X'0231363432353731303038313337373536373032'), + ('emoji','emoji','1','2701','2701',X'0231373738313036343330333034393434313238'), + ('emoji','emoji','1','3087','3087',X'0231393030383733373535343037303336343738'), ('guild_space','guild_space','1','4','4',X'023331313333333135333632353636343535333336'), ('guild_space','guild_space','1','9','9',X'023331313534383638343234373234343633363837'), ('guild_space','guild_space','1','14','14',X'023331323139303338323637383430393235383138'), ('guild_space','guild_space','1','19','19',X'023331323839363030383537323437303535383733'), - ('guild_space','guild_space','1','24','24',X'023331333536353037353335313132323738303836'), - ('guild_space','guild_space','1','29','29',X'0231323937323732313833373136303532393933'), - ('guild_space','guild_space','1','34','34',X'0231363437393532363337373630363334383831'), - ('guild_space','guild_space','1','39','39',X'0231383737303635303431393930353136373637'), + ('guild_space','guild_space','1','24','24',X'023331333435363431323031393032323838393837'), + ('guild_space','guild_space','1','29','29',X'0231323733383737363437323234393935383431'), + ('guild_space','guild_space','1','34','34',X'0231353239313736313536333938363832313135'), + ('guild_space','guild_space','1','39','39',X'0231383730313138363530373638363730373530'), ('member_power','member_power','1 1','0 0','0 0',X'03350f40636164656e63653a636164656e63652e6d6f652a'), ('sim_proxy','sim_proxy','1','23','23',X'025531363733363165392d656137652d343530392d623533302d356531613863613735336237'), ('sim_proxy','sim_proxy','1','47','47',X'025532653561626332312d326332622d346133352d386237642d366432383162363036653932'), @@ -210,39 +205,38 @@ INSERT INTO "sqlite_stat4" ("tbl","idx","neq","nlt","ndlt","sample") VALUES ('re ('sim_proxy','sim_proxy','1','143','143',X'025561616630313539652d623165312d343231342d396266652d313334613536303738323231'), ('sim_proxy','sim_proxy','1','167','167',X'025563396534393633372d663061352d343566352d383234382d366436393565643861316434'), ('sim_proxy','sim_proxy','1','191','191',X'025565333734613634362d386231332d343365392d393635392d653233326366653866626265'), - ('member_cache','member_cache','1 1','122 122','82 122',X'034b2d214d5071594e414a62576b72474f544a7461703a636164656e63652e6d6f6540726e6c3a636164656e63652e6d6f65'), - ('member_cache','member_cache','5 1','126 130','85 130',X'034b43214e446249714e704a795076664b526e4e63723a636164656e63652e6d6f6540776f756e6465645f77617272696f723a6d61747269782e6f7267'), - ('member_cache','member_cache','4 1','136 137','90 137',X'034b3d214f485844457370624d485348716c4445614f3a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), - ('member_cache','member_cache','5 1','204 206','135 206',X'034b51215450616f6a5454444446444847776c7276743a636164656e63652e6d6f65406a61636b736f6e6368656e3636363a6a61636b736f6e6368656e3636362e636f6d'), - ('member_cache','member_cache','76 1','213 245','140 245',X'034b412154716c79516d69667847556767456d64424e3a636164656e63652e6d6f65406869726f616e7461676f6e6973743a6d61747269782e6f7267'), - ('member_cache','member_cache','4 1','301 303','150 303',X'034b352156624f77675559777146614e4c5345644e413a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), - ('member_cache','member_cache','4 1','338 340','170 340',X'034b3321586f4c466b65786a45466c57447959544d453a636164656e63652e6d6f65406d65636879613a636164656e63652e6d6f65'), - ('member_cache','member_cache','10 1','344 344','173 344',X'034b3d21594b46454e79716667696951686956496b533a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), - ('member_cache','member_cache','2 1','368 368','183 368',X'034b35215a51714f4757704f52676e544f7254474f583a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), - ('member_cache','member_cache','152 1','396 491','203 491',X'034b412163427874565278446c5a765356684a58564b3a636164656e63652e6d6f65406d61727368616d616c6c6f773a616c74686165612e7a6f6e65'), - ('member_cache','member_cache','4 1','553 555','207 555',X'034b33216356514d45455158494d5047554b43467a763a636164656e63652e6d6f65406d65636879613a636164656e63652e6d6f65'), - ('member_cache','member_cache','8 1','571 576','219 576',X'034b2b21654856655270706e6c6f57587177704a6e553a636164656e63652e6d6f6540656c6c69753a68617368692e7265'), - ('member_cache','member_cache','7 1','583 584','223 584',X'034b4b2165724f7079584e465a486a48724568784e583a636164656e63652e6d6f6540616d796973636f6f6c7a3a6d61747269782e6174697573616d792e636f6d'), - ('member_cache','member_cache','165 1','599 614','230 614',X'034b3321676865544b5a7451666c444e7070684c49673a636164656e63652e6d6f65406172636f6e79783a6d61747269782e6f7267'), - ('member_cache','member_cache','165 1','599 737','230 737',X'034b3921676865544b5a7451666c444e7070684c49673a636164656e63652e6d6f654073756368616372636b723a6d61747269782e6f7267'), - ('member_cache','member_cache','6 1','764 764','231 764',X'034b3d21676b6b686f756d42506643415055664554783a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), - ('member_cache','member_cache','10 1','772 775','234 775',X'034b412168424a766e654e4f63646b77694e464c67483a636164656e63652e6d6f65406265617264616e64726f7365733a636164656e63652e6d6f65'), - ('member_cache','member_cache','13 1','795 800','244 800',X'03513d2169537958674e7851634575586f587073536e3a707573737468656361742e6f72674063726f6e793a63726f6e79616b617473756b692e78797a'), - ('member_cache','member_cache','12 1','851 860','274 860',X'034b35216c7570486a715444537a774f744d59476d493a636164656e63652e6d6f654068656c6c63703a6f70656e737573652e6f7267'), - ('member_cache','member_cache','4 1','872 874','280 874',X'034b39216d616767455367755a427147425a74536e723a636164656e63652e6d6f65406875636b6c65746f6e3a636164656e63652e6d6f65'), - ('member_cache','member_cache','4 1','924 924','315 924',X'034b352172454f73706e5971644f414c4149466e69563a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), - ('member_cache','member_cache','4 1','966 966','342 966',X'034b3d21766e717a56767678534a586c5a504f5276533a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), - ('member_cache','member_cache','71 1','981 983','350 983',X'034b3d2177574f667376757356486f4e4e567242585a3a636164656e63652e6d6f6540616c65783a6d61747269782e7370656564626f792e6368'), - ('member_cache','member_cache','14 1','1056 1062','354 1062',X'034b3d21776c534544496a44676c486d42474b7254703a636164656e63652e6d6f65406b61746c796e3a69732d686172646c792e6f6e6c696e65'), - ('direct','direct','1','0','0',X'024d4070726f666573736f725f706f6f7469735f7068643a6d61747269782e6f7267'), - ('file','file','1','4034','4034',X'03821368747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3131323736303636393137383234313032342f313135303430343234303037393036393237362f636d64726e656f6e2d313632383534343535313536373635303831362d32303233303232325f3134353733322d766964312e6d7034'), - ('file','file','1','8069','8069',X'03814968747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3131393239333936393731353639313532322f313230393030323833373230383630343638322f636869702d77696c736f6e2e77656270'), - ('file','file','1','12104','12104',X'03813b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3133343037373735333438353033333437322f313139383038373532313531393830303436312f696d6167652e706e67'), - ('file','file','1','16139','16139',X'03814168747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f313337363732343737393830353034383838322f313339323632373139373133383730323336362f707265766965772e706e67'), - ('file','file','1','20174','20174',X'03817568747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3135393136353731343139343735393638302f313236383537313138333330363131373230302f53637265656e73686f745f32303234303830315f3130303834315f446973636f72642e6a7067'), - ('file','file','1','24209','24209',X'03816568747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3238383838323935333331343839333832352f313135343539393530343936303537333533322f53637265656e73686f745f32303233303631302d3130333631332e706e67'), - ('file','file','1','28244','28244',X'03813b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3635353231363137333639363238363734362f313330383839373634363434353938393930392f696d6167652e706e67'), - ('file','file','1','32279','32279',X'027f68747470733a2f2f63646e2e646973636f72646170702e636f6d2f656d6f6a69732f313034393232363234323836313332323238312e706e67'), + ('member_cache','member_cache','4 1','98 99','66 99',X'034b35214a48614a71425870303030303030303030303a636164656e63652e6d6f6540657a7261637574653a6d61747269782e6f7267'), + ('member_cache','member_cache','4 1','119 122','80 122',X'034b43214c684978654c4d54303030303030303030303a636164656e63652e6d6f6540737461727368696e656c756e6163793a6d61747269782e6f7267'), + ('member_cache','member_cache','1 1','124 124','82 124',X'034b2d214d5071594e414a62303030303030303030303a636164656e63652e6d6f6540726e6c3a636164656e63652e6d6f65'), + ('member_cache','member_cache','5 1','128 131','85 131',X'034b3b214e446249714e704a303030303030303030303a636164656e63652e6d6f65406761627269656c766f6e643a6d61747269782e6f7267'), + ('member_cache','member_cache','4 1','138 139','90 139',X'034b3d214f48584445737062303030303030303030303a636164656e63652e6d6f6540616d693a7468652d61706f746865636172792e636c7562'), + ('member_cache','member_cache','5 1','207 209','135 209',X'034b51215450616f6a545444303030303030303030303a636164656e63652e6d6f65406a61636b736f6e6368656e3636363a6a61636b736f6e6368656e3636362e636f6d'), + ('member_cache','member_cache','76 1','216 249','140 249',X'034b2d2154716c79516d6966303030303030303030303a636164656e63652e6d6f65406963656d616e3a656e76732e6e6574'), + ('member_cache','member_cache','4 1','345 345','171 345',X'034b3521586f4c466b65786a303030303030303030303a636164656e63652e6d6f6540636164656e63653a636164656e63652e6d6f65'), + ('member_cache','member_cache','10 1','351 354','174 354',X'034b3521594b46454e797166303030303030303030303a636164656e63652e6d6f654066617269656c6c653a6d61747269782e6f7267'), + ('member_cache','member_cache','1 1','374 374','183 374',X'034b2d21596f54644f55766a303030303030303030303a636164656e63652e6d6f6540726e6c3a636164656e63652e6d6f65'), + ('member_cache','member_cache','152 1','405 499','205 499',X'034b45216342787456527844303030303030303030303a636164656e63652e6d6f65406d61726975733835313030303a6d617269757364617669642e6672'), + ('member_cache','member_cache','4 1','562 563','209 563',X'034b35216356514d45455158303030303030303030303a636164656e63652e6d6f6540657a7261637574653a6d61747269782e6f7267'), + ('member_cache','member_cache','8 1','582 586','223 586',X'034b3721654856655270706e303030303030303030303a636164656e63652e6d6f65406563686f3a66757272797265667567652e636f6d'), + ('member_cache','member_cache','7 1','594 600','227 600',X'034b3b2165724f7079584e46303030303030303030303a636164656e63652e6d6f6540766962656973766572796f3a6d61747269782e6f7267'), + ('member_cache','member_cache','165 1','613 624','235 624',X'034b4921676865544b5a7451303030303030303030303a636164656e63652e6d6f6540616d70666c6f7765723a7468652d61706f746865636172792e636c7562'), + ('member_cache','member_cache','165 1','613 749','235 749',X'034b4521676865544b5a7451303030303030303030303a636164656e63652e6d6f654073706c617473756e653a636861742e6e6575726172696f2e636f6d'), + ('member_cache','member_cache','6 1','778 782','236 782',X'034b3321676b6b686f756d42303030303030303030303a636164656e63652e6d6f65406b6162693a6361746769726c2e776f726b73'), + ('member_cache','member_cache','10 1','786 794','239 794',X'034b332168424a766e654e4f303030303030303030303a636164656e63652e6d6f65406d65636879613a636164656e63652e6d6f65'), + ('member_cache','member_cache','13 1','809 819','249 819',X'034b2b2169537958674e7851303030303030303030303a636164656e63652e6d6f65406d69646f753a656e76732e6e6574'), + ('member_cache','member_cache','4 1','856 858','273 858',X'034b3b216b73724f45554666303030303030303030303a636164656e63652e6d6f65406761627269656c766f6e643a6d61747269782e6f7267'), + ('member_cache','member_cache','12 1','865 874','279 874',X'034b35216c7570486a715444303030303030303030303a636164656e63652e6d6f654068656c6c63703a6f70656e737573652e6f7267'), + ('member_cache','member_cache','4 1','886 887','285 887',X'034b2d216d61676745536775303030303030303030303a636164656e63652e6d6f65406361743a6d61756e69756d2e6e6574'), + ('member_cache','member_cache','71 1','999 999','357 999',X'034b2d2177574f6673767573303030303030303030303a636164656e63652e6d6f654061613a6361747669626572732e6d65'), + ('member_cache','member_cache','14 1','1074 1085','361 1085',X'034b3721776c534544496a44303030303030303030303a636164656e63652e6d6f654073646f6d693a6861636b657273706163652e706c'), + ('file','file','1','4054','4054',X'03816568747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3131323736303636393137383234313032342f313135313733303032323332383035373930372f53637265656e73686f745f32303233303931345f3036303333352e6a7067'), + ('file','file','1','8109','8109',X'03814168747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3131393239333936393731353639313532322f313231383430393538323539343838373638302f494d475f343738322e6a7067'), + ('file','file','1','12164','12164',X'03813b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3133343037373735333438353033333437322f313139393131323831343931373333333133332f696d6167652e706e67'), + ('file','file','1','16219','16219',X'03814168747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f313337363732343737393830353034383838322f313339323938363435323538343936303032312f707265766965772e706e67'), + ('file','file','1','20274','20274',X'03816568747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3135393136353731343139343735393638302f313236353735383536303531323338313030392f53637265656e73686f745f32303234303732342d3135353232382e706e67'), + ('file','file','1','24329','24329',X'03813b68747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3238383838323935333331343839333832352f313134373538383535363839343738313539332f696d6167652e706e67'), + ('file','file','1','28384','28384',X'03817168747470733a2f2f63646e2e646973636f72646170702e636f6d2f6174746163686d656e74732f3635353231363137333639363238363734362f313330383239363937343136333737353530392f31373331393932363837323838343136383737323137393036333831303733392e6a7067'), + ('file','file','1','32439','32439',X'027f68747470733a2f2f63646e2e646973636f72646170702e636f6d2f656d6f6a69732f313034323532383239323539363632313434322e706e67'), ('lottie','lottie','1','2','2',X'0231373439303532393434363832353832303336'), ('lottie','lottie','1','5','5',X'0231373531363036333739333430333635383634'), ('lottie','lottie','1','8','8',X'0231373534313038373731383532323232353634'), diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index a3fcf0dd..bdf6bf9d 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -64,6 +64,7 @@ export type Models = { historical_room_index: number reference_channel_id: string room_id: string + upgraded_timestamp: number } invite: { diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 78ab2b45..ab11c8b7 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -23,7 +23,7 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom ('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL), ('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL); -INSERT INTO historical_channel_room (reference_channel_id, room_id) SELECT channel_id, room_id FROM channel_room; +INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) SELECT channel_id, room_id, 0 FROM channel_room; INSERT INTO sim (user_id, username, sim_name, mxid) VALUES ('0', 'Matrix Bridge', 'bot', '@_ooye_bot:cadence.moe'), From c298f78f96e631d45cd84a268dce2db1cd69747b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Dec 2025 16:48:11 +1300 Subject: [PATCH 033/153] Update global profiles for sims --- src/d2m/actions/register-pk-user.js | 11 +---- src/d2m/actions/register-user.js | 59 +++++++++++++++++++++--- src/d2m/actions/register-webhook-user.js | 11 +---- src/matrix/api.js | 35 +++++++++++--- 4 files changed, 85 insertions(+), 31 deletions(-) diff --git a/src/d2m/actions/register-pk-user.js b/src/d2m/actions/register-pk-user.js index e73fa44c..3c914b6d 100644 --- a/src/d2m/actions/register-pk-user.js +++ b/src/d2m/actions/register-pk-user.js @@ -152,16 +152,9 @@ async function syncUser(messageID, author, roomID, shouldActuallySync) { const mxid = await ensureSimJoined(pkMessage, roomID) if (shouldActuallySync) { - // Build current profile data + // Build current profile data and sync if the hash has changed const content = await memberToStateContent(pkMessage, author) - const currentHash = registerUser._hashProfileContent(content, 0) - const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() - - // Only do the actual sync if the hash has changed since we last looked - if (existingHash !== currentHash) { - await api.sendState(roomID, "m.room.member", mxid, content, mxid) - db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) - } + await registerUser._sendSyncUser(roomID, mxid, content, null) } return mxid diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index 4bb8e982..85e8a08e 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -22,6 +22,8 @@ let hasher = null // @ts-ignore require("xxhash-wasm")().then(h => hasher = h) +const supportsMsc4069 = api.versions().then(v => !!v?.unstable_features?.["org.matrix.msc4069"]).catch(() => false) + /** * A sim is an account that is being simulated by the bridge to copy events from the other side. * @param {DiscordTypes.APIUser} user @@ -97,6 +99,23 @@ async function ensureSimJoined(user, roomID) { return mxid } +/** + * @param {DiscordTypes.APIUser} user + */ +async function userToGlobalProfile(user) { + const globalProfile = {} + + globalProfile.displayname = user.username + if (user.global_name) globalProfile.displayname = user.global_name + + if (user.avatar) { + const avatarPath = file.userAvatar(user) // the user avatar only + globalProfile.avatar_url = await file.uploadDiscordFileToMxc(avatarPath) + } + + return globalProfile +} + /** * @param {DiscordTypes.APIUser} user * @param {Omit | undefined} member @@ -200,21 +219,45 @@ async function syncUser(user, member, channel, guild, roomID) { const mxid = await ensureSimJoined(user, roomID) const content = await memberToStateContent(user, member, guild.id) const powerLevel = memberToPowerLevel(user, member, guild, channel) - const currentHash = _hashProfileContent(content, powerLevel) + await _sendSyncUser(roomID, mxid, content, powerLevel, { + // do not overwrite pre-existing data if we already have data and `member` is not accessible, because this would replace good data with bad data + allowOverwrite: !!member, + globalProfile: await userToGlobalProfile(user) + }) + return mxid +} + +/** + * @param {string} roomID + * @param {string} mxid + * @param {{displayname: string, avatar_url?: string}} content + * @param {number | null} powerLevel + * @param {{allowOverwrite?: boolean, globalProfile?: {displayname: string, avatar_url?: string}}} [options] + */ +async function _sendSyncUser(roomID, mxid, content, powerLevel, options) { + const currentHash = _hashProfileContent(content, powerLevel ?? 0) const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() // only do the actual sync if the hash has changed since we last looked const hashHasChanged = existingHash !== currentHash - // however, do not overwrite pre-existing data if we already have data and `member` is not accessible, because this would replace good data with bad data - const wouldOverwritePreExisting = existingHash && !member - if (hashHasChanged && !wouldOverwritePreExisting) { + // always okay to add new data. for overwriting, restrict based on options.allowOverwrite, if present + const overwriteOkay = !existingHash || (options?.allowOverwrite ?? true) + if (hashHasChanged && overwriteOkay) { + const actions = [] // Update room member state - await api.sendState(roomID, "m.room.member", mxid, content, mxid) + actions.push(api.sendState(roomID, "m.room.member", mxid, content, mxid)) // Update power levels - await api.setUserPower(roomID, mxid, powerLevel) + if (powerLevel != null) { + actions.push(api.setUserPower(roomID, mxid, powerLevel)) + } + // Update global profile (if supported by server) + if (await supportsMsc4069) { + actions.push(api.profileSetDisplayname(mxid, options?.globalProfile?.displayname || content.displayname, true)) + actions.push(api.profileSetAvatarUrl(mxid, options?.globalProfile?.avatar_url || content.avatar_url, true)) + } + await Promise.all(actions) // Update cached hash db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) } - return mxid } /** @@ -251,5 +294,7 @@ module.exports._hashProfileContent = _hashProfileContent module.exports.ensureSim = ensureSim module.exports.ensureSimJoined = ensureSimJoined module.exports.syncUser = syncUser +module.exports._sendSyncUser = _sendSyncUser module.exports.syncAllUsersInRoom = syncAllUsersInRoom module.exports._memberToPowerLevel = memberToPowerLevel +module.exports.supportsMsc4069 = supportsMsc4069 diff --git a/src/d2m/actions/register-webhook-user.js b/src/d2m/actions/register-webhook-user.js index 869d7d85..309a120b 100644 --- a/src/d2m/actions/register-webhook-user.js +++ b/src/d2m/actions/register-webhook-user.js @@ -128,16 +128,9 @@ async function syncUser(author, roomID, shouldActuallySync) { const mxid = await ensureSimJoined(fakeUserID, author, roomID) if (shouldActuallySync) { - // Build current profile data + // Build current profile data and sync if the hash has changed const content = await authorToStateContent(author) - const currentHash = registerUser._hashProfileContent(content, 0) - const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() - - // Only do the actual sync if the hash has changed since we last looked - if (existingHash !== currentHash) { - await api.sendState(roomID, "m.room.member", mxid, content, mxid) - db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) - } + await registerUser._sendSyncUser(roomID, mxid, content, null) } return mxid diff --git a/src/matrix/api.js b/src/matrix/api.js index d0892ff8..d746b252 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -317,16 +317,34 @@ async function sendTyping(roomID, isTyping, mxid, duration) { }) } -async function profileSetDisplayname(mxid, displayname) { - await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/displayname`, mxid), { +/** + * @param {string} mxid + * @param {string} displayname + * @param {boolean} [inhibitPropagate] + */ +async function profileSetDisplayname(mxid, displayname, inhibitPropagate) { + const params = {} + if (inhibitPropagate) params["org.matrix.msc4069.propagate"] = false + await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/displayname`, mxid, params), { displayname }) } -async function profileSetAvatarUrl(mxid, avatar_url) { - await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/avatar_url`, mxid), { - avatar_url - }) +/** + * @param {string} mxid + * @param {string} avatar_url + * @param {boolean} [inhibitPropagate] + */ +async function profileSetAvatarUrl(mxid, avatar_url, inhibitPropagate) { + const params = {} + if (inhibitPropagate) params["org.matrix.msc4069.propagate"] = false + if (avatar_url) { + await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/avatar_url`, mxid, params), { + avatar_url + }) + } else { + await mreq.mreq("DELETE", path(`/client/v3/profile/${mxid}/avatar_url`, mxid, params)) + } } /** @@ -490,6 +508,10 @@ function getProfile(mxid) { return mreq.mreq("GET", `/client/v3/profile/${mxid}`) } +function versions() { + return mreq.mreq("GET", "/client/versions") +} + module.exports.path = path module.exports.register = register module.exports.createRoom = createRoom @@ -526,3 +548,4 @@ module.exports.getAccountData = getAccountData module.exports.setAccountData = setAccountData module.exports.setPresence = setPresence module.exports.getProfile = getProfile +module.exports.versions = versions From 1758b7aa2286699ee09b59ec389ab0b8a1e43071 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 26 Nov 2025 17:21:15 +1300 Subject: [PATCH 034/153] m->d: make image-replies work --- src/m2d/converters/event-to-message.js | 294 ++++++++++---------- src/m2d/converters/event-to-message.test.js | 93 +++++++ 2 files changed, 241 insertions(+), 146 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index fd9289dd..eca30080 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -605,7 +605,7 @@ async function eventToMessage(event, guild, di) { } attachments.push({id: "0", filename}) pendingFiles.push({name: filename, mxc: event.content.url}) - } else if (shouldProcessTextEvent) { + } else { // Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events. // this event ---is an edit of--> original event ---is a reply to--> past event await (async () => { @@ -738,157 +738,159 @@ async function eventToMessage(event, guild, di) { replyLine = `-# > ${replyLine}${contentPreview}\n` })() - if (event.content.format === "org.matrix.custom.html" && event.content.formatted_body) { - let input = event.content.formatted_body - if (event.content.msgtype === "m.emote") { - input = `* ${displayName} ${input}` - } - - // Handling mentions of Discord users - input = input.replace(/("https:\/\/matrix.to\/#\/((?:@|%40)[^"]+)")>/g, (whole, attributeValue, mxid) => { - mxid = decodeURIComponent(mxid) - if (mxUtils.eventSenderIsFromDiscord(mxid)) { - // Handle mention of an OOYE sim user by their mxid - const id = select("sim", "user_id", {mxid}).pluck().get() - if (!id) return whole - return `${attributeValue} data-user-id="${id}">` - } else { - // Handle mention of a Matrix user by their mxid - // Check if this Matrix user is actually the sim user from another old bridge in the room? - const match = mxid.match(/[^:]*discord[^:]*_([0-9]{6,}):/) // try to match @_discord_123456, @_discordpuppet_123456, etc. - if (match) return `${attributeValue} data-user-id="${match[1]}">` - // Nope, just a real Matrix user. - return whole + if (shouldProcessTextEvent) { + if (event.content.format === "org.matrix.custom.html" && event.content.formatted_body) { + let input = event.content.formatted_body + if (event.content.msgtype === "m.emote") { + input = `* ${displayName} ${input}` } - }) - // Handling mentions of rooms and room-messages - input = await handleRoomOrMessageLinks(input, di) - - // Stripping colons after mentions - input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1") - input = input.replace(/("https:\/\/matrix.to.*?<\/a>):?/g, "$1") - - // Element adds a bunch of
before but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those. - input = input.replace(/(?:\n|
\s*)*<\/blockquote>/g, "") - - // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. - // But I should not count it if it's between block elements. - input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => { - // console.error(beforeContext, beforeTag, afterContext, afterTag) - if (typeof beforeTag !== "string" && typeof afterTag !== "string") { - return "
" - } - beforeContext = beforeContext || "" - beforeTag = beforeTag || "" - afterContext = afterContext || "" - afterTag = afterTag || "" - if (!mxUtils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !mxUtils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) { - return beforeContext + "
" + afterContext - } else { - return whole - } - }) - - // Note: Element's renderers on Web and Android currently collapse whitespace, like the browser does. Turndown also collapses whitespace which is good for me. - // If later I'm using a client that doesn't collapse whitespace and I want turndown to follow suit, uncomment the following line of code, and it Just Works: - // input = input.replace(/ /g, " ") - // There is also a corresponding test to uncomment, named "event2message: whitespace is retained" - - // Handling written @mentions: we need to look for candidate Discord members to join to the room - // This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here. - // We're using the domino parser because Turndown uses the same and can reuse this tree. - const doc = domino.createDocument( - // DOM parsers arrange elements in the and . Wrapping in a custom element ensures elements are reliably arranged in a single element. - '' + input + '' - ); - const root = doc.getElementById("turndown-root"); - async function forEachNode(node) { - for (; node; node = node.nextSibling) { - // Check written mentions - if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) { - const result = await checkWrittenMentions(node.nodeValue, event.sender, event.room_id, guild, di) - if (result) { - node.nodeValue = result.content - ensureJoined.push(...result.ensureJoined) - allowedMentionsParse.push(...result.allowedMentionsParse) - } + // Handling mentions of Discord users + input = input.replace(/("https:\/\/matrix.to\/#\/((?:@|%40)[^"]+)")>/g, (whole, attributeValue, mxid) => { + mxid = decodeURIComponent(mxid) + if (mxUtils.eventSenderIsFromDiscord(mxid)) { + // Handle mention of an OOYE sim user by their mxid + const id = select("sim", "user_id", {mxid}).pluck().get() + if (!id) return whole + return `${attributeValue} data-user-id="${id}">` + } else { + // Handle mention of a Matrix user by their mxid + // Check if this Matrix user is actually the sim user from another old bridge in the room? + const match = mxid.match(/[^:]*discord[^:]*_([0-9]{6,}):/) // try to match @_discord_123456, @_discordpuppet_123456, etc. + if (match) return `${attributeValue} data-user-id="${match[1]}">` + // Nope, just a real Matrix user. + return whole } - // Check for incompatible backticks in code blocks - let preNode - if (node.nodeType === 3 && node.nodeValue.includes("```") && (preNode = nodeIsChildOf(node, ["PRE"]))) { - if (preNode.firstChild?.nodeName === "CODE") { - const ext = preNode.firstChild.className.match(/language-(\S+)/)?.[1] || "txt" - const filename = `inline_code.${ext}` - // Build the replacement node - const replacementCode = doc.createElement("code") - replacementCode.textContent = `[${filename}]` - // Build its containing node - const replacement = doc.createElement("span") - replacement.appendChild(doc.createTextNode(" ")) - replacement.appendChild(replacementCode) - replacement.appendChild(doc.createTextNode(" ")) - // Replace the code block with the - preNode.replaceWith(replacement) - // Upload the code as an attachment - const content = getCodeContent(preNode.firstChild) - attachments.push({id: String(attachments.length), filename}) - pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")}) - } + }) + + // Handling mentions of rooms and room-messages + input = await handleRoomOrMessageLinks(input, di) + + // Stripping colons after mentions + input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1") + input = input.replace(/("https:\/\/matrix.to.*?<\/a>):?/g, "$1") + + // Element adds a bunch of
before but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those. + input = input.replace(/(?:\n|
\s*)*<\/blockquote>/g, "") + + // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. + // But I should not count it if it's between block elements. + input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => { + // console.error(beforeContext, beforeTag, afterContext, afterTag) + if (typeof beforeTag !== "string" && typeof afterTag !== "string") { + return "
" + } + beforeContext = beforeContext || "" + beforeTag = beforeTag || "" + afterContext = afterContext || "" + afterTag = afterTag || "" + if (!mxUtils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !mxUtils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) { + return beforeContext + "
" + afterContext + } else { + return whole + } + }) + + // Note: Element's renderers on Web and Android currently collapse whitespace, like the browser does. Turndown also collapses whitespace which is good for me. + // If later I'm using a client that doesn't collapse whitespace and I want turndown to follow suit, uncomment the following line of code, and it Just Works: + // input = input.replace(/ /g, " ") + // There is also a corresponding test to uncomment, named "event2message: whitespace is retained" + + // Handling written @mentions: we need to look for candidate Discord members to join to the room + // This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here. + // We're using the domino parser because Turndown uses the same and can reuse this tree. + const doc = domino.createDocument( + // DOM parsers arrange elements in the and . Wrapping in a custom element ensures elements are reliably arranged in a single element. + '' + input + '' + ); + const root = doc.getElementById("turndown-root"); + async function forEachNode(node) { + for (; node; node = node.nextSibling) { + // Check written mentions + if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) { + const result = await checkWrittenMentions(node.nodeValue, event.sender, event.room_id, guild, di) + if (result) { + node.nodeValue = result.content + ensureJoined.push(...result.ensureJoined) + allowedMentionsParse.push(...result.allowedMentionsParse) + } + } + // Check for incompatible backticks in code blocks + let preNode + if (node.nodeType === 3 && node.nodeValue.includes("```") && (preNode = nodeIsChildOf(node, ["PRE"]))) { + if (preNode.firstChild?.nodeName === "CODE") { + const ext = preNode.firstChild.className.match(/language-(\S+)/)?.[1] || "txt" + const filename = `inline_code.${ext}` + // Build the replacement node + const replacementCode = doc.createElement("code") + replacementCode.textContent = `[${filename}]` + // Build its containing node + const replacement = doc.createElement("span") + replacement.appendChild(doc.createTextNode(" ")) + replacement.appendChild(replacementCode) + replacement.appendChild(doc.createTextNode(" ")) + // Replace the code block with the + preNode.replaceWith(replacement) + // Upload the code as an attachment + const content = getCodeContent(preNode.firstChild) + attachments.push({id: String(attachments.length), filename}) + pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")}) + } + } + await forEachNode(node.firstChild) } - await forEachNode(node.firstChild) } + await forEachNode(root) + + // SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet. + // First we need to determine which emojis are at the end. + endOfMessageEmojis = [] + let match + let last = input.length + while ((match = input.slice(0, last).match(/]*>\s*$/))) { + if (!match[0].includes("data-mx-emoticon")) break + const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/) + if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1]) + assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec") + last = match.index + } + + // @ts-ignore bad type from turndown + content = turndownService.turndown(root) + + // Put < > around any surviving matrix.to links to hide the URL previews + content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/g, "<$&>") + + // It's designed for commonmark, we need to replace the space-space-newline with just newline + content = content.replace(/ \n/g, "\n") + + // If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated + if (replyLine && content.startsWith("> ")) content = "\n" + content + + // SPRITE SHEET EMOJIS FEATURE: + content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader) + } else { + // Looks like we're using the plaintext body! + content = event.content.body + + if (event.content.msgtype === "m.emote") { + content = `* ${displayName} ${content}` + } + + content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible + content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews + + const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di) + if (result) { + content = result.content + ensureJoined.push(...result.ensureJoined) + allowedMentionsParse.push(...result.allowedMentionsParse) + } + + // Markdown needs to be escaped, though take care not to escape the middle of links + // @ts-ignore bad type from turndown + content = turndownService.escape(content) } - await forEachNode(root) - - // SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet. - // First we need to determine which emojis are at the end. - endOfMessageEmojis = [] - let match - let last = input.length - while ((match = input.slice(0, last).match(/]*>\s*$/))) { - if (!match[0].includes("data-mx-emoticon")) break - const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/) - if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1]) - assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec") - last = match.index - } - - // @ts-ignore bad type from turndown - content = turndownService.turndown(root) - - // Put < > around any surviving matrix.to links to hide the URL previews - content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/g, "<$&>") - - // It's designed for commonmark, we need to replace the space-space-newline with just newline - content = content.replace(/ \n/g, "\n") - - // If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated - if (replyLine && content.startsWith("> ")) content = "\n" + content - - // SPRITE SHEET EMOJIS FEATURE: - content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader) - } else { - // Looks like we're using the plaintext body! - content = event.content.body - - if (event.content.msgtype === "m.emote") { - content = `* ${displayName} ${content}` - } - - content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible - content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews - - const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di) - if (result) { - content = result.content - ensureJoined.push(...result.ensureJoined) - allowedMentionsParse.push(...result.allowedMentionsParse) - } - - // Markdown needs to be escaped, though take care not to escape the middle of links - // @ts-ignore bad type from turndown - content = turndownService.escape(content) } } diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 73ca4e96..2e347f54 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -2671,6 +2671,99 @@ test("event2message: rich reply to a state event with no body", async t => { ) }) +test("event2message: rich reply with an image", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + body: "image.png", + info: { + size: 470379, + mimetype: "image/png", + thumbnail_info: { + w: 800, + h: 450, + mimetype: "image/png", + size: 183014 + }, + w: 1920, + h: 1080, + "xyz.amorgan.blurhash": "L24_wtVt00xuxvR%NFX74Toz?waL", + thumbnail_url: "mxc://cadence.moe/lPtnjlleowWCXGOHKVDyoXGn" + }, + msgtype: "m.image", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4" + } + }, + url: "mxc://cadence.moe/yxMobQMbSqNHpajxgSHtaooG" + }, + origin_server_ts: 1764127662631, + unsigned: { + membership: "join", + age: 97, + transaction_id: "m1764127662540.2" + }, + event_id: "$QOxkw7u8vjTrrdKxEUO13JWSixV7UXAZU1freT1SkHc", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }, data.guild.general, { + api: { + getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + t.equal(eventID, "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4") + return { + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "you have to check every diff above insane on this set https://osu.ppy.sh/beatmapsets/2263303#osu/4826296" + }, + origin_server_ts: 1763639396419, + unsigned: { + membership: "join", + age: 486586696, + transaction_id: "m1763639396324.578" + }, + event_id: "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + } + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [ + { + content: "-# > <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/112760669178241024/1128118177155526666 **Ⓜcadence [they]**: you have to check every diff above insane on this...", + allowed_mentions: { + parse: ["users", "roles"] + }, + attachments: [ + { + filename: "image.png", + id: "0", + }, + ], + avatar_url: undefined, + pendingFiles: [ + { + mxc: "mxc://cadence.moe/yxMobQMbSqNHpajxgSHtaooG", + name: "image.png", + }, + ], + username: "cadence [they]", + }, + ] + } + ) +}) + test("event2message: raw mentioning discord users in plaintext body works", async t => { t.deepEqual( await eventToMessage({ From f176b547ce2d0ec92e4fbba80fe8f2399d5d836a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 27 Nov 2025 21:48:49 +1300 Subject: [PATCH 035/153] Maybe accept invites more reliably --- src/d2m/actions/create-room.js | 1 + src/m2d/event-dispatcher.js | 16 +++++-- src/matrix/api.js | 19 ++++++++ src/types.d.ts | 81 ++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 3 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 61e79f30..87c3701d 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -568,6 +568,7 @@ module.exports.createAllForGuild = createAllForGuild module.exports.channelToKState = channelToKState module.exports.postApplyPowerLevels = postApplyPowerLevels module.exports._convertNameAndTopic = convertNameAndTopic +module.exports._syncSpaceMember = _syncSpaceMember module.exports.unbridgeChannel = unbridgeChannel module.exports.unbridgeDeletedChannel = unbridgeDeletedChannel module.exports.existsOrAutocreatable = existsOrAutocreatable diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index ce3638c1..1f816db3 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -322,14 +322,25 @@ sync.addTemporaryListener(as, "type:m.room.member", guard("m.room.member", */ async event => { if (event.state_key[0] !== "@") return + const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` - if (event.content.membership === "invite" && event.state_key === `@${reg.sender_localpart}:${reg.ooye.server_name}`) { + if (event.content.membership === "invite" && event.state_key === bot) { // We were invited to a room. We should join, and register the invite details for future reference in web. + let attemptedApiMessage = "According to unsigned invite data." + let inviteRoomState = event.unsigned?.invite_room_state + if (!Array.isArray(inviteRoomState) || inviteRoomState.length === 0) { + try { + inviteRoomState = await api.getInviteState(event.room_id) + attemptedApiMessage = "According to SSS API." + } catch (e) { + attemptedApiMessage = "According to unsigned invite data. SSS API unavailable: " + e.toString() + } + } const name = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.name", "name") const topic = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.topic", "topic") const avatar = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.avatar", "url") const creationType = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.create", "type") - if (!name) return await api.leaveRoomWithReason(event.room_id, "Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite!") + if (!name) return await api.leaveRoomWithReason(event.room_id, `Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite! (${attemptedApiMessage})`) await api.joinRoom(event.room_id) db.prepare("INSERT OR IGNORE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)").run(event.sender, event.room_id, creationType, name, topic, avatar) if (avatar) utils.getPublicUrlForMxc(avatar) // make sure it's available in the media_proxy allowed URLs @@ -342,7 +353,6 @@ async event => { db.prepare("DELETE FROM member_cache WHERE room_id = ? and mxid = ?").run(event.room_id, event.state_key) // Unregister room's use as a direct chat if the bot itself left - const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` if (event.state_key === bot) { db.prepare("DELETE FROM direct WHERE room_id = ?").run(event.room_id) } diff --git a/src/matrix/api.js b/src/matrix/api.js index e529d0f8..d0892ff8 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -137,6 +137,24 @@ function getStateEvent(roomID, type, key) { return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state/${type}/${key}`) } +/** + * @param {string} roomID + * @returns {Promise} + */ +async function getInviteState(roomID) { + /** @type {Ty.R.SSS} */ + const root = await mreq.mreq("POST", "/client/unstable/org.matrix.simplified_msc3575/sync", { + room_subscriptions: { + [roomID]: { + timeline_limit: 0, + required_state: [] + } + } + }) + const roomResponse = root.rooms[roomID] + return "stripped_state" in roomResponse ? roomResponse.stripped_state : roomResponse.invite_state +} + /** * "Any of the AS's users must be in the room. This API is primarily for Application Services and should be faster to respond than /members as it can be implemented more efficiently on the server." * @param {string} roomID @@ -483,6 +501,7 @@ module.exports.getEvent = getEvent module.exports.getEventForTimestamp = getEventForTimestamp module.exports.getAllState = getAllState module.exports.getStateEvent = getStateEvent +module.exports.getInviteState = getInviteState module.exports.getJoinedMembers = getJoinedMembers module.exports.getMembers = getMembers module.exports.getHierarchy = getHierarchy diff --git a/src/types.d.ts b/src/types.d.ts index cafd9bea..f9488b97 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -166,6 +166,37 @@ export namespace Event { content: any } + export type InviteStrippedState = { + type: string + state_key: string + sender: string + content: Event.M_Room_Create | Event.M_Room_Name | Event.M_Room_Avatar | Event.M_Room_Topic | Event.M_Room_JoinRules | Event.M_Room_CanonicalAlias + } + + export type M_Room_Create = { + additional_creators: string[] + "m.federate"?: boolean + room_version: string + type?: string + predecessor?: { + room_id: string + event_id?: string + } + } + + export type M_Room_JoinRules = { + join_rule: "public" | "knock" | "invite" | "private" | "restricted" | "knock_restricted" + allow?: { + type: string + room_id: string + }[] + } + + export type M_Room_CanonicalAlias = { + alias?: string + alt_aliases?: string[] + } + export type M_Room_Message = { msgtype: "m.text" | "m.emote" body: string @@ -375,8 +406,58 @@ export namespace R { room_id: string servers: string[] } + + export type SSS = { + pos: string + lists: { + [list_key: string]: { + count: number + } + } + rooms: { + [room_id: string]: { + bump_stamp: number + /** Omitted if user not in room (peeking) */ + membership?: Membership + /** Names of lists that match this room */ + lists: string[] + } + // If user has been in the room - at least, that's what the spec says. Synapse returns some of these, such as `name` and `avatar`, for invites as well. Go nuts. + & { + name?: string + avatar?: string + heroes?: any[] + /** According to account data */ + is_dm?: boolean + /** If false, omitted fields are unchanged from their previous value. If true, omitted fields means the fields are not set. */ + initial?: boolean + expanded_timeline?: boolean + required_state?: Event.StateOuter[] + timeline_events?: Event.Outer[] + prev_batch?: string + limited?: boolean + num_live?: number + joined_count?: number + invited_count?: number + notification_count?: number + highlight_count?: number + } + // If user is invited or knocked + & ({ + /** @deprecated */ + invite_state: Event.InviteStrippedState[] + } | { + stripped_state: Event.InviteStrippedState[] + }) + } + extensions: { + [extension_key: string]: any + } + } } +export type Membership = "invite" | "knock" | "join" | "leave" | "ban" + export type Pagination = { chunk: T[] next_batch?: string From 493bc2560222f64d0359dc54771b5c8dafb072c7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 28 Nov 2025 17:20:44 +1300 Subject: [PATCH 036/153] Fix unbridging procedure --- src/d2m/actions/create-room.js | 4 ++-- src/m2d/event-dispatcher.js | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 87c3701d..4e59a17a 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -434,7 +434,7 @@ async function unbridgeChannel(channelID) { async function unbridgeDeletedChannel(channel, guildID) { const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get() assert.ok(roomID) - const row = from("guild_space").join("guild_active", "guild_id").select("space_id", "autocreate").get() + const row = from("guild_space").join("guild_active", "guild_id").select("space_id", "autocreate").where({guild_id: guildID}).get() assert.ok(row) let botInRoom = true @@ -458,7 +458,7 @@ async function unbridgeDeletedChannel(channel, guildID) { // delete webhook on discord const webhook = select("webhook", ["webhook_id", "webhook_token"], {channel_id: channel.id}).get() if (webhook) { - await discord.snow.webhook.deleteWebhook(webhook.webhook_id, webhook.webhook_token) + await discord.snow.webhook.deleteWebhook(webhook.webhook_id, webhook.webhook_token).catch(() => {}) db.prepare("DELETE FROM webhook WHERE channel_id = ?").run(channel.id) } diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 1f816db3..c102d018 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -7,6 +7,7 @@ const util = require("util") const Ty = require("../types") const {discord, db, sync, as, select} = require("../passthrough") +const {tag} = require("@cloudrac3r/html-template-tag") /** @type {import("./actions/send-event")} */ const sendEvent = sync.require("./actions/send-event") @@ -121,10 +122,10 @@ async function sendError(roomID, source, type, e, payload) { // Where const stack = stringifyErrorStack(e) - builder.addLine(`Error trace:\n${stack}`, `
Error trace
${stack}
`) + builder.addLine(`Error trace:\n${stack}`, tag`
Error trace
${stack}
`) // How - builder.addLine("", `
Original payload
${util.inspect(payload, false, 4, false)}
`) + builder.addLine("", tag`
Original payload
${util.inspect(payload, false, 4, false)}
`) } // Send From c7313035a45396a9b0e9fdb1a73c9c3237714ab7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Dec 2025 16:48:11 +1300 Subject: [PATCH 037/153] Update global profiles for sims --- src/d2m/actions/register-pk-user.js | 11 +---- src/d2m/actions/register-user.js | 59 +++++++++++++++++++++--- src/d2m/actions/register-webhook-user.js | 11 +---- src/matrix/api.js | 35 +++++++++++--- 4 files changed, 85 insertions(+), 31 deletions(-) diff --git a/src/d2m/actions/register-pk-user.js b/src/d2m/actions/register-pk-user.js index e17f0613..bf573e73 100644 --- a/src/d2m/actions/register-pk-user.js +++ b/src/d2m/actions/register-pk-user.js @@ -151,16 +151,9 @@ async function syncUser(messageID, author, roomID, shouldActuallySync) { const mxid = await ensureSimJoined(pkMessage, roomID) if (shouldActuallySync) { - // Build current profile data + // Build current profile data and sync if the hash has changed const content = await memberToStateContent(pkMessage, author) - const currentHash = registerUser._hashProfileContent(content, 0) - const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() - - // Only do the actual sync if the hash has changed since we last looked - if (existingHash !== currentHash) { - await api.sendState(roomID, "m.room.member", mxid, content, mxid) - db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) - } + await registerUser._sendSyncUser(roomID, mxid, content, null) } return mxid diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index 674853a2..b58ad66a 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -23,6 +23,8 @@ let hasher = null // @ts-ignore require("xxhash-wasm")().then(h => hasher = h) +const supportsMsc4069 = api.versions().then(v => !!v?.unstable_features?.["org.matrix.msc4069"]).catch(() => false) + /** * A sim is an account that is being simulated by the bridge to copy events from the other side. * @param {DiscordTypes.APIUser} user @@ -98,6 +100,23 @@ async function ensureSimJoined(user, roomID) { return mxid } +/** + * @param {DiscordTypes.APIUser} user + */ +async function userToGlobalProfile(user) { + const globalProfile = {} + + globalProfile.displayname = user.username + if (user.global_name) globalProfile.displayname = user.global_name + + if (user.avatar) { + const avatarPath = file.userAvatar(user) // the user avatar only + globalProfile.avatar_url = await file.uploadDiscordFileToMxc(avatarPath) + } + + return globalProfile +} + /** * @param {DiscordTypes.APIUser} user * @param {Omit | undefined} member @@ -201,21 +220,45 @@ async function syncUser(user, member, channel, guild, roomID) { const mxid = await ensureSimJoined(user, roomID) const content = await memberToStateContent(user, member, guild.id) const powerLevel = memberToPowerLevel(user, member, guild, channel) - const currentHash = _hashProfileContent(content, powerLevel) + await _sendSyncUser(roomID, mxid, content, powerLevel, { + // do not overwrite pre-existing data if we already have data and `member` is not accessible, because this would replace good data with bad data + allowOverwrite: !!member, + globalProfile: await userToGlobalProfile(user) + }) + return mxid +} + +/** + * @param {string} roomID + * @param {string} mxid + * @param {{displayname: string, avatar_url?: string}} content + * @param {number | null} powerLevel + * @param {{allowOverwrite?: boolean, globalProfile?: {displayname: string, avatar_url?: string}}} [options] + */ +async function _sendSyncUser(roomID, mxid, content, powerLevel, options) { + const currentHash = _hashProfileContent(content, powerLevel ?? 0) const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() // only do the actual sync if the hash has changed since we last looked const hashHasChanged = existingHash !== currentHash - // however, do not overwrite pre-existing data if we already have data and `member` is not accessible, because this would replace good data with bad data - const wouldOverwritePreExisting = existingHash && !member - if (hashHasChanged && !wouldOverwritePreExisting) { + // always okay to add new data. for overwriting, restrict based on options.allowOverwrite, if present + const overwriteOkay = !existingHash || (options?.allowOverwrite ?? true) + if (hashHasChanged && overwriteOkay) { + const actions = [] // Update room member state - await api.sendState(roomID, "m.room.member", mxid, content, mxid) + actions.push(api.sendState(roomID, "m.room.member", mxid, content, mxid)) // Update power levels - await api.setUserPower(roomID, mxid, powerLevel) + if (powerLevel != null) { + actions.push(api.setUserPower(roomID, mxid, powerLevel)) + } + // Update global profile (if supported by server) + if (await supportsMsc4069) { + actions.push(api.profileSetDisplayname(mxid, options?.globalProfile?.displayname || content.displayname, true)) + actions.push(api.profileSetAvatarUrl(mxid, options?.globalProfile?.avatar_url || content.avatar_url, true)) + } + await Promise.all(actions) // Update cached hash db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) } - return mxid } /** @@ -254,5 +297,7 @@ module.exports._hashProfileContent = _hashProfileContent module.exports.ensureSim = ensureSim module.exports.ensureSimJoined = ensureSimJoined module.exports.syncUser = syncUser +module.exports._sendSyncUser = _sendSyncUser module.exports.syncAllUsersInRoom = syncAllUsersInRoom module.exports._memberToPowerLevel = memberToPowerLevel +module.exports.supportsMsc4069 = supportsMsc4069 diff --git a/src/d2m/actions/register-webhook-user.js b/src/d2m/actions/register-webhook-user.js index 869d7d85..309a120b 100644 --- a/src/d2m/actions/register-webhook-user.js +++ b/src/d2m/actions/register-webhook-user.js @@ -128,16 +128,9 @@ async function syncUser(author, roomID, shouldActuallySync) { const mxid = await ensureSimJoined(fakeUserID, author, roomID) if (shouldActuallySync) { - // Build current profile data + // Build current profile data and sync if the hash has changed const content = await authorToStateContent(author) - const currentHash = registerUser._hashProfileContent(content, 0) - const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() - - // Only do the actual sync if the hash has changed since we last looked - if (existingHash !== currentHash) { - await api.sendState(roomID, "m.room.member", mxid, content, mxid) - db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) - } + await registerUser._sendSyncUser(roomID, mxid, content, null) } return mxid diff --git a/src/matrix/api.js b/src/matrix/api.js index d0892ff8..d746b252 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -317,16 +317,34 @@ async function sendTyping(roomID, isTyping, mxid, duration) { }) } -async function profileSetDisplayname(mxid, displayname) { - await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/displayname`, mxid), { +/** + * @param {string} mxid + * @param {string} displayname + * @param {boolean} [inhibitPropagate] + */ +async function profileSetDisplayname(mxid, displayname, inhibitPropagate) { + const params = {} + if (inhibitPropagate) params["org.matrix.msc4069.propagate"] = false + await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/displayname`, mxid, params), { displayname }) } -async function profileSetAvatarUrl(mxid, avatar_url) { - await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/avatar_url`, mxid), { - avatar_url - }) +/** + * @param {string} mxid + * @param {string} avatar_url + * @param {boolean} [inhibitPropagate] + */ +async function profileSetAvatarUrl(mxid, avatar_url, inhibitPropagate) { + const params = {} + if (inhibitPropagate) params["org.matrix.msc4069.propagate"] = false + if (avatar_url) { + await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/avatar_url`, mxid, params), { + avatar_url + }) + } else { + await mreq.mreq("DELETE", path(`/client/v3/profile/${mxid}/avatar_url`, mxid, params)) + } } /** @@ -490,6 +508,10 @@ function getProfile(mxid) { return mreq.mreq("GET", `/client/v3/profile/${mxid}`) } +function versions() { + return mreq.mreq("GET", "/client/versions") +} + module.exports.path = path module.exports.register = register module.exports.createRoom = createRoom @@ -526,3 +548,4 @@ module.exports.getAccountData = getAccountData module.exports.setAccountData = setAccountData module.exports.setPresence = setPresence module.exports.getProfile = getProfile +module.exports.versions = versions From 0bb7a27164ef6e1333465dae44a33fbea6202ed5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 3 Dec 2025 00:55:20 +1300 Subject: [PATCH 038/153] Semaphore retries per room --- src/m2d/event-dispatcher.js | 43 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index c102d018..985036e5 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -8,6 +8,7 @@ const util = require("util") const Ty = require("../types") const {discord, db, sync, as, select} = require("../passthrough") const {tag} = require("@cloudrac3r/html-template-tag") +const {Semaphore} = require("@chriscdn/promise-semaphore") /** @type {import("./actions/send-event")} */ const sendEvent = sync.require("./actions/send-event") @@ -153,34 +154,38 @@ function guard(type, fn) { } } +const errorRetrySema = new Semaphore() + /** * @param {Ty.Event.Outer} reactionEvent */ async function onRetryReactionAdd(reactionEvent) { const roomID = reactionEvent.room_id - const event = await api.getEvent(roomID, reactionEvent.content["m.relates_to"]?.event_id) + errorRetrySema.request(async () => { + const event = await api.getEvent(roomID, reactionEvent.content["m.relates_to"]?.event_id) - // Check that it's a real error from OOYE - const error = event.content["moe.cadence.ooye.error"] - if (event.sender !== `@${reg.sender_localpart}:${reg.ooye.server_name}` || !error) return + // Check that it's a real error from OOYE + const error = event.content["moe.cadence.ooye.error"] + if (event.sender !== `@${reg.sender_localpart}:${reg.ooye.server_name}` || !error) return - // To stop people injecting misleading messages, the reaction needs to come from either the original sender or a room moderator - if (reactionEvent.sender !== event.sender) { - // Check if it's a room moderator - const powerLevelsStateContent = await api.getStateEvent(roomID, "m.room.power_levels", "") - const powerLevel = powerLevelsStateContent.users?.[reactionEvent.sender] || 0 - if (powerLevel < 50) return - } + // To stop people injecting misleading messages, the reaction needs to come from either the original sender or a room moderator + if (reactionEvent.sender !== event.sender) { + // Check if it's a room moderator + const powerLevelsStateContent = await api.getStateEvent(roomID, "m.room.power_levels", "") + const powerLevel = powerLevelsStateContent.users?.[reactionEvent.sender] || 0 + if (powerLevel < 50) return + } - // Retry - if (error.source === "matrix") { - as.emit(`type:${error.payload.type}`, error.payload) - } else if (error.source === "discord") { - discord.cloud.emit("event", error.payload) - } + // Retry + if (error.source === "matrix") { + as.emit(`type:${error.payload.type}`, error.payload) + } else if (error.source === "discord") { + discord.cloud.emit("event", error.payload) + } - // Redact the error to stop people from executing multiple retries - await api.redactEvent(roomID, event.event_id) + // Redact the error to stop people from executing multiple retries + await api.redactEvent(roomID, event.event_id) + }, roomID) } sync.addTemporaryListener(as, "type:m.room.message", guard("m.room.message", From 261bb1b8c8ef9bb80f1debd6cbdf94da1c505d94 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 5 Dec 2025 17:13:56 +1300 Subject: [PATCH 039/153] Future-proof permissions --- addbot.js | 11 ++++++++++- src/m2d/event-dispatcher.js | 2 +- src/web/routes/oauth.js | 4 ++-- test/addbot.test.js | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/addbot.js b/addbot.js index ef1cc63e..f0e850ce 100755 --- a/addbot.js +++ b/addbot.js @@ -1,12 +1,20 @@ #!/usr/bin/env node // @ts-check +const DiscordTypes = require("discord-api-types/v10") + const {reg} = require("./src/matrix/read-registration") const token = reg.ooye.discord_token const id = Buffer.from(token.split(".")[0], "base64").toString() +const permissions = +( DiscordTypes.PermissionFlagsBits.ManageWebhooks +| DiscordTypes.PermissionFlagsBits.ManageGuildExpressions +| DiscordTypes.PermissionFlagsBits.ManageMessages +| DiscordTypes.PermissionFlagsBits.PinMessages +| DiscordTypes.PermissionFlagsBits.UseExternalEmojis) function addbot() { - return `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=${id}&scope=bot&permissions=1610883072 ` + return `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=${id}&scope=bot&permissions=${permissions} ` } /* c8 ignore next 3 */ @@ -16,3 +24,4 @@ if (process.argv.find(a => a.endsWith("addbot") || a.endsWith("addbot.js"))) { module.exports.id = id module.exports.addbot = addbot +module.exports.permissions = permissions diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 985036e5..9fe6ed57 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -161,7 +161,7 @@ const errorRetrySema = new Semaphore() */ async function onRetryReactionAdd(reactionEvent) { const roomID = reactionEvent.room_id - errorRetrySema.request(async () => { + await errorRetrySema.request(async () => { const event = await api.getEvent(roomID, reactionEvent.content["m.relates_to"]?.event_id) // Check that it's a real error from OOYE diff --git a/src/web/routes/oauth.js b/src/web/routes/oauth.js index 80765d6a..fe352306 100644 --- a/src/web/routes/oauth.js +++ b/src/web/routes/oauth.js @@ -8,7 +8,7 @@ const DiscordTypes = require("discord-api-types/v10") const getRelativePath = require("get-relative-path") const {discord, as, db, sync} = require("../../passthrough") -const {id} = require("../../../addbot") +const {id, permissions} = require("../../../addbot") /** @type {import("../auth")} */ const auth = sync.require("../auth") const {reg} = require("../../matrix/read-registration") @@ -51,7 +51,7 @@ as.router.get("/oauth", defineEventHandler(async event => { async function tryAgain() { const newState = randomUUID() await session.update({state: newState}) - return sendRedirect(event, `https://discord.com/oauth2/authorize?client_id=${id}&scope=${scope}&permissions=1610883072&response_type=code&redirect_uri=${redirect_uri}&state=${newState}`) + return sendRedirect(event, `https://discord.com/oauth2/authorize?client_id=${id}&scope=${scope}&permissions=${permissions}&response_type=code&redirect_uri=${redirect_uri}&state=${newState}`) } const parsedQuery = await getValidatedQuery(event, schema.code.safeParse) diff --git a/test/addbot.test.js b/test/addbot.test.js index 17c6dda2..41300516 100644 --- a/test/addbot.test.js +++ b/test/addbot.test.js @@ -4,5 +4,5 @@ const {addbot} = require("../addbot") const {test} = require("supertape") test("addbot: returns message and invite link", t => { - t.equal(addbot(), `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=684280192553844747&scope=bot&permissions=1610883072 `) + t.equal(addbot(), `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=684280192553844747&scope=bot&permissions=2251801424568320 `) }) From 653e38a9d2ba4548117d1375f2573a9cbf6ad2f4 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 6 Dec 2025 03:10:51 +1300 Subject: [PATCH 040/153] Interpret Matrix media spoilers --- src/m2d/converters/event-to-message.js | 45 +++++-- src/m2d/converters/event-to-message.test.js | 133 +++++++++++++++++++- test/ooye-test-data.sql | 3 +- 3 files changed, 165 insertions(+), 16 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 7002c385..cbdeded9 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -555,25 +555,40 @@ async function eventToMessage(event, guild, di) { // Handle images first - might need to handle their `body`/`formatted_body` as well, which will fall through to the text processor let shouldProcessTextEvent = event.type === "m.room.message" && (event.content.msgtype === "m.text" || event.content.msgtype === "m.emote") if (event.type === "m.room.message" && (event.content.msgtype === "m.file" || event.content.msgtype === "m.video" || event.content.msgtype === "m.audio" || event.content.msgtype === "m.image")) { + // Build message content in addition to the uploaded file + const fileIsSpoiler = event.content["page.codeberg.everypizza.msc4193.spoiler"] + const fileSpoilerReason = event.content["page.codeberg.everypizza.msc4193.spoiler.reason"] + content = "" + const captionContent = new mxUtils.MatrixStringBuilder() + + // Caption from Matrix message + const fileHasCaption = (event.content.body && event.content.filename && event.content.body !== event.content.filename) || event.content.formatted_body + if (fileHasCaption) { + captionContent.addLine(event.content.body || "", event.content.formatted_body || tag`${event.content.body || ""}`) + } + + // Spoiler message + if (fileIsSpoiler && typeof fileSpoilerReason === "string") { + captionContent.addLine(`(Spoiler: ${fileSpoilerReason})`) + } + + // File link as alternative to uploading if (!("file" in event.content) && event.content.info?.size > getFileSizeForGuild(guild)) { // Upload (unencrypted) file as link, because it's too large for Discord // Do this by constructing a sample Matrix message with the link and then use the text processor to convert that + the original caption. const url = mxUtils.getPublicUrlForMxc(event.content.url) assert(url) const filename = event.content.filename || event.content.body - const newText = new mxUtils.MatrixStringBuilder() const emoji = attachmentEmojis.has(event.content.msgtype) ? attachmentEmojis.get(event.content.msgtype) + " " : "" - newText.addLine(`${emoji}Uploaded file: ${url} (${pb(event.content.info.size)})`, tag`${emoji}Uploaded file: ${filename} (${pb(event.content.info.size)})`) - // Check if the event has a caption that we need to add as well - if ((event.content.body && event.content.filename && event.content.body !== event.content.filename) || event.content.formatted_body) { - newText.addLine(event.content.body || "", event.content.formatted_body || tag`${event.content.body || ""}`) + if (fileIsSpoiler) { + captionContent.addLine(`${emoji}Uploaded SPOILER file: <${url}> (${pb(event.content.info.size)})`, tag`${emoji}Uploaded SPOILER file: ${filename} (${pb(event.content.info.size)})`) // the space is necessary to work around a bug in Discord's URL previewer. the preview still gets blurred in the client. + } else { + captionContent.addLine(`${emoji}Uploaded file: ${url} (${pb(event.content.info.size)})`, tag`${emoji}Uploaded file: ${filename} (${pb(event.content.info.size)})`) } - Object.assign(event.content, newText.get()) - shouldProcessTextEvent = true } else { // Upload file as file - content = "" - const filename = event.content.filename || event.content.body + let filename = event.content.filename || event.content.body + if (fileIsSpoiler) filename = "SPOILER_" + filename if ("file" in event.content) { // Encrypted assert.equal(event.content.file.key.alg, "A256CTR") @@ -584,12 +599,16 @@ async function eventToMessage(event, guild, di) { attachments.push({id: "0", filename}) pendingFiles.push({name: filename, mxc: event.content.url}) } - // Check if we also need to process a text event for this image - if it has a caption that's different from its filename - if ((event.content.body && event.content.filename && event.content.body !== event.content.filename) || event.content.formatted_body) { - shouldProcessTextEvent = true - } + } + + // Add result to content + const result = captionContent.get() + if (result.body) { + Object.assign(event.content, {body: result.body, format: result.format, formatted_body: result.formatted_body}) + shouldProcessTextEvent = true } } + if (event.type === "m.sticker") { content = "" let filename = event.content.body diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 2e347f54..439e07fb 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -404,6 +404,135 @@ test("event2message: spoiler reasons work", async t => { ) }) +test("event2message: media spoilers work", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "pitstop.png", + filename: "pitstop.png", + info: { + h: 870, + mimetype: "image/png", + size: 729990, + w: 674, + "xyz.amorgan.blurhash": "UqOMmRM{_Mx[xZaxR*tQ.8ayxtWBRkRkWUWB" + }, + msgtype: "m.image", + "page.codeberg.everypizza.msc4193.spoiler": true, + url: "mxc://agiadn.org/JY5NvEFojTvYDp5znjGIkkQ7Ez7GwsdT" + }, + origin_server_ts: 1764885561299, + room_id: "!zq94fae5bVKUubZLp7:agiadn.org", + sender: "@underscore_x:agiadn.org", + type: "m.room.message", + event_id: "$6P7u-lpu2u73ZrHUru2UG1rPfsh8PfYLPK21o3SNIN4", + user_id: "@underscore_x:agiadn.org" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "underscore_x", + content: "", + avatar_url: undefined, + attachments: [{id: "0", filename: "SPOILER_pitstop.png"}], + pendingFiles: [{ + mxc: "mxc://agiadn.org/JY5NvEFojTvYDp5znjGIkkQ7Ez7GwsdT", + name: "SPOILER_pitstop.png", + }] + }] + } + ) +}) + +test("event2message: media spoilers with reason work", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "pitstop.png", + filename: "pitstop.png", + info: { + h: 870, + mimetype: "image/png", + size: 729990, + w: 674, + "xyz.amorgan.blurhash": "UqOMmRM{_Mx[xZaxR*tQ.8ayxtWBRkRkWUWB" + }, + msgtype: "m.image", + "page.codeberg.everypizza.msc4193.spoiler": true, + "page.codeberg.everypizza.msc4193.spoiler.reason": "golden witch solutions", + url: "mxc://agiadn.org/JY5NvEFojTvYDp5znjGIkkQ7Ez7GwsdT" + }, + origin_server_ts: 1764885561299, + room_id: "!zq94fae5bVKUubZLp7:agiadn.org", + sender: "@underscore_x:agiadn.org", + type: "m.room.message", + event_id: "$6P7u-lpu2u73ZrHUru2UG1rPfsh8PfYLPK21o3SNIN4", + user_id: "@underscore_x:agiadn.org" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "underscore_x", + allowed_mentions: { + parse: ["users", "roles"] + }, + content: "(Spoiler: golden witch solutions)", + avatar_url: undefined, + attachments: [{id: "0", filename: "SPOILER_pitstop.png"}], + pendingFiles: [{ + mxc: "mxc://agiadn.org/JY5NvEFojTvYDp5znjGIkkQ7Ez7GwsdT", + name: "SPOILER_pitstop.png", + }] + }] + } + ) +}) + +test("event2message: spoiler files too large for Discord are linked and retain reason", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "pitstop.png", + filename: "pitstop.png", + info: { + h: 870, + mimetype: "image/png", + size: 40000000, + w: 674, + "xyz.amorgan.blurhash": "UqOMmRM{_Mx[xZaxR*tQ.8ayxtWBRkRkWUWB" + }, + msgtype: "m.image", + "page.codeberg.everypizza.msc4193.spoiler": true, + "page.codeberg.everypizza.msc4193.spoiler.reason": "golden witch secrets", + url: "mxc://agiadn.org/JY5NvEFojTvYDp5znjGIkkQ7Ez7GwsdT" + }, + origin_server_ts: 1764885561299, + room_id: "!zq94fae5bVKUubZLp7:agiadn.org", + sender: "@underscore_x:agiadn.org", + type: "m.room.message", + event_id: "$6P7u-lpu2u73ZrHUru2UG1rPfsh8PfYLPK21o3SNIN4", + user_id: "@underscore_x:agiadn.org" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "underscore_x", + allowed_mentions: { + parse: ["users", "roles"] + }, + content: "(Spoiler: golden witch secrets)\n🖼️ _Uploaded **SPOILER** file: ||[pitstop.png](https://bridge.example.org/download/matrix/agiadn.org/JY5NvEFojTvYDp5znjGIkkQ7Ez7GwsdT )|| (40 MB)_", + avatar_url: undefined + }] + } + ) +}) + test("event2message: markdown syntax is escaped", async t => { t.deepEqual( await eventToMessage({ @@ -4236,7 +4365,7 @@ test("event2message: files too large for Discord can have a plaintext caption", messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "🖼️ _Uploaded file: [cool cat.png](https://bridge.example.org/download/matrix/cadence.moe/IvxVJFLEuksCNnbojdSIeEvn) (40 MB)_\nCat emoji surrounded by pink hearts", + content: "Cat emoji surrounded by pink hearts\n🖼️ _Uploaded file: [cool cat.png](https://bridge.example.org/download/matrix/cadence.moe/IvxVJFLEuksCNnbojdSIeEvn) (40 MB)_", avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", allowed_mentions: { parse: ["users", "roles"] @@ -4283,7 +4412,7 @@ test("event2message: files too large for Discord can have a formatted caption", messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "🖼️ _Uploaded file: [5740.jpg](https://bridge.example.org/download/matrix/thomcat.rocks/RTHsXmcMPXmuHqVNsnbKtRbh) (40 MB)_\nthis event has `formatting`", + content: "this event has `formatting`\n🖼️ _Uploaded file: [5740.jpg](https://bridge.example.org/download/matrix/thomcat.rocks/RTHsXmcMPXmuHqVNsnbKtRbh) (40 MB)_", avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", allowed_mentions: { parse: ["users", "roles"] diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index ab11c8b7..9ca3c5a3 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -170,7 +170,8 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url, power_level) V ('!TqlyQmifxGUggEmdBN:cadence.moe', '@ampflower:matrix.org', 'Ampflower 🌺', 'mxc://cadence.moe/PRfhXYBTOalvgQYtmCLeUXko', 0), ('!TqlyQmifxGUggEmdBN:cadence.moe', '@aflower:syndicated.gay', 'Rose', 'mxc://syndicated.gay/ZkBUPXCiXTjdJvONpLJmcbKP', 0), ('!TqlyQmifxGUggEmdBN:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', NULL, 0), -('!iSyXgNxQcEuXoXpsSn:pussthecat.org', '@austin:tchncs.de', 'Austin Huang', 'mxc://tchncs.de/090a2b5e07eed2f71e84edad5207221e6c8f8b8e', 0); +('!iSyXgNxQcEuXoXpsSn:pussthecat.org', '@austin:tchncs.de', 'Austin Huang', 'mxc://tchncs.de/090a2b5e07eed2f71e84edad5207221e6c8f8b8e', 0), +('!zq94fae5bVKUubZLp7:agiadn.org', '@underscore_x:agiadn.org', 'underscore_x', NULL, 100); INSERT INTO reaction (hashed_event_id, message_id, encoded_emoji) VALUES (5162930312280790092, '1141501302736695317', '%F0%9F%90%88'); From 2563a47e784d5516731fa04b076cb2676bd44d15 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 11 Dec 2025 12:46:28 +1300 Subject: [PATCH 041/153] Properly hide embeds for links in reply previews --- src/d2m/converters/message-to-event.js | 1 + src/m2d/converters/event-to-message.js | 2 +- src/types.d.ts | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index fecd2d44..ab011c71 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -370,6 +370,7 @@ async function messageToEvent(message, guild, options = {}, di) { return `https://matrix.to/#/${oldestRow.room_id}/${event_id}?${via}` } catch (e) { // M_NOT_FOUND: Unable to find event from in direction Direction.FORWARDS + // not supported in Conduit and descendants return `[unknown event, timestamp resolution failed, in room: https://matrix.to/#/${oldestRow.room_id}?${via}]` } } diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index cbdeded9..7fd7b8a3 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -739,11 +739,11 @@ async function eventToMessage(event, guild, di) { return convertEmoji(mxcUrlMatch?.[1], titleTextMatch?.[1], false, false) }) repliedToContent = repliedToContent.replace(/<[^:>][^>]*>/g, "") // Completely strip all HTML tags and formatting. - repliedToContent = repliedToContent.replace(/\bhttps?:\/\/[^ )]*/g, "<$&>") repliedToContent = entities.decodeHTML5Strict(repliedToContent) // Remove entities like & " const contentPreviewChunks = chunk(repliedToContent, 50) if (contentPreviewChunks.length) { contentPreview = ": " + contentPreviewChunks[0] + contentPreview = contentPreview.replace(/\bhttps?:\/\/[^ )]*/g, "<$&>") if (contentPreviewChunks.length > 1) contentPreview = contentPreview.replace(/[,.']$/, "") + "..." } else { contentPreview = "" diff --git a/src/types.d.ts b/src/types.d.ts index f9488b97..8b8bae9b 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -221,6 +221,7 @@ export namespace Event { filename?: string url: string info?: any + "page.codeberg.everypizza.msc4193.spoiler"?: boolean "m.relates_to"?: { "m.in_reply_to": { event_id: string @@ -238,6 +239,7 @@ export namespace Event { format?: "org.matrix.custom.html" formatted_body?: string filename?: string + "page.codeberg.everypizza.msc4193.spoiler"?: boolean file: { url: string iv: string From 696a45f344901573d96c61ba7d0261fc508843df Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 13 Dec 2025 14:03:10 +1300 Subject: [PATCH 042/153] MAS compatibility --- src/matrix/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/api.js b/src/matrix/api.js index d746b252..ddaf9b59 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -44,6 +44,7 @@ async function register(username) { try { await mreq.mreq("POST", "/client/v3/register", { type: "m.login.application_service", + inhibit_login: true, // https://github.com/element-hq/matrix-bot-sdk/pull/70/changes https://github.com/matrix-org/matrix-spec-proposals/blob/quenting/as-device-management/proposals/4190-as-device-management.md username }) } catch (e) { From 239568a8e518475bca257bfec795b05598ad341f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Dec 2025 00:34:37 +1300 Subject: [PATCH 043/153] Fully remove failed experiment --- src/d2m/actions/create-room.js | 3 --- src/matrix/kstate.js | 6 ------ test/data.js | 1 - 3 files changed, 10 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 5b7d3be1..f386e694 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -158,8 +158,6 @@ async function channelToKState(channel, guild, di) { }, users: {...spacePower, ...globalAdminPower} }, - "chat.schildi.hide_ui/read_receipts": { - }, [`uk.half-shot.bridge/moe.cadence.ooye://discord/${guild.id}/${channel.id}`]: { bridgebot: `@${reg.sender_localpart}:${reg.ooye.server_name}`, protocol: { @@ -256,7 +254,6 @@ async function postApplyPowerLevels(kstate, callback) { const powerLevelContent = kstate["m.room.power_levels/"] const kstateWithoutPowerLevels = {...kstate} delete kstateWithoutPowerLevels["m.room.power_levels/"] - delete kstateWithoutPowerLevels["chat.schildi.hide_ui/read_receipts"] /** @type {string} */ const roomID = await callback(kstateWithoutPowerLevels) diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index 03d09e06..11155f54 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -87,12 +87,6 @@ function diffKState(actual, target) { diff[key] = temp } - } else if (key === "chat.schildi.hide_ui/read_receipts") { - // Special handling: don't add this key if it's new. Do overwrite if already present. - if (key in actual) { - diff[key] = target[key] - } - } else if (key in actual) { // diff if (!isDeepStrictEqual(actual[key], target[key])) { diff --git a/test/data.js b/test/data.js index bc7c192a..b9e9da41 100644 --- a/test/data.js +++ b/test/data.js @@ -132,7 +132,6 @@ module.exports = { room: 0 } }, - "chat.schildi.hide_ui/read_receipts": {}, "uk.half-shot.bridge/moe.cadence.ooye://discord/112760669178241024/112760669178241024": { bridgebot: "@_ooye_bot:cadence.moe", protocol: { From 4bc7e794ab936b38ecc1631f883fba9de0433d70 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Dec 2025 00:36:11 +1300 Subject: [PATCH 044/153] Refactor private chat creation --- src/matrix/api.js | 33 +++++++++++- src/web/routes/log-in-with-matrix.js | 25 +-------- src/web/routes/log-in-with-matrix.test.js | 64 ++--------------------- 3 files changed, 36 insertions(+), 86 deletions(-) diff --git a/src/matrix/api.js b/src/matrix/api.js index ddaf9b59..d2a0e9d1 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -5,7 +5,7 @@ const assert = require("assert").strict const streamWeb = require("stream/web") const passthrough = require("../passthrough") -const {sync} = passthrough +const {sync, db, select} = passthrough /** @type {import("./mreq")} */ const mreq = sync.require("./mreq") /** @type {import("./txnid")} */ @@ -513,6 +513,36 @@ function versions() { return mreq.mreq("GET", "/client/versions") } +/** + * @param {string} mxid + */ +async function usePrivateChat(mxid) { + // Check if we have an existing DM + let roomID = select("direct", "room_id", {mxid}).pluck().get() + if (roomID) { + // Check that the person is/still in the room + try { + var member = await getStateEvent(roomID, "m.room.member", mxid) + } catch (e) {} + + // Invite them back to the room if needed + if (!member || member.membership === "leave") { + await inviteToRoom(roomID, mxid) + } + return roomID + } + + // No existing DM, create a new room and invite + roomID = await createRoom({ + invite: [mxid], + is_direct: true, + preset: "trusted_private_chat" + }) + // Store the newly created room in the database (not using account data due to awkward bugs with misaligned state) + db.prepare("REPLACE INTO direct (mxid, room_id) VALUES (?, ?)").run(mxid, roomID) + return roomID +} + module.exports.path = path module.exports.register = register module.exports.createRoom = createRoom @@ -550,3 +580,4 @@ module.exports.setAccountData = setAccountData module.exports.setPresence = setPresence module.exports.getProfile = getProfile module.exports.versions = versions +module.exports.usePrivateChat = usePrivateChat diff --git a/src/web/routes/log-in-with-matrix.js b/src/web/routes/log-in-with-matrix.js index 574c312f..d36d8fa4 100644 --- a/src/web/routes/log-in-with-matrix.js +++ b/src/web/routes/log-in-with-matrix.js @@ -79,30 +79,7 @@ as.router.post("/api/log-in-with-matrix", defineEventHandler(async event => { } } - // Check if we have an existing DM - let roomID = select("direct", "room_id", {mxid}).pluck().get() - if (roomID) { - // Check that the person is/still in the room - try { - var member = await api.getStateEvent(roomID, "m.room.member", mxid) - } catch (e) {} - - // Invite them back to the room if needed - if (!member || member.membership === "leave") { - await api.inviteToRoom(roomID, mxid) - } - } - - // No existing DM, create a new room and invite - else { - roomID = await api.createRoom({ - invite: [mxid], - is_direct: true, - preset: "trusted_private_chat" - }) - // Store the newly created room in account data (Matrix doesn't do this for us automatically, sigh...) - db.prepare("REPLACE INTO direct (mxid, room_id) VALUES (?, ?)").run(mxid, roomID) - } + const roomID = await api.usePrivateChat(mxid) const token = randomUUID() diff --git a/src/web/routes/log-in-with-matrix.test.js b/src/web/routes/log-in-with-matrix.test.js index bc9c7e02..a4030555 100644 --- a/src/web/routes/log-in-with-matrix.test.js +++ b/src/web/routes/log-in-with-matrix.test.js @@ -34,7 +34,7 @@ test("log in with matrix: checks if mxid domain format looks valid", async t => t.match(error.data.fieldErrors.mxid, /must match pattern/) }) -test("log in with matrix: sends message when there is no existing dm room", async t => { +test("log in with matrix: sends message to log in", async t => { const event = {} let called = 0 await router.test("post", "/api/log-in-with-matrix", { @@ -42,8 +42,9 @@ test("log in with matrix: sends message when there is no existing dm room", asyn mxid: "@cadence:cadence.moe" }, api: { - async createRoom() { + async usePrivateChat(mxid) { called++ + t.equal(mxid, "@cadence:cadence.moe") return "!created:cadence.moe" }, async sendEvent(roomID, type, content) { @@ -72,65 +73,6 @@ test("log in with matrix: does not send another message when a log in is in prog t.match(event.node.res.getHeader("location"), /We already sent you a link on Matrix/) }) -test("log in with matrix: reuses room from direct", async t => { - const event = {} - let called = 0 - await router.test("post", "/api/log-in-with-matrix", { - body: { - mxid: "@user1:example.org" - }, - api: { - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!existing:cadence.moe") - t.equal(type, "m.room.member") - t.equal(key, "@user1:example.org") - return {membership: "join"} - }, - async sendEvent(roomID) { - called++ - t.equal(roomID, "!existing:cadence.moe") - return "" - } - }, - event - }) - t.match(event.node.res.getHeader("location"), /Please check your inbox on Matrix/) - t.equal(called, 2) -}) - -test("log in with matrix: reuses room from direct, reinviting if user has left", async t => { - const event = {} - let called = 0 - await router.test("post", "/api/log-in-with-matrix", { - body: { - mxid: "@user2:example.org" - }, - api: { - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!existing:cadence.moe") - t.equal(type, "m.room.member") - t.equal(key, "@user2:example.org") - throw new MatrixServerError({errcode: "M_NOT_FOUND"}) - }, - async inviteToRoom(roomID, mxid) { - called++ - t.equal(roomID, "!existing:cadence.moe") - t.equal(mxid, "@user2:example.org") - }, - async sendEvent(roomID) { - called++ - t.equal(roomID, "!existing:cadence.moe") - return "" - } - }, - event - }) - t.match(event.node.res.getHeader("location"), /Please check your inbox on Matrix/) - t.equal(called, 3) -}) - // ***** third request ***** From a6bb248c0ad8f01162d831f9fdba784747cd90ac Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Dec 2025 00:36:43 +1300 Subject: [PATCH 045/153] Fix pointer being included for cross-room replies --- src/d2m/converters/message-to-event.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index ab011c71..5360702e 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -267,6 +267,7 @@ async function messageToEvent(message, guild, options = {}, di) { const mentions = {} /** @type {{event_id: string, room_id: string, source: number, channel_id: string}?} */ let repliedToEventRow = null + let repliedToEventInDifferentRoom = false let repliedToUnknownEvent = false let repliedToEventSenderMxid = null @@ -491,6 +492,7 @@ async function messageToEvent(message, guild, options = {}, di) { if (repliedToEventRow) { // Generate a reply pointing to the Matrix event we found const latestRoomID = select("channel_room", "room_id", {channel_id: repliedToEventRow.channel_id}).pluck().get() // native replies don't work across room upgrades, so make sure the old and new message are in the same room + if (latestRoomID !== repliedToEventRow.room_id) repliedToEventInDifferentRoom = true html = (latestRoomID === repliedToEventRow.room_id ? "" : "") + `
In reply to ${repliedToUserHtml}` @@ -789,7 +791,7 @@ async function messageToEvent(message, guild, options = {}, di) { } // Rich replies - if (repliedToEventRow) { + if (repliedToEventRow && !repliedToEventInDifferentRoom) { Object.assign(events[0], { "m.relates_to": { "m.in_reply_to": { From e4d0838af5a1fe351e28ea4e5e0b145f22ab0407 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Dec 2025 02:15:17 +1300 Subject: [PATCH 046/153] Support creating v12 rooms --- src/d2m/actions/create-room.js | 54 +++++++++++++++++----------- src/d2m/actions/create-room.test.js | 6 ++-- src/d2m/actions/create-space.js | 20 +++++++---- src/d2m/actions/create-space.test.js | 6 +++- src/db/orm.test.js | 2 +- src/m2d/converters/utils.js | 47 +++++++++++++++++++++++- src/matrix/api.js | 11 ++++++ src/matrix/kstate.js | 31 +++++++++++++--- src/types.d.ts | 7 +++- test/data.js | 3 +- 10 files changed, 149 insertions(+), 38 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index f386e694..a1d99407 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -126,15 +126,21 @@ async function channelToKState(channel, guild, di) { const everyoneCanSend = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendMessages) const everyoneCanMentionEveryone = dUtils.hasAllPermissions(everyonePermissions, ["MentionEveryone"]) - const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all() - const globalAdminPower = globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {}) - /** @type {Ty.Event.M_Power_Levels} */ const spacePowerEvent = await di.api.getStateEvent(guildSpaceID, "m.room.power_levels", "") const spacePower = spacePowerEvent.users + const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all() + const globalAdminPower = globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {}) + const additionalCreators = select("member_power", "mxid", {room_id: "*"}, "AND power_level > 100").pluck().all() + + const creationContent = {} + creationContent.additional_creators = additionalCreators + if (channel.type === DiscordTypes.ChannelType.GuildForum) creationContent.type = "m.space" + /** @type {any} */ const channelKState = { + "m.room.create/": creationContent, "m.room.name/": {name: convertedName}, "m.room.topic/": {topic: convertedTopic}, "m.room.avatar/": avatarEventContent, @@ -193,7 +199,7 @@ async function channelToKState(channel, guild, di) { /** * Create a bridge room, store the relationship in the database, and add it to the guild's space. * @param {DiscordTypes.APIGuildTextChannel} channel - * @param guild + * @param {DiscordTypes.APIGuild} guild * @param {string} spaceID * @param {any} kstate * @param {number} privacyLevel @@ -203,9 +209,6 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { let threadParent = null if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id - let spaceCreationContent = {} - if (channel.type === DiscordTypes.ChannelType.GuildForum) spaceCreationContent = {creation_content: {type: "m.space"}} - // Name and topic can be done earlier in room creation rather than in initial_state // https://spec.matrix.org/latest/client-server-api/#creation const name = kstate["m.room.name/"].name @@ -215,7 +218,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { delete kstate["m.room.topic/"] assert(topic) - const roomID = await postApplyPowerLevels(kstate, async kstate => { + const roomCreate = await postApplyPowerLevels(kstate, async kstate => { const roomID = await api.createRoom({ name, topic, @@ -223,16 +226,20 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel], invite: [], initial_state: await ks.kstateToState(kstate), - ...spaceCreationContent + creation_content: ks.kstateToCreationContent(kstate) }) + /** @type {Ty.Event.StateOuter} */ + const roomCreate = await api.getStateEventOuter(roomID, "m.room.create", "") + db.transaction(() => { - db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent) + db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, guild_id) VALUES (?, ?, ?, NULL, ?, ?)").run(channel.id, roomID, channel.name, threadParent, guild.id) db.prepare("INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) VALUES (?, ?, 0)").run(channel.id, roomID) })() - return roomID + return roomCreate }) + const roomID = roomCreate.room_id // Put the newly created child into the space await _syncSpaceMember(channel, spaceID, roomID, guild.id) @@ -247,25 +254,30 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { * https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/room.py#L1170-L1210 * https://github.com/matrix-org/matrix-spec/issues/492 * @param {any} kstate - * @param {(_: any) => Promise} callback must return room ID - * @returns {Promise} room ID + * @param {(_: any) => Promise>} callback must return room ID and room version + * @returns {Promise>} room ID */ async function postApplyPowerLevels(kstate, callback) { const powerLevelContent = kstate["m.room.power_levels/"] const kstateWithoutPowerLevels = {...kstate} delete kstateWithoutPowerLevels["m.room.power_levels/"] - /** @type {string} */ - const roomID = await callback(kstateWithoutPowerLevels) + const roomCreate = await callback(kstateWithoutPowerLevels) + const roomID = roomCreate.room_id // Now *really* apply the power level overrides on top of what Synapse *really* set if (powerLevelContent) { - const newRoomKState = await ks.roomToKState(roomID) - const newRoomPowerLevelsDiff = ks.diffKState(newRoomKState, {"m.room.power_levels/": powerLevelContent}) - await ks.applyKStateDiffToRoom(roomID, newRoomPowerLevelsDiff) + mUtils.removeCreatorsFromPowerLevels(roomCreate, powerLevelContent) + + const originalPowerLevels = await api.getStateEvent(roomID, "m.room.power_levels", "") + const powerLevelsDiff = ks.diffKState( + {"m.room.power_levels/": originalPowerLevels}, + {"m.room.power_levels/": powerLevelContent} + ) + await ks.applyKStateDiffToRoom(roomID, powerLevelsDiff) } - return roomID + return roomCreate } /** @@ -392,8 +404,8 @@ async function _syncRoom(channelID, shouldActuallySync) { // sync channel state to room const roomKState = await ks.roomToKState(roomID) - if (+roomKState["m.room.create/"].room_version <= 8) { - // join_rule `restricted` is not available in room version < 8 and not working properly in version == 8 + if (!mUtils.roomHasAtLeastVersion(roomKState["m.room.create/"].room_version, 9)) { + // join_rule `restricted` is not available in room version < 8 and not working properly in version == 8, so require version 9 // read more: https://spec.matrix.org/v1.8/rooms/v9/ // we have to use `public` instead, otherwise the room will be unjoinable. channelKState["m.room.join_rules/"] = {join_rule: "public"} diff --git a/src/d2m/actions/create-room.test.js b/src/d2m/actions/create-room.test.js index e653744a..ca09f735 100644 --- a/src/d2m/actions/create-room.test.js +++ b/src/d2m/actions/create-room.test.js @@ -124,7 +124,9 @@ test("channel2room: read-only discord channel", async t => { return {} } const expected = { - "chat.schildi.hide_ui/read_receipts": {}, + "m.room.create/": { + additional_creators: ["@test_auto_invite:example.org"], + }, "m.room.avatar/": { url: { $url: "/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024", @@ -161,7 +163,7 @@ test("channel2room: read-only discord channel", async t => { room: 20, }, users: { - "@test_auto_invite:example.org": 100, + "@test_auto_invite:example.org": 150, }, }, "m.space.parent/!jjmvBegULiLucuWEHU:cadence.moe": { diff --git a/src/d2m/actions/create-space.js b/src/d2m/actions/create-space.js index 8bce3adc..1fb1911f 100644 --- a/src/d2m/actions/create-space.js +++ b/src/d2m/actions/create-space.js @@ -35,8 +35,8 @@ async function createSpace(guild, kstate) { const enablePresenceByDefault = +(memberCount < 50) // scary! all active users in a presence-enabled guild will be pinging the server every <30 seconds to stay online const globalAdmins = select("member_power", "mxid", {room_id: "*"}).pluck().all() - const roomID = await createRoom.postApplyPowerLevels(kstate, async kstate => { - return api.createRoom({ + const roomCreate = await createRoom.postApplyPowerLevels(kstate, async kstate => { + const roomID = await api.createRoom({ name, preset: createRoom.PRIVACY_ENUMS.PRESET[createRoom.DEFAULT_PRIVACY_LEVEL], // New spaces will have to use the default privacy level; we obviously can't look up the existing entry visibility: createRoom.PRIVACY_ENUMS.VISIBILITY[createRoom.DEFAULT_PRIVACY_LEVEL], @@ -46,12 +46,14 @@ async function createSpace(guild, kstate) { }, invite: globalAdmins, topic, - creation_content: { - type: "m.space" - }, - initial_state: await ks.kstateToState(kstate) + initial_state: await ks.kstateToState(kstate), + creation_content: ks.kstateToCreationContent(kstate) }) + const roomCreate = await api.getStateEventOuter(roomID, "m.room.create", "") + return roomCreate }) + const roomID = roomCreate.room_id + db.prepare("INSERT INTO guild_space (guild_id, space_id, presence) VALUES (?, ?, ?)").run(guild.id, roomID, enablePresenceByDefault) return roomID } @@ -63,7 +65,13 @@ async function createSpace(guild, kstate) { async function guildToKState(guild, privacyLevel) { assert.equal(typeof privacyLevel, "number") const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all() + const additionalCreators = select("member_power", "mxid", {room_id: "*"}, "AND power_level > 100").pluck().all() + const guildKState = { + "m.room.create/": { + type: "m.space", + additional_creators: additionalCreators + }, "m.room.name/": {name: guild.name}, "m.room.avatar/": { $if: guild.icon, diff --git a/src/d2m/actions/create-space.test.js b/src/d2m/actions/create-space.test.js index cb4d90a4..fc6eba41 100644 --- a/src/d2m/actions/create-space.test.js +++ b/src/d2m/actions/create-space.test.js @@ -13,6 +13,10 @@ test("guild2space: can generate kstate for a guild, passing privacy level 0", as t.deepEqual( await kstateUploadMxc(kstateStripConditionals(await guildToKState(testData.guild.general, 0))), { + "m.room.create/": { + additional_creators: ["@test_auto_invite:example.org"], + type: "m.space" + }, "m.room.avatar/": { url: "mxc://cadence.moe/zKXGZhmImMHuGQZWJEFKJbsF" }, @@ -30,7 +34,7 @@ test("guild2space: can generate kstate for a guild, passing privacy level 0", as }, "m.room.power_levels/": { users: { - "@test_auto_invite:example.org": 100 + "@test_auto_invite:example.org": 150 }, }, } diff --git a/src/db/orm.test.js b/src/db/orm.test.js index 56a3257d..6f6018ee 100644 --- a/src/db/orm.test.js +++ b/src/db/orm.test.js @@ -66,5 +66,5 @@ test("orm: select unsafe works (to select complex column names that can't be typ .and("where member_power.room_id = '*' and member_cache.power_level != member_power.power_level") .selectUnsafe("mxid", "member_cache.room_id", "member_power.power_level") .all() - t.equal(results[0].power_level, 100) + t.equal(results[0].power_level, 150) }) diff --git a/src/m2d/converters/utils.js b/src/m2d/converters/utils.js index 41cb0af0..59035fef 100644 --- a/src/m2d/converters/utils.js +++ b/src/m2d/converters/utils.js @@ -1,7 +1,7 @@ // @ts-check const assert = require("assert").strict - +const Ty = require("../../types") const passthrough = require("../../passthrough") const {db} = passthrough @@ -232,6 +232,49 @@ function getPublicUrlForMxc(mxc) { return `${reg.ooye.bridge_origin}/download/matrix/${serverAndMediaID}` } +/** + * @param {string} roomVersionString + * @param {number} desiredVersion + */ +function roomHasAtLeastVersion(roomVersionString, desiredVersion) { + /* + I hate this. + The spec instructs me to compare room versions ordinally, for example, "In room versions 12 and higher..." + So if the real room version is 13, this should pass the check. + However, the spec also says "room versions are not intended to be parsed and should be treated as opaque identifiers", "due to versions not being ordered or hierarchical". + So versions are unordered and opaque and you can't parse them, but you're still expected to parse them to a number and compare them to another number to measure if it's "12 or higher"? + Theoretically MSC3244 would clean this up, but that isn't happening since Element removed support for MSC3244: https://github.com/element-hq/element-web/commit/644b8415912afb9c5eed54859a444a2ee7224117 + Element replaced it with the following function: + */ + + // Assumption: all unstable room versions don't support the feature. Calling code can check for unstable + // room versions explicitly if it wants to. The spec reserves [0-9] and `.` for its room versions. + if (!roomVersionString.match(/^[\d.]+$/)) { + return false; + } + + // Element dev note: While the spec says room versions are not linear, we can make reasonable assumptions + // until the room versions prove themselves to be non-linear in the spec. We should see this coming + // from a mile away and can course-correct this function if needed. + return Number(roomVersionString) >= Number(desiredVersion); +} + +/** + * Starting in room version 12, creators may not be specified in power levels users. + * Modifies the input power levels. + * @param {Ty.Event.StateOuter} roomCreateOuter + * @param {Ty.Event.M_Power_Levels} powerLevels + */ +function removeCreatorsFromPowerLevels(roomCreateOuter, powerLevels) { + assert(roomCreateOuter.sender) + if (roomHasAtLeastVersion(roomCreateOuter.content.room_version, 12)) { + for (const creator of (roomCreateOuter.content.additional_creators ?? []).concat(roomCreateOuter.sender)) { + delete powerLevels.users[creator] + } + } + return powerLevels +} + module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord module.exports.getPublicUrlForMxc = getPublicUrlForMxc @@ -239,3 +282,5 @@ module.exports.getEventIDHash = getEventIDHash module.exports.MatrixStringBuilder = MatrixStringBuilder module.exports.getViaServers = getViaServers module.exports.getViaServersQuery = getViaServersQuery +module.exports.roomHasAtLeastVersion = roomHasAtLeastVersion +module.exports.removeCreatorsFromPowerLevels = removeCreatorsFromPowerLevels diff --git a/src/matrix/api.js b/src/matrix/api.js index d2a0e9d1..c17d7893 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -138,6 +138,16 @@ function getStateEvent(roomID, type, key) { return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state/${type}/${key}`) } +/** + * @param {string} roomID + * @param {string} type + * @param {string} key + * @returns {Promise} the entire state event + */ +function getStateEventOuter(roomID, type, key) { + return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state/${type}/${key}?format=event`) +} + /** * @param {string} roomID * @returns {Promise} @@ -554,6 +564,7 @@ module.exports.getEvent = getEvent module.exports.getEventForTimestamp = getEventForTimestamp module.exports.getAllState = getAllState module.exports.getStateEvent = getStateEvent +module.exports.getStateEventOuter = getStateEventOuter module.exports.getInviteState = getInviteState module.exports.getJoinedMembers = getJoinedMembers module.exports.getMembers = getMembers diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index 11155f54..d1318896 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -10,6 +10,8 @@ const {sync} = passthrough const file = sync.require("./file") /** @type {import("./api")} */ const api = sync.require("./api") +/** @type {import("../m2d/converters/utils")} */ +const utils = sync.require("../m2d/converters/utils") /** Mutates the input. Not recursive - can only include or exclude entire state events. */ function kstateStripConditionals(kstate) { @@ -45,12 +47,13 @@ async function kstateUploadMxc(obj) { return obj } -/** Automatically strips conditionals and uploads URLs to mxc. */ +/** Automatically strips conditionals and uploads URLs to mxc. m.room.create is removed. */ async function kstateToState(kstate) { const events = [] kstateStripConditionals(kstate) await kstateUploadMxc(kstate) for (const [k, content] of Object.entries(kstate)) { + if (k === "m.room.create/") continue const slashIndex = k.indexOf("/") assert(slashIndex > 0) const type = k.slice(0, slashIndex) @@ -60,6 +63,11 @@ async function kstateToState(kstate) { return events } +/** Extracts m.room.create for use in room creation_content. */ +function kstateToCreationContent(kstate) { + return kstate["m.room.create/"] || {} +} + /** * @param {import("../types").Event.BaseStateEvent[]} events * @returns {any} @@ -68,6 +76,11 @@ function stateToKState(events) { const kstate = {} for (const event of events) { kstate[event.type + "/" + event.state_key] = event.content + + // need to remember m.room.create sender for later... + if (event.type === "m.room.create" && event.state_key === "") { + kstate["m.room.create/outer"] = event + } } return kstate } @@ -81,12 +94,21 @@ function diffKState(actual, target) { if (key === "m.room.power_levels/") { // Special handling for power levels, we want to deep merge the actual and target into the final state. if (!(key in actual)) throw new Error(`want to apply a power levels diff, but original power level data is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) - const temp = mixin({}, actual[key], target[key]) - if (!isDeepStrictEqual(actual[key], temp)) { + const mixedTarget = mixin({}, actual[key], target[key]) + if (!isDeepStrictEqual(actual[key], mixedTarget)) { // they differ. use the newly prepared object as the diff. - diff[key] = temp + // if the diff includes users, it needs to be cleaned wrt room version 12 + if (target[key].users && Object.keys(target[key].users).length > 0) { + if (!("m.room.create/" in actual)) throw new Error(`want to apply a power levels diff, but original m.room.create/ is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) + if (!("m.room.create/outer" in actual)) throw new Error(`want to apply a power levels diff, but original m.room.create/outer is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) + utils.removeCreatorsFromPowerLevels(actual["m.room.create/outer"], mixedTarget) + } + diff[key] = mixedTarget } + } else if (key === "m.room.create/") { + // can't be modified - only for kstateToCreationContent + } else if (key in actual) { // diff if (!isDeepStrictEqual(actual[key], target[key])) { @@ -129,6 +151,7 @@ async function applyKStateDiffToRoom(roomID, kstate) { module.exports.kstateStripConditionals = kstateStripConditionals module.exports.kstateUploadMxc = kstateUploadMxc module.exports.kstateToState = kstateToState +module.exports.kstateToCreationContent = kstateToCreationContent module.exports.stateToKState = stateToKState module.exports.diffKState = diffKState module.exports.roomToKState = roomToKState diff --git a/src/types.d.ts b/src/types.d.ts index 8b8bae9b..3b0e5af9 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -174,7 +174,7 @@ export namespace Event { } export type M_Room_Create = { - additional_creators: string[] + additional_creators?: string[] "m.federate"?: boolean room_version: string type?: string @@ -356,6 +356,11 @@ export namespace Event { }> & { redacts: string } + + export type M_Room_Tombstone = { + body: string + replacement_room: string + } } export namespace R { diff --git a/test/data.js b/test/data.js index b9e9da41..387ad6a8 100644 --- a/test/data.js +++ b/test/data.js @@ -101,6 +101,7 @@ module.exports = { }, room: { general: { + "m.room.create/": {additional_creators: ["@test_auto_invite:example.org"]}, "m.room.name/": {name: "main"}, "m.room.topic/": {topic: "#collective-unconscious | https://docs.google.com/document/d/blah/edit | I spread, pipe, and whip because it is my will. :headstone:\n\nChannel ID: 112760669178241024\nGuild ID: 112760669178241024"}, "m.room.guest_access/": {guest_access: "can_join"}, @@ -126,7 +127,7 @@ module.exports = { "m.room.redaction": 0 }, users: { - "@test_auto_invite:example.org": 100 + "@test_auto_invite:example.org": 150 }, notifications: { room: 0 From 231b26113e863fe8dc9e68ad53cddac82c3de786 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Dec 2025 12:17:34 +1300 Subject: [PATCH 047/153] Fix topic diffing from original creation --- src/matrix/kstate.js | 8 +++++++ src/matrix/kstate.test.js | 50 +++++++++++++++++++++++---------------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index d1318896..b0fe9479 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -109,6 +109,14 @@ function diffKState(actual, target) { } else if (key === "m.room.create/") { // can't be modified - only for kstateToCreationContent + } else if (key === "m.room.topic/") { + // synapse generates different m.room.topic events on original creation + // https://github.com/element-hq/synapse/blob/0f2b29511fd88d1dc2278f41fd6e4e2f2989fcb7/synapse/handlers/room.py#L1729 + // diff the `topic` to determine change + if (!(key in actual) || actual[key].topic !== target[key].topic) { + diff[key] = target[key] + } + } else if (key in actual) { // diff if (!isDeepStrictEqual(actual[key], target[key])) { diff --git a/src/matrix/kstate.test.js b/src/matrix/kstate.test.js index 1b67ad51..ff65e9c6 100644 --- a/src/matrix/kstate.test.js +++ b/src/matrix/kstate.test.js @@ -235,30 +235,38 @@ test("diffKState: kstate keys must contain a slash separator", t => { t.pass() }) -test("diffKState: don't add hide_ui when not present", t => { - test("diffKState: detects new properties", t => { - t.deepEqual( - diffKState({ - }, { - "chat.schildi.hide_ui/read_receipts/": {} - }), - { +test("diffKState: topic does not change if the topic key has not changed", t => { + t.deepEqual(diffKState({ + "m.room.topic/": { + topic: "hello", + "m.topic": { + "m.text": "hello" } - ) - }) + } + }, { + "m.room.topic/": { + topic: "hello" + } + }), + {}) }) -test("diffKState: overwriten hide_ui when present", t => { - test("diffKState: detects new properties", t => { - t.deepEqual( - diffKState({ - "chat.schildi.hide_ui/read_receipts/": {hidden: true} - }, { - "chat.schildi.hide_ui/read_receipts/": {} - }), - { - "chat.schildi.hide_ui/read_receipts/": {} +test("diffKState: topic changes if the topic key has changed", t => { + t.deepEqual(diffKState({ + "m.room.topic/": { + topic: "hello", + "m.topic": { + "m.text": "hello" } - ) + } + }, { + "m.room.topic/": { + topic: "hello you" + } + }), + { + "m.room.topic/": { + topic: "hello you" + } }) }) From 04d26026f5efdb0583de5ef3e3077869e485e677 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Dec 2025 12:37:43 +1300 Subject: [PATCH 048/153] Pre-upload icons before diffing --- src/d2m/actions/create-room.js | 1 + src/d2m/actions/create-space.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index a1d99407..61276282 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -401,6 +401,7 @@ async function _syncRoom(channelID, shouldActuallySync) { console.log(`[room sync] to matrix: ${channel.name}`) const {spaceID, channelKState} = await channelToKState(channel, guild, {api}) // calling this in both branches because we don't want to calculate this if not syncing + await ks.kstateUploadMxc(channelKState) // pre-upload icons before diffing // sync channel state to room const roomKState = await ks.roomToKState(roomID) diff --git a/src/d2m/actions/create-space.js b/src/d2m/actions/create-space.js index 1fb1911f..89e0f084 100644 --- a/src/d2m/actions/create-space.js +++ b/src/d2m/actions/create-space.js @@ -124,6 +124,8 @@ async function _syncSpace(guild, shouldActuallySync) { console.log(`[space sync] to matrix: ${guild.name}`) const guildKState = await guildToKState(guild, privacy_level) // calling this in both branches because we don't want to calculate this if not syncing + ks.kstateStripConditionals(guildKState) // pre-upload icons before diffing + await ks.kstateUploadMxc(guildKState) // sync guild state to space const spaceKState = await ks.roomToKState(spaceID) @@ -185,6 +187,8 @@ async function syncSpaceFully(guildID) { console.log(`[space sync] to matrix: ${guild.name}`) const guildKState = await guildToKState(guild, privacy_level) + ks.kstateStripConditionals(guildKState) // pre-upload icons before diffing + await ks.kstateUploadMxc(guildKState) // sync guild state to space const spaceKState = await ks.roomToKState(spaceID) From 694379f6590b1300051863796335b2e61f429df6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Dec 2025 21:32:17 +1300 Subject: [PATCH 049/153] Consider creators when calculating via servers --- src/m2d/converters/utils.js | 69 ++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/src/m2d/converters/utils.js b/src/m2d/converters/utils.js index 59035fef..7335c8ea 100644 --- a/src/m2d/converters/utils.js +++ b/src/m2d/converters/utils.js @@ -13,6 +13,8 @@ let hasher = null // @ts-ignore require("xxhash-wasm")().then(h => hasher = h) +const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` + const BLOCK_ELEMENTS = [ "ADDRESS", "ARTICLE", "ASIDE", "AUDIO", "BLOCKQUOTE", "BODY", "CANVAS", "CENTER", "DD", "DETAILS", "DIR", "DIV", "DL", "DT", "FIELDSET", "FIGCAPTION", "FIGURE", @@ -127,7 +129,7 @@ class MatrixStringBuilder { * https://spec.matrix.org/v1.9/appendices/#routing * https://gitdab.com/cadence/out-of-your-element/issues/11 * @param {string} roomID - * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api + * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api */ async function getViaServers(roomID, api) { const candidates = [] @@ -136,25 +138,17 @@ async function getViaServers(roomID, api) { candidates.push(reg.ooye.server_name) // Candidate 1: Highest joined non-sim non-bot power level user in the room // https://github.com/matrix-org/matrix-react-sdk/blob/552c65db98b59406fb49562e537a2721c8505517/src/utils/permalinks/Permalinks.ts#L172 - try { - /** @type {{users?: {[mxid: string]: number}}} */ - const powerLevels = await api.getStateEvent(roomID, "m.room.power_levels", "") - if (powerLevels.users) { - const sorted = Object.entries(powerLevels.users).sort((a, b) => b[1] - a[1]) // Highest... - for (const power of sorted) { - const mxid = power[0] - if (!(mxid in joined)) continue // joined... - if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot... - const match = mxid.match(/:(.*)/) - assert(match) - if (!candidates.includes(match[1])) { - candidates.push(match[1]) - break - } - } - } - } catch (e) { - // power levels event not found + const {allCreators, powerLevels} = await getEffectivePower(roomID, [bot], api) + const sorted = allCreators.concat(Object.entries(powerLevels.users ?? {}).sort((a, b) => b[1] - a[1]).map(([mxid]) => mxid)) // Highest... + for (const power of sorted) { + const mxid = power[0] + if (!(mxid in joined)) continue // joined... + if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot... + const match = mxid.match(/:(.*)/) + assert(match) + if (candidates.includes(match[1])) continue // from a different server + candidates.push(match[1]) + break } // Candidates 2-3: Most popular servers in the room /** @type {Map} */ @@ -194,7 +188,7 @@ async function getViaServers(roomID, api) { * https://spec.matrix.org/v1.9/appendices/#routing * https://gitdab.com/cadence/out-of-your-element/issues/11 * @param {string} roomID - * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api + * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api * @returns {Promise} */ async function getViaServersQuery(roomID, api) { @@ -275,6 +269,38 @@ function removeCreatorsFromPowerLevels(roomCreateOuter, powerLevels) { return powerLevels } +/** + * @template {string} T + * @param {string} roomID + * @param {T[]} mxids + * @param {{[K in "getStateEvent" | "getStateEventOuter"]: import("../../matrix/api")[K]}} api + * @returns {Promise<{powers: Record, allCreators: string[], tombstone: number, roomCreate: Ty.Event.StateOuter, powerLevels: Ty.Event.M_Power_Levels}>} + */ +async function getEffectivePower(roomID, mxids, api) { + /** @type {[Ty.Event.StateOuter, Ty.Event.M_Power_Levels]} */ + const [roomCreate, powerLevels] = await Promise.all([ + api.getStateEventOuter(roomID, "m.room.create", ""), + api.getStateEvent(roomID, "m.room.power_levels", "") + ]) + const allCreators = + ( roomHasAtLeastVersion(roomCreate.content.room_version, 12) ? (roomCreate.content.additional_creators ?? []).concat(roomCreate.sender) + : []) + const tombstone = + ( roomHasAtLeastVersion(roomCreate.content.room_version, 12) ? powerLevels.events?.["m.room.tombstone"] ?? 150 + : powerLevels.events?.["m.room.tombstone"] ?? powerLevels.state_default ?? 50) + /** @type {Record} */ // @ts-ignore + const powers = {} + for (const mxid of mxids) { + powers[mxid] = + ( roomHasAtLeastVersion(roomCreate.content.room_version, 12) && allCreators.includes(mxid) ? Infinity + : powerLevels.users?.[mxid] + ?? powerLevels.users_default + ?? 0) + } + return {powers, allCreators, tombstone, roomCreate, powerLevels} +} + +module.exports.bot = bot module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord module.exports.getPublicUrlForMxc = getPublicUrlForMxc @@ -284,3 +310,4 @@ module.exports.getViaServers = getViaServers module.exports.getViaServersQuery = getViaServersQuery module.exports.roomHasAtLeastVersion = roomHasAtLeastVersion module.exports.removeCreatorsFromPowerLevels = removeCreatorsFromPowerLevels +module.exports.getEffectivePower = getEffectivePower From 5a401a187d089d762f4972cef7ee8c57a621b90b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Dec 2025 21:47:22 +1300 Subject: [PATCH 050/153] Suppress embed if an auxiliary m.notice is deleted --- src/m2d/actions/redact.js | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/m2d/actions/redact.js b/src/m2d/actions/redact.js index f980a540..9b26e8e4 100644 --- a/src/m2d/actions/redact.js +++ b/src/m2d/actions/redact.js @@ -1,6 +1,6 @@ // @ts-check -const assert = require("assert").strict +const DiscordTypes = require("discord-api-types/v10") const Ty = require("../../types") const passthrough = require("../../passthrough") @@ -22,6 +22,19 @@ async function deleteMessage(event) { db.prepare("DELETE FROM message_room WHERE message_id = ?").run(rows[0].message_id) } +/** + * @param {Ty.Event.Outer_M_Room_Redaction} event + */ +async function suppressEmbeds(event) { + const rows = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("reference_channel_id", "message_id").where({event_id: event.redacts}).all() + if (!rows.length) return + db.prepare("DELETE FROM event_message WHERE event_id = ?").run(event.redacts) + for (const row of rows) { + await discord.snow.channel.editMessage(row.reference_channel_id, row.message_id, {flags: DiscordTypes.MessageFlags.SuppressEmbeds}) + } +} + /** * @param {Ty.Event.Outer_M_Room_Redaction} event */ @@ -39,7 +52,12 @@ async function removeReaction(event) { * @param {Ty.Event.Outer_M_Room_Redaction} event */ async function handle(event) { - await deleteMessage(event) + const row = select("event_message", ["event_type", "event_subtype", "part"], {event_id: event.redacts}).get() + if (row && row.event_type === "m.room.message" && row.event_subtype === "m.notice" && row.part === 1) { + await suppressEmbeds(event) + } else { + await deleteMessage(event) + } await removeReaction(event) } From 17251c61d5202b8026fdb27b5a3d779f841d3742 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 24 Dec 2025 17:13:23 +1300 Subject: [PATCH 051/153] Suppress link embeds where applicable * If the guild has disabled EMBED_LINKS for default users * If the user puts < > around the link --- src/m2d/converters/event-to-message.js | 44 ++++++- src/m2d/converters/event-to-message.test.js | 135 ++++++++++++++++++++ 2 files changed, 176 insertions(+), 3 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 7fd7b8a3..3fe8776c 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -136,10 +136,11 @@ turndownService.addRule("inlineLink", { if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}` if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>` const href = node.getAttribute("href") + const suppressedHref = node.hasAttribute("data-suppress") ? "<" + href + ">" : href content = content.replace(/ @.*/, "") - if (href === content) return href + if (href === content) return suppressedHref if (decodeURIComponent(href).startsWith("https://matrix.to/#/@") && content[0] !== "@") content = "@" + content - return "[" + content + "](" + href + ")" + return "[" + content + "](" + suppressedHref + ")" } }) @@ -860,6 +861,21 @@ async function eventToMessage(event, guild, di) { pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")}) } } + // Suppress link embeds + if (node.nodeType === 1 && node.tagName === "A") { + // Suppress if sender tried to add angle brackets + const inBody = event.content.body.indexOf(node.getAttribute("href")) + let shouldSuppress = inBody !== -1 && event.content.body[inBody-1] === "<" + if (!shouldSuppress && guild?.roles) { + // Suppress if regular users don't have permission + const permissions = dUtils.getPermissions([], guild.roles) + const canEmbedLinks = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.EmbedLinks) + shouldSuppress = !canEmbedLinks + } + if (shouldSuppress) { + node.setAttribute("data-suppress", "") + } + } await forEachNode(node.firstChild) } } @@ -901,7 +917,29 @@ async function eventToMessage(event, guild, di) { } content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible - content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews + + let offset = 0 + for (const match of [...content.matchAll(/\bhttps?:\/\/[^ )>]*/g)]) { + assert(typeof match.index === "number") + + // Respect sender's angle brackets + const alreadySuppressed = content[match.index-1+offset] === "<" && content[match.index+match.length+offset] === ">" + console.error(content, match.index-1+offset, content[match.index-1+offset]) + if (alreadySuppressed) continue + // Put < > around any surviving matrix.to links + let shouldSuppress = !!match[0].match(/^https?:\/\/matrix\.to\//) + if (!shouldSuppress && guild?.roles) { + // Suppress if regular users don't have permission + const permissions = dUtils.getPermissions([], guild.roles) + const canEmbedLinks = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.EmbedLinks) + shouldSuppress = !canEmbedLinks + } + + if (shouldSuppress) { + content = content.slice(0, match.index + offset) + "<" + match[0] + ">" + content.slice(match.index + match[0].length + offset) + offset += 2 + } + } const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di) if (result) { diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 439e07fb..6665e873 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -1,6 +1,7 @@ const assert = require("assert").strict const fs = require("fs") const {test} = require("supertape") +const DiscordTypes = require("discord-api-types/v10") const {eventToMessage} = require("./event-to-message") const {convertImageStream} = require("./emoji-sheet") const data = require("../../../test/data") @@ -302,6 +303,140 @@ test("event2message: markdown in link text does not attempt to be escaped becaus ) }) +test("event2message: links are escaped if the guild does not have embed links permission (formatted body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "posting one of my favourite songs recently (starts at timestamp) https://youtu.be/RhV2X7WQMPA?t=364", + format: "org.matrix.custom.html", + formatted_body: `posting one of my favourite songs recently (starts at timestamp) https://youtu.be/RhV2X7WQMPA?t=364`, + msgtype: "m.text" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + }, { + id: "123", + roles: [{ + id: "123", + name: "@everyone", + permissions: DiscordTypes.PermissionFlagsBits.SendMessages + }] + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "posting one of my favourite songs recently (starts at timestamp) ", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: links are escaped if the guild does not have embed links permission (plaintext body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "posting one of my favourite songs recently (starts at timestamp) https://youtu.be/RhV2X7WQMPA?t=364", + msgtype: "m.text" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + }, { + id: "123", + roles: [{ + id: "123", + name: "@everyone", + permissions: DiscordTypes.PermissionFlagsBits.SendMessages + }] + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "posting one of my favourite songs recently (starts at timestamp) ", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: links retain angle brackets (formatted body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "posting one of my favourite songs recently (starts at timestamp) ", + format: "org.matrix.custom.html", + formatted_body: `posting one of my favourite songs recently (starts at timestamp) https://youtu.be/RhV2X7WQMPA?t=364`, + msgtype: "m.text" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "posting one of my favourite songs recently (starts at timestamp) ", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: links retain angle brackets (plaintext body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "posting one of my favourite songs recently (starts at timestamp) ", + msgtype: "m.text" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "posting one of my favourite songs recently (starts at timestamp) ", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + test("event2message: basic html is converted to markdown", async t => { t.deepEqual( await eventToMessage({ From 092a4cf7b052d81950b85aac2f91c7a4158fc98f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 6 Jan 2026 16:07:23 +1300 Subject: [PATCH 052/153] Add traefik documentation --- docs/get-started.md | 6 ++ docs/third-party/reverse-proxy-traefik.md | 113 ++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 docs/third-party/reverse-proxy-traefik.md diff --git a/docs/get-started.md b/docs/get-started.md index ae31f5f7..a819b478 100644 --- a/docs/get-started.md +++ b/docs/get-started.md @@ -106,3 +106,9 @@ bridge.cadence.moe { reverse_proxy 127.0.0.1:6693 } ``` + +## Example reverse proxy for traefik + +Note: Out Of Your Element has no official Docker support. This guide is for using traefik when OOYE is ***not*** in a container. + +See [third-party/reverse-proxy-traefik.md](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/third-party/reverse-proxy-traefik.md) diff --git a/docs/third-party/reverse-proxy-traefik.md b/docs/third-party/reverse-proxy-traefik.md new file mode 100644 index 00000000..8b06a9be --- /dev/null +++ b/docs/third-party/reverse-proxy-traefik.md @@ -0,0 +1,113 @@ +> This guide was written by @bgtlover:stealthy.club, a community contributor. The author of Out Of Your Element hopes it will be useful, but cannot say whether the information is accurate or complete. + +## Example reverse proxy configuration with traefik + +Note: This guide describes setting up the reverse proxy configuration when OOYE is ***not*** in a Docker container. + +Because traefik is generally used in Docker, this guide assumes the user already has it configured properly. However, given that Docker is very complex and the smallest mistakes can cascade in catastrophic, not immediately observable, and unpredictable ways, a fairly complete setup will be reproduced. Therefore, system administrators are advised to diff this sample setup against theirs rather than copy it wholesale. + +### Note on variable substitution + +Variables will be denoted as `{{var}}`. This syntax has been chosen because that's also how YAML substitution works. The values that fit each variable will be explained after the code block containing the placeholder. + +### Base compose configuration for traefik + +This file defines the traefik service stack. It's responsible for mounting volumes correctly, declaring ports that should be opened on the host side, and the external traefik network (created manually). + +In compose.yml, put the following: + +```yaml +services: + traefik: + image: "traefik:latest" + restart: always + command: + - "--configFile=/etc/traefik/static_config.yml" + ports: + - "80:80" #http + - "443:443" #https + networks: + - traefik + volumes: + - ./letsencrypt:/letsencrypt + - /etc/localtime:/etc/localtime:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./static_config.yml:/etc/traefik/static_config.yml + - ./config:/etc/traefik/config +networks: + traefik: + external: true +``` + +### Static traefik configuration + +The static traefik configuration is used to define base traefik behavior, for example entry points, access and runtime logs, a file or directory for per-service configuration, etc. + +In static_config.yml, put the following: + +```yaml +api: + dashboard: true + +providers: + docker: + endpoint: "unix:///var/run/docker.sock" + exposedByDefault: false + network: "traefik" + file: + directory: /etc/traefik/config/ + watch: true + +entryPoints: + web-secure: + address: ":443" + asDefault: true + http3: {} + http: + tls: + certResolver: default + web: + address: ":80" + http: + redirections: + entryPoint: + to: web-secure + +certificatesResolvers: + default: + acme: + email: {{email}} + storage: "/letsencrypt/acme.json" + tlsChallenge: {} + +``` + +Replace `{{email}}` with a valid email address. + +### Out of your element traefik dynamic configuration + +Traefik's dynamic configuration files configure proxy behaviors on a per-application level. + +In config/out-of-your-element.yml, put the following: + +```yaml +http: + routers: + out-of-your-element: + rule: Host(`bridge.stealthy.club`) + service: out-of-your-element-service + services: + out-of-your-element-service: + loadBalancer: + servers: + - url: "http://{{ip}}:{{port}}" + +``` + +The `{{port}}` is 6693 unless you changed it during Out Of Your Element's first time setup. + +Replace `{{ip}}` with the ***external*** IP of your server. + +Make sure the port is allowed through your firewall if applicable. + +For context, the external IP is required because of Docker networking. Because Docker modifies the host-side iptables firewall and creates virtual interfaces for its networks, and because the networking inside containers is configured such that localhost points to the IP of the container instead of the actual host, placing localhost in the url field above would make the traefik container establish an HTTP connection to itself, which would cause a bad gateway error. From 55e0e5dfa1367a16badde4db71160aa73dd6e5c8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 7 Jan 2026 02:43:20 +1300 Subject: [PATCH 053/153] Room version 12 and room upgrades --- package-lock.json | 94 +++++---- package.json | 2 +- src/d2m/actions/create-room.js | 15 +- src/d2m/actions/create-room.test.js | 113 +++++----- .../message-to-event.embeds.test.js | 26 +-- src/d2m/converters/message-to-event.test.js | 89 ++------ .../converters/thread-to-announcement.test.js | 9 +- src/db/migrations/0028-add-room-upgrade.sql | 10 + src/db/migrations/0029-force-guild-ids.js | 59 ++++++ src/db/migrations/0030-require-guild-id.sql | 44 ++++ src/db/orm-defs.d.ts | 5 + src/discord/interactions/permissions.js | 22 +- src/discord/interactions/permissions.test.js | 75 ++++--- src/m2d/converters/event-to-message.js | 6 +- src/m2d/converters/event-to-message.test.js | 49 ++++- src/m2d/converters/utils.js | 8 +- src/m2d/converters/utils.test.js | 96 +++++---- src/m2d/event-dispatcher.js | 45 ++-- src/matrix/api.js | 4 +- src/matrix/kstate.js | 2 +- src/matrix/matrix-command-handler.js | 9 +- src/matrix/mreq.js | 9 +- src/matrix/room-upgrade.js | 94 +++++++++ src/types.d.ts | 15 -- src/web/routes/link.js | 25 +-- src/web/routes/link.test.js | 194 ++++++++---------- test/ooye-test-data.sql | 30 +-- 27 files changed, 666 insertions(+), 483 deletions(-) create mode 100644 src/db/migrations/0028-add-room-upgrade.sql create mode 100644 src/db/migrations/0029-force-guild-ids.js create mode 100644 src/db/migrations/0030-require-guild-id.sql create mode 100644 src/matrix/room-upgrade.js diff --git a/package-lock.json b/package-lock.json index 65abcf45..d58d72b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.14.0", - "discord-api-types": "^0.38.31", + "discord-api-types": "^0.38.36", "domino": "^2.1.6", "enquirer": "^2.4.1", "entities": "^5.0.0", @@ -53,6 +53,20 @@ "node": ">=20" } }, + "../extended-errors/enhance-errors": { + "version": "1.0.0", + "extraneous": true, + "license": "UNLICENSED", + "dependencies": { + "ts-expose-internals": "^5.6.3", + "ts-patch": "^3.3.0", + "typescript": "^5.9.3" + }, + "devDependencies": { + "@types/node": "^22.19.1", + "ts-node": "^10.9.2" + } + }, "../tap-dot": { "name": "@cloudrac3r/tap-dot", "version": "2.0.0", @@ -67,27 +81,30 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.24.8", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz", - "integrity": "sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ==", + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", - "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.25.6", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.25.6.tgz", - "integrity": "sha512-trGdfBdbD0l1ZPmcJ83eNxB9rbEax4ALFTF7fN386TMYbeCQbyme5cOEXQhbGXKebwGaB/J52w1mrklMcbgy6Q==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", + "license": "MIT", "dependencies": { - "@babel/types": "^7.25.6" + "@babel/types": "^7.28.5" }, "bin": { "parser": "bin/babel-parser.js" @@ -97,13 +114,13 @@ } }, "node_modules/@babel/types": { - "version": "7.25.6", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.25.6.tgz", - "integrity": "sha512-/l42B1qxpG6RdfYf343Uw1vmDjeNhneUXtzhojE7pDgfpEypmRhI6j1kr17XCVv4Cgl9HdAiQY2x0GwKm7rWCw==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", + "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.24.8", - "@babel/helper-validator-identifier": "^7.24.7", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" }, "engines": { "node": ">=6.9.0" @@ -355,9 +372,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.0.tgz", - "integrity": "sha512-oAYoQnCYaQZKVS53Fq23ceWMRxq5EhQsE0x0RdQ55jT7wagMu5k+fS39v1fiSLrtrLQlXwVINenqhLMtTrV/1Q==", + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", + "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==", "license": "MIT", "optional": true, "dependencies": { @@ -983,16 +1000,18 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "dev": true + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.19", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", - "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "dev": true, + "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" @@ -1702,9 +1721,9 @@ } }, "node_modules/discord-api-types": { - "version": "0.38.33", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.33.tgz", - "integrity": "sha512-oau1V7OzrNX8yNi+DfQpoLZCNCv7cTFmvPKwHfMrA/tewsO6iQKrMTzA7pa3iBSj0fED6NlklJ/1B/cC1kI08Q==", + "version": "0.38.36", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.36.tgz", + "integrity": "sha512-qrbUbjjwtyeBg5HsAlm1C859epfOyiLjPqAOzkdWlCNsZCWJrertnETF/NwM8H+waMFU58xGSc5eXUfXah+WTQ==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" @@ -2958,10 +2977,11 @@ } }, "node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", "dev": true, + "license": "MIT", "dependencies": { "ansi-regex": "^6.0.1" }, @@ -3176,14 +3196,6 @@ "dev": true, "license": "MIT" }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "engines": { - "node": ">=4" - } - }, "node_modules/token-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/token-stream/-/token-stream-1.0.0.tgz", diff --git a/package.json b/package.json index b7bce862..4a80f227 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.14.0", - "discord-api-types": "^0.38.31", + "discord-api-types": "^0.38.36", "domino": "^2.1.6", "enquirer": "^2.4.1", "entities": "^5.0.0", diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 61276282..c5fdd603 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -77,7 +77,7 @@ function convertNameAndTopic(channel, guild, customName) { * Async because it may create the guild and/or upload the guild icon to mxc. * @param {DiscordTypes.APIGuildTextChannel | DiscordTypes.APIThreadChannel} channel * @param {DiscordTypes.APIGuild} guild - * @param {{api: {getStateEvent: typeof api.getStateEvent}}} di simple-as-nails dependency injection for the matrix API + * @param {{api: {getStateEvent: typeof api.getStateEvent, getStateEventOuter: typeof api.getStateEventOuter}}} di simple-as-nails dependency injection for the matrix API */ async function channelToKState(channel, guild, di) { // @ts-ignore @@ -126,16 +126,17 @@ async function channelToKState(channel, guild, di) { const everyoneCanSend = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendMessages) const everyoneCanMentionEveryone = dUtils.hasAllPermissions(everyonePermissions, ["MentionEveryone"]) - /** @type {Ty.Event.M_Power_Levels} */ - const spacePowerEvent = await di.api.getStateEvent(guildSpaceID, "m.room.power_levels", "") - const spacePower = spacePowerEvent.users + const spacePowerDetails = await mUtils.getEffectivePower(guildSpaceID, [], di.api) + const spaceCreatorsAndFounders = spacePowerDetails.allCreators + .concat(Object.entries(spacePowerDetails.powerLevels.users ?? {}).filter(([, power]) => power >= spacePowerDetails.tombstone).map(([mxid]) => mxid)) const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all() const globalAdminPower = globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {}) - const additionalCreators = select("member_power", "mxid", {room_id: "*"}, "AND power_level > 100").pluck().all() + const additionalCreators = select("member_power", "mxid", {room_id: "*"}, "AND power_level > 100").pluck().all().concat(spaceCreatorsAndFounders) const creationContent = {} creationContent.additional_creators = additionalCreators + if (channel.type === DiscordTypes.ChannelType.GuildForum) creationContent.type = "m.space" /** @type {any} */ @@ -162,10 +163,10 @@ async function channelToKState(channel, guild, di) { notifications: { room: everyoneCanMentionEveryone ? 0 : 20 }, - users: {...spacePower, ...globalAdminPower} + users: {...spacePowerDetails.powerLevels.users, ...globalAdminPower} }, [`uk.half-shot.bridge/moe.cadence.ooye://discord/${guild.id}/${channel.id}`]: { - bridgebot: `@${reg.sender_localpart}:${reg.ooye.server_name}`, + bridgebot: mUtils.bot, protocol: { id: "discord", displayname: "Discord" diff --git a/src/d2m/actions/create-room.test.js b/src/d2m/actions/create-room.test.js index ca09f735..36fccba4 100644 --- a/src/d2m/actions/create-room.test.js +++ b/src/d2m/actions/create-room.test.js @@ -9,19 +9,44 @@ const testData = require("../../../test/data") const passthrough = require("../../passthrough") const {db} = passthrough +function mockAPI(t) { + let called = 0 + return { + getCalled() { + return called + }, + async getStateEvent(roomID, type, key) { // getting power levels from space to apply to room + called++ + t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return {users: {"@example:matrix.org": 50}, events: {"m.room.tombstone": 100}} + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + content: { + room_version: "11" + }, + event_id: "$create", + origin_server_ts: 0, + room_id: "!jjmvBegULiLucuWEHU:cadence.moe", + sender: "@_ooye_bot:cadence.moe" + } + } + } +} test("channel2room: discoverable privacy room", async t => { - let called = 0 - async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return {users: {"@example:matrix.org": 50}} - } + const api = mockAPI(t) db.prepare("UPDATE guild_space SET privacy_level = 2").run() t.deepEqual( - kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)), + kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api}).then(x => x.channelKState)), Object.assign({}, testData.room.general, { "m.room.guest_access/": {guest_access: "forbidden"}, "m.room.join_rules/": {join_rule: "public"}, @@ -29,58 +54,37 @@ test("channel2room: discoverable privacy room", async t => { "m.room.power_levels/": mixin({users: {"@example:matrix.org": 50}}, testData.room.general["m.room.power_levels/"]) }) ) - t.equal(called, 1) + t.equal(api.getCalled(), 2) }) test("channel2room: linkable privacy room", async t => { - let called = 0 - async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return {users: {"@example:matrix.org": 50}} - } + const api = mockAPI(t) db.prepare("UPDATE guild_space SET privacy_level = 1").run() t.deepEqual( - kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)), + kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api}).then(x => x.channelKState)), Object.assign({}, testData.room.general, { "m.room.guest_access/": {guest_access: "forbidden"}, "m.room.join_rules/": {join_rule: "public"}, "m.room.power_levels/": mixin({users: {"@example:matrix.org": 50}}, testData.room.general["m.room.power_levels/"]) }) ) - t.equal(called, 1) + t.equal(api.getCalled(), 2) }) test("channel2room: invite-only privacy room", async t => { - let called = 0 - async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return {users: {"@example:matrix.org": 50}} - } + const api = mockAPI(t) db.prepare("UPDATE guild_space SET privacy_level = 0").run() t.deepEqual( - kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)), + kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api}).then(x => x.channelKState)), Object.assign({}, testData.room.general, { "m.room.power_levels/": mixin({users: {"@example:matrix.org": 50}}, testData.room.general["m.room.power_levels/"]) }) ) - t.equal(called, 1) + t.equal(api.getCalled(), 2) }) test("channel2room: room where limited people can mention everyone", async t => { - let called = 0 - async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return {users: {"@example:matrix.org": 50}} - } + const api = mockAPI(t) const limitedGuild = mixin({}, testData.guild.general) limitedGuild.roles[0].permissions = (BigInt(limitedGuild.roles[0].permissions) - 131072n).toString() const limitedRoom = mixin({}, testData.room.general, {"m.room.power_levels/": { @@ -88,41 +92,27 @@ test("channel2room: room where limited people can mention everyone", async t => users: {"@example:matrix.org": 50} }}) t.deepEqual( - kstateStripConditionals(await channelToKState(testData.channel.general, limitedGuild, {api: {getStateEvent}}).then(x => x.channelKState)), + kstateStripConditionals(await channelToKState(testData.channel.general, limitedGuild, {api}).then(x => x.channelKState)), limitedRoom ) - t.equal(called, 1) + t.equal(api.getCalled(), 2) }) test("channel2room: matrix room that already has a custom topic set", async t => { - let called = 0 - async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return {} - } + const api = mockAPI(t) db.prepare("UPDATE channel_room SET custom_topic = 1 WHERE channel_id = ?").run(testData.channel.general.id) - const expected = mixin({}, testData.room.general, {"m.room.power_levels/": {notifications: {room: 20}}}) + const expected = mixin({}, testData.room.general, {"m.room.power_levels/": {notifications: {room: 20}, users: {"@example:matrix.org": 50}}}) // @ts-ignore delete expected["m.room.topic/"] t.deepEqual( - kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)), + kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api}).then(x => x.channelKState)), expected ) - t.equal(called, 1) + t.equal(api.getCalled(), 2) }) test("channel2room: read-only discord channel", async t => { - let called = 0 - async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return {} - } + const api = mockAPI(t) const expected = { "m.room.create/": { additional_creators: ["@test_auto_invite:example.org"], @@ -164,6 +154,7 @@ test("channel2room: read-only discord channel", async t => { }, users: { "@test_auto_invite:example.org": 150, + "@example:matrix.org": 50 }, }, "m.space.parent/!jjmvBegULiLucuWEHU:cadence.moe": { @@ -193,10 +184,10 @@ test("channel2room: read-only discord channel", async t => { } } t.deepEqual( - kstateStripConditionals(await channelToKState(testData.channel.updates, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)), + kstateStripConditionals(await channelToKState(testData.channel.updates, testData.guild.general, {api}).then(x => x.channelKState)), expected ) - t.equal(called, 1) + t.equal(api.getCalled(), 2) }) test("convertNameAndTopic: custom name and topic", t => { diff --git a/src/d2m/converters/message-to-event.embeds.test.js b/src/d2m/converters/message-to-event.embeds.test.js index ed165c64..cddd4275 100644 --- a/src/d2m/converters/message-to-event.embeds.test.js +++ b/src/d2m/converters/message-to-event.embeds.test.js @@ -1,6 +1,7 @@ const {test} = require("supertape") const {messageToEvent} = require("./message-to-event") const data = require("../../../test/data") +const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") const {db} = require("../../passthrough") test("message2event embeds: nothing but a field", async t => { @@ -86,17 +87,7 @@ test("message2event embeds: blockquote in embed", async t => { let called = 0 const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general, {}, { api: { - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { called++ t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe") @@ -124,7 +115,7 @@ test("message2event embeds: blockquote in embed", async t => { formatted_body: "

⏺️ minimus

reply draft

The following is a message composed via consensus of the Stinker Council.

For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.

Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.

There will be no further communication.

Go to Message

", "m.mentions": {} }]) - t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each") + t.equal(called, 1, "should call getJoinedMembers once") }) test("message2event embeds: crazy html is all escaped", async t => { @@ -343,16 +334,7 @@ test("message2event embeds: tenor gif should show a video link without a provide test("message2event embeds: if discord creates an embed preview for a discord channel link, don't copy that embed", async t => { const events = await messageToEvent(data.message_with_embeds.discord_server_included_punctuation_bad_discord, data.guild.general, {}, { api: { - async getStateEvent(roomID, type, key) { - t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe") return { diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index cae88b3e..4b213e49 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -2,6 +2,7 @@ const {test} = require("supertape") const {messageToEvent} = require("./message-to-event") const {MatrixServerError} = require("../../matrix/mreq") const data = require("../../../test/data") +const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") const Ty = require("../../types") /** @@ -66,17 +67,7 @@ test("message2event: simple room mention", async t => { let called = 0 const events = await messageToEvent(data.message.simple_room_mention, data.guild.general, {}, { api: { - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!BnKuBPCvyfOkhcUjEu:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { called++ t.equal(roomID, "!BnKuBPCvyfOkhcUjEu:cadence.moe") @@ -97,24 +88,14 @@ test("message2event: simple room mention", async t => { format: "org.matrix.custom.html", formatted_body: '#worm-farm' }]) - t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each") + t.equal(called, 1, "should call getJoinedMembers") }) test("message2event: simple room link", async t => { let called = 0 const events = await messageToEvent(data.message.simple_room_link, data.guild.general, {}, { api: { - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!BnKuBPCvyfOkhcUjEu:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { called++ t.equal(roomID, "!BnKuBPCvyfOkhcUjEu:cadence.moe") @@ -135,24 +116,14 @@ test("message2event: simple room link", async t => { format: "org.matrix.custom.html", formatted_body: '#worm-farm' }]) - t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each") + t.equal(called, 1, "should call getJoinedMembers once") }) test("message2event: nicked room mention", async t => { let called = 0 const events = await messageToEvent(data.message.nicked_room_mention, data.guild.general, {}, { api: { - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { called++ t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") @@ -173,7 +144,7 @@ test("message2event: nicked room mention", async t => { format: "org.matrix.custom.html", formatted_body: '#main' }]) - t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each") + t.equal(called, 1, "should call getJoinedMembers once") }) test("message2event: unknown room mention", async t => { @@ -224,17 +195,7 @@ test("message2event: simple message link", async t => { let called = 0 const events = await messageToEvent(data.message.simple_message_link, data.guild.general, {}, { api: { - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { called++ t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") @@ -255,13 +216,14 @@ test("message2event: simple message link", async t => { format: "org.matrix.custom.html", formatted_body: 'https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg?via=cadence.moe&via=super.invalid' }]) - t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each") + t.equal(called, 1, "should call getJoinedMembers once") }) test("message2event: message link that OOYE doesn't know about", async t => { let called = 0 const events = await messageToEvent(data.message.message_link_to_before_ooye, data.guild.general, {}, { api: { + getEffectivePower: mockGetEffectivePower(), async getEventForTimestamp(roomID, ts) { called++ t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") @@ -270,17 +232,6 @@ test("message2event: message link that OOYE doesn't know about", async t => { origin_server_ts: 1613287812754 } }, - async getStateEvent(roomID, type, key) { // for ?via calculation - called++ - t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, async getJoinedMembers(roomID) { // for ?via calculation called++ t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") @@ -303,7 +254,7 @@ test("message2event: message link that OOYE doesn't know about", async t => { formatted_body: "Me: I'll scroll up to find a certain message I'll send
scrolls up and clicks message links for god knows how long
completely forgets what they were looking for and simply begins scrolling up to find some fun moments
stumbles upon: " + 'https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U?via=cadence.moe&via=matrix.org' }]) - t.equal(called, 3, "getEventForTimestamp, getStateEvent, and getJoinedMembers should be called once each") + t.equal(called, 2, "getEventForTimestamp and getJoinedMembers should be called once each") }) test("message2event: message timestamp failed to fetch", async t => { @@ -318,17 +269,7 @@ test("message2event: message timestamp failed to fetch", async t => { error: "Unable to find event from 1726762095974 in direction Direction.FORWARDS" }, {}) }, - async getStateEvent(roomID, type, key) { // for ?via calculation - called++ - t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { // for ?via calculation called++ t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") @@ -351,7 +292,7 @@ test("message2event: message timestamp failed to fetch", async t => { formatted_body: "Me: I'll scroll up to find a certain message I'll send
scrolls up and clicks message links for god knows how long
completely forgets what they were looking for and simply begins scrolling up to find some fun moments
stumbles upon: " + '[unknown event, timestamp resolution failed, in room: https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org]' }]) - t.equal(called, 3, "getEventForTimestamp, getStateEvent, and getJoinedMembers should be called once each") + t.equal(called, 2, "getEventForTimestamp and getJoinedMembers should be called once each") }) test("message2event: message link from another server", async t => { @@ -1136,6 +1077,7 @@ test("message2event: forwarded image", async t => { test("message2event: constructed forwarded message", async t => { const events = await messageToEvent(data.message.constructed_forwarded_message, {}, {}, { api: { + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers() { return { joined: { @@ -1194,6 +1136,7 @@ test("message2event: constructed forwarded message", async t => { test("message2event: constructed forwarded text", async t => { const events = await messageToEvent(data.message.constructed_forwarded_text, {}, {}, { api: { + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers() { return { joined: { @@ -1331,6 +1274,7 @@ test("message2event: vc invite event renders embed", async t => { test("message2event: vc invite event renders embed with room link", async t => { const events = await messageToEvent({content: "https://discord.gg/placeholder?event=1381174024801095751"}, {}, {}, { api: { + getEffectivePower: mockGetEffectivePower(), getJoinedMembers: async () => ({ joined: { "@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null}, @@ -1380,6 +1324,7 @@ test("message2event: channel links are converted even inside lists (parser post- + "\nThis list will probably change in the future" }, data.guild.general, {}, { api: { + getEffectivePower: mockGetEffectivePower(), getJoinedMembers(roomID) { called++ t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe") diff --git a/src/d2m/converters/thread-to-announcement.test.js b/src/d2m/converters/thread-to-announcement.test.js index 3d5d1ebc..8d011fd0 100644 --- a/src/d2m/converters/thread-to-announcement.test.js +++ b/src/d2m/converters/thread-to-announcement.test.js @@ -2,6 +2,7 @@ const {test} = require("supertape") const {threadToAnnouncement} = require("./thread-to-announcement") const data = require("../../../test/data") const Ty = require("../../types") +const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") /** * @param {string} roomID @@ -30,13 +31,7 @@ function mockGetEvent(t, roomID_in, eventID_in, outer) { } const viaApi = { - async getStateEvent(roomID, type, key) { - return { - users: { - "@_ooye_bot:cadence.moe": 100 - } - } - }, + getEffectivePower: mockGetEffectivePower(), async getJoinedMembers(roomID) { return { joined: { diff --git a/src/db/migrations/0028-add-room-upgrade.sql b/src/db/migrations/0028-add-room-upgrade.sql new file mode 100644 index 00000000..fed6f215 --- /dev/null +++ b/src/db/migrations/0028-add-room-upgrade.sql @@ -0,0 +1,10 @@ +BEGIN TRANSACTION; + +CREATE TABLE room_upgrade_pending ( + new_room_id TEXT NOT NULL, + old_room_id TEXT NOT NULL UNIQUE, + PRIMARY KEY (new_room_id), + FOREIGN KEY (old_room_id) REFERENCES channel_room (room_id) ON DELETE CASCADE +) WITHOUT ROWID; + +COMMIT; diff --git a/src/db/migrations/0029-force-guild-ids.js b/src/db/migrations/0029-force-guild-ids.js new file mode 100644 index 00000000..e5757838 --- /dev/null +++ b/src/db/migrations/0029-force-guild-ids.js @@ -0,0 +1,59 @@ +/* + a. If the bridge bot sim already has the correct ID: + - No rows updated. + + b. If the bridge bot sim has the wrong ID but there's no duplicate: + - One row updated. + + c. If the bridge bot sim has the wrong ID and there's a duplicate: + - One row updated (replaces an existing row). +*/ + +const {discord} = require("../../passthrough") + +const ones = "₀₁₂₃₄₅₆₇₈₉" +const tens = "0123456789" + +module.exports = async function(db) { + /** @type {{name: string, channel_id: string, thread_parent: string | null}[]} */ + const rows = db.prepare("SELECT name, channel_id, thread_parent FROM channel_room WHERE guild_id IS NULL").all() + + /** @type {Map} channel or thread ID -> guild ID */ + const cache = new Map() + + // Process channels + process.stdout.write(` loading metadata for ${rows.length} channels/threads... `) + for (let counter = 1; counter <= rows.length; counter++) { + process.stdout.write(String(counter).at(-1) === "0" ? tens[(counter/10)%10] : ones[counter%10]) + const row = rows[counter-1] + const id = row.thread_parent || row.channel_id + if (cache.has(id)) continue + + try { + var channel = await discord.snow.channel.getChannel(id) + } catch (e) { + continue + } + + const guildID = channel.guild_id + const channels = await discord.snow.guild.getGuildChannels(guildID) + for (const channel of channels) { + cache.set(channel.id, guildID) + } + } + + // Update channels and threads + process.stdout.write("\n") + db.transaction(() => { + // Fill in missing data + for (const row of rows) { + const guildID = cache.get(row.thread_parent) || cache.get(row.channel_id) + if (guildID) { + db.prepare("UPDATE channel_room SET guild_id = ? WHERE channel_id = ?").run(guildID, row.channel_id) + } else { + db.prepare("DELETE FROM webhook WHERE channel_id = ?").run(row.channel_id) + db.prepare("DELETE FROM channel_room WHERE channel_id = ?").run(row.channel_id) + } + } + })() +} diff --git a/src/db/migrations/0030-require-guild-id.sql b/src/db/migrations/0030-require-guild-id.sql new file mode 100644 index 00000000..264b69b7 --- /dev/null +++ b/src/db/migrations/0030-require-guild-id.sql @@ -0,0 +1,44 @@ +-- https://sqlite.org/lang_altertable.html + +-- 1 +PRAGMA foreign_keys=OFF; +-- 2 +BEGIN TRANSACTION; + +-- 4 +CREATE TABLE "new_channel_room" ( + "channel_id" TEXT NOT NULL, + "room_id" TEXT NOT NULL UNIQUE, + "name" TEXT NOT NULL, + "nick" TEXT, + "thread_parent" TEXT, + "custom_avatar" TEXT, + "last_bridged_pin_timestamp" INTEGER, + "speedbump_id" TEXT, + "speedbump_checked" INTEGER, + "speedbump_webhook_id" TEXT, + "guild_id" TEXT NOT NULL, + "custom_topic" INTEGER DEFAULT 0, + PRIMARY KEY("channel_id"), + FOREIGN KEY("guild_id") REFERENCES "guild_active"("guild_id") ON DELETE CASCADE +) WITHOUT ROWID; + +-- 5 +INSERT INTO new_channel_room + (channel_id, room_id, name, nick, thread_parent, custom_avatar, last_bridged_pin_timestamp, speedbump_id, speedbump_checked, speedbump_webhook_id, guild_id, custom_topic) +SELECT channel_id, room_id, name, nick, thread_parent, custom_avatar, last_bridged_pin_timestamp, speedbump_id, speedbump_checked, speedbump_webhook_id, guild_id, custom_topic + FROM channel_room; + +-- 6 +DROP TABLE channel_room; + +-- 7 +ALTER TABLE new_channel_room RENAME TO channel_room; + +-- 10 +PRAGMA foreign_key_check; + +-- 11 +COMMIT; +-- 12 +PRAGMA foreign_keys=ON; diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index bdf6bf9d..38932cc7 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -103,6 +103,11 @@ export type Models = { historical_room_index: number } + room_upgrade_pending: { + new_room_id: string + old_room_id: string + } + sim: { user_id: string username: string diff --git a/src/discord/interactions/permissions.js b/src/discord/interactions/permissions.js index 07324acc..c780a2ae 100644 --- a/src/discord/interactions/permissions.js +++ b/src/discord/interactions/permissions.js @@ -9,13 +9,15 @@ const {InteractionMethods} = require("snowtransfer") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") +/** @type {import("../../m2d/converters/utils")} */ +const utils = sync.require("../../m2d/converters/utils") /** * @param {DiscordTypes.APIContextMenuGuildInteraction} interaction - * @param {{api: typeof api}} di + * @param {{api: typeof api, utils: typeof utils}} di * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} */ -async function* _interact({data, guild_id}, {api}) { +async function* _interact({data, guild_id}, {api, utils}) { // Get message info const row = from("event_message") .join("message_room", "message_id").join("historical_channel_room", "historical_room_index") @@ -45,12 +47,10 @@ async function* _interact({data, guild_id}, {api}) { assert(spaceID) // Get the power level - /** @type {Ty.Event.M_Power_Levels} */ - const powerLevelsContent = await api.getStateEvent(spaceID, "m.room.power_levels", "") - const userPower = powerLevelsContent.users?.[event.sender] || 0 + const {powers: {[event.sender]: userPower, [utils.bot]: botPower}} = await utils.getEffectivePower(spaceID, [event.sender, utils.bot], api) - // Administrators equal to the bot cannot be demoted - if (userPower >= 100) { + // Administrators/founders equal to the bot cannot be demoted + if (userPower >= botPower) { return yield {createInteractionResponse: { type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, data: { @@ -60,6 +60,8 @@ async function* _interact({data, guild_id}, {api}) { }} } + const adminLabel = botPower === 100 ? "Admin (you cannot undo this!)" : "Admin" + yield {createInteractionResponse: { type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, data: { @@ -82,9 +84,9 @@ async function* _interact({data, guild_id}, {api}) { value: "moderator", default: userPower >= 50 && userPower < 100 }, { - label: "Admin (you cannot undo this!)", + label: adminLabel, value: "admin", - default: userPower === 100 + default: userPower >= 100 } ] } @@ -138,7 +140,7 @@ async function* _interactEdit({data, guild_id, message}, {api}) { /** @param {DiscordTypes.APIContextMenuGuildInteraction} interaction */ async function interact(interaction) { - for await (const response of _interact(interaction, {api})) { + for await (const response of _interact(interaction, {api, utils})) { if (response.createInteractionResponse) { // TODO: Test if it is reasonable to remove `await` from these calls. Or zip these calls with the next interaction iteration and use Promise.all. await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, response.createInteractionResponse) diff --git a/src/discord/interactions/permissions.test.js b/src/discord/interactions/permissions.test.js index a7da859b..ef3fef2d 100644 --- a/src/discord/interactions/permissions.test.js +++ b/src/discord/interactions/permissions.test.js @@ -2,6 +2,7 @@ const {test} = require("supertape") const DiscordTypes = require("discord-api-types/v10") const {select, db} = require("../../passthrough") const {_interact, _interactEdit} = require("./permissions") +const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") /** * @template T @@ -46,6 +47,10 @@ test("permissions: reports permissions of selected matrix user (implicit default }, guild_id: "112760669178241024" }, { + utils: { + bot: "@_ooye_bot:cadence.moe", + getEffectivePower: mockGetEffectivePower() + }, api: { async getEvent(roomID, eventID) { called++ @@ -54,22 +59,13 @@ test("permissions: reports permissions of selected matrix user (implicit default return { sender: "@cadence:cadence.moe" } - }, - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") // space ID - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: {} - } } } })) t.equal(msgs.length, 1) t.equal(msgs[0].createInteractionResponse.data.content, "Showing permissions for `@cadence:cadence.moe`. Click to edit.") t.deepEqual(msgs[0].createInteractionResponse.data.components[0].components[0].options[0], {label: "Default", value: "default", default: true}) - t.equal(called, 2) + t.equal(called, 1) }) test("permissions: reports permissions of selected matrix user (moderator)", async t => { @@ -80,6 +76,10 @@ test("permissions: reports permissions of selected matrix user (moderator)", asy }, guild_id: "112760669178241024" }, { + utils: { + bot: "@_ooye_bot:cadence.moe", + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe"], {"@cadence:cadence.moe": 50}) + }, api: { async getEvent(roomID, eventID) { called++ @@ -88,27 +88,16 @@ test("permissions: reports permissions of selected matrix user (moderator)", asy return { sender: "@cadence:cadence.moe" } - }, - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") // space ID - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return { - users: { - "@cadence:cadence.moe": 50 - } - } } } })) t.equal(msgs.length, 1) t.equal(msgs[0].createInteractionResponse.data.content, "Showing permissions for `@cadence:cadence.moe`. Click to edit.") t.deepEqual(msgs[0].createInteractionResponse.data.components[0].components[0].options[1], {label: "Moderator", value: "moderator", default: true}) - t.equal(called, 2) + t.equal(called, 1) }) -test("permissions: reports permissions of selected matrix user (admin)", async t => { +test("permissions: reports permissions of selected matrix user (admin v12 can be demoted)", async t => { let called = 0 const msgs = await fromAsync(_interact({ data: { @@ -116,6 +105,10 @@ test("permissions: reports permissions of selected matrix user (admin)", async t }, guild_id: "112760669178241024" }, { + utils: { + bot: "@_ooye_bot:cadence.moe", + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe"], {"@cadence:cadence.moe": 100}) + }, api: { async getEvent(roomID, eventID) { called++ @@ -124,16 +117,34 @@ test("permissions: reports permissions of selected matrix user (admin)", async t return { sender: "@cadence:cadence.moe" } - }, - async getStateEvent(roomID, type, key) { + } + } + })) + t.equal(msgs.length, 1) + t.equal(msgs[0].createInteractionResponse.data.content, "Showing permissions for `@cadence:cadence.moe`. Click to edit.") + t.deepEqual(msgs[0].createInteractionResponse.data.components[0].components[0].options[2], {label: "Admin", value: "admin", default: true}) + t.equal(called, 1) +}) + +test("permissions: reports permissions of selected matrix user (admin v11 cannot be demoted)", async t => { + let called = 0 + const msgs = await fromAsync(_interact({ + data: { + target_id: "1128118177155526666" + }, + guild_id: "112760669178241024" + }, { + utils: { + bot: "@_ooye_bot:cadence.moe", + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe"], {"@cadence:cadence.moe": 100, "@_ooye_bot:cadence.moe": 100}, "11") + }, + api: { + async getEvent(roomID, eventID) { called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") // space ID - t.equal(type, "m.room.power_levels") - t.equal(key, "") + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") // room ID + t.equal(eventID, "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4") return { - users: { - "@cadence:cadence.moe": 100 - } + sender: "@cadence:cadence.moe" } } } @@ -141,7 +152,7 @@ test("permissions: reports permissions of selected matrix user (admin)", async t t.equal(msgs.length, 1) t.equal(msgs[0].createInteractionResponse.data.content, "`@cadence:cadence.moe` has administrator permissions. This cannot be edited.") t.notOk(msgs[0].createInteractionResponse.data.components) - t.equal(called, 2) + t.equal(called, 1) }) test("permissions: can update user to moderator", async t => { diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 3fe8776c..a030ac5f 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -447,9 +447,8 @@ async function checkWrittenMentions(content, senderMxid, roomID, guild, di) { let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+ if (writtenMentionMatch) { if (writtenMentionMatch[1] === "room") { // convert @room to @everyone - const powerLevels = await di.api.getStateEvent(roomID, "m.room.power_levels", "") - const userPower = powerLevels.users?.[senderMxid] || 0 - if (userPower >= powerLevels.notifications?.room) { + const {powers: {[senderMxid]: userPower}, powerLevels} = await mxUtils.getEffectivePower(roomID, [senderMxid], di.api) + if (userPower >= (powerLevels.notifications?.room ?? 50)) { return { // @ts-ignore - typescript doesn't know about indices yet content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `@everyone` + content.slice(writtenMentionMatch.indices[1][1]), @@ -924,7 +923,6 @@ async function eventToMessage(event, guild, di) { // Respect sender's angle brackets const alreadySuppressed = content[match.index-1+offset] === "<" && content[match.index+match.length+offset] === ">" - console.error(content, match.index-1+offset, content[match.index-1+offset]) if (alreadySuppressed) continue // Put < > around any surviving matrix.to links let shouldSuppress = !!match[0].match(/^https?:\/\/matrix\.to\//) diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 6665e873..b298f5b1 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -4985,7 +4985,7 @@ test("event2message: @room converts to @everyone and is allowed when the room do event_id: "$SiXetU9h9Dg-M9Frcw_C6ahnoXZ3QPZe3MVJR5tcB9A" }, data.guild.general, { api: { - getStateEvent(roomID, type, key) { + async getStateEvent(roomID, type, key) { called++ t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") t.equal(type, "m.room.power_levels") @@ -4996,6 +4996,19 @@ test("event2message: @room converts to @everyone and is allowed when the room do room: 0 } } + }, + async getStateEventOuter(roomID, type, key) { + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + content: { + room_version: "11" + } + } } } }), @@ -5016,7 +5029,6 @@ test("event2message: @room converts to @everyone and is allowed when the room do }) test("event2message: @room converts to @everyone but is not allowed when the room restricts who can use it", async t => { - let called = 0 t.deepEqual( await eventToMessage({ type: "m.room.message", @@ -5031,8 +5043,7 @@ test("event2message: @room converts to @everyone but is not allowed when the roo event_id: "$SiXetU9h9Dg-M9Frcw_C6ahnoXZ3QPZe3MVJR5tcB9A" }, data.guild.general, { api: { - getStateEvent(roomID, type, key) { - called++ + async getStateEvent(roomID, type, key) { t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") t.equal(type, "m.room.power_levels") t.equal(key, "") @@ -5042,6 +5053,19 @@ test("event2message: @room converts to @everyone but is not allowed when the roo room: 20 } } + }, + async getStateEventOuter(roomID, type, key) { + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + content: { + room_version: "11" + } + } } } }), @@ -5062,7 +5086,6 @@ test("event2message: @room converts to @everyone but is not allowed when the roo }) test("event2message: @room converts to @everyone and is allowed if the user has sufficient power to use it", async t => { - let called = 0 t.deepEqual( await eventToMessage({ type: "m.room.message", @@ -5077,8 +5100,7 @@ test("event2message: @room converts to @everyone and is allowed if the user has event_id: "$SiXetU9h9Dg-M9Frcw_C6ahnoXZ3QPZe3MVJR5tcB9A" }, data.guild.general, { api: { - getStateEvent(roomID, type, key) { - called++ + async getStateEvent(roomID, type, key) { t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") t.equal(type, "m.room.power_levels") t.equal(key, "") @@ -5090,6 +5112,19 @@ test("event2message: @room converts to @everyone and is allowed if the user has room: 20 } } + }, + async getStateEventOuter(roomID, type, key) { + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + content: { + room_version: "11" + } + } } } }), diff --git a/src/m2d/converters/utils.js b/src/m2d/converters/utils.js index 7335c8ea..ccdef831 100644 --- a/src/m2d/converters/utils.js +++ b/src/m2d/converters/utils.js @@ -129,7 +129,7 @@ class MatrixStringBuilder { * https://spec.matrix.org/v1.9/appendices/#routing * https://gitdab.com/cadence/out-of-your-element/issues/11 * @param {string} roomID - * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api + * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("../../matrix/api")[K]} | {getEffectivePower: (roomID: string, mxids: string[], api: any) => Promise<{powers: Record, allCreators: string[], tombstone: number, roomCreate: Ty.Event.StateOuter, powerLevels: Ty.Event.M_Power_Levels}>, getJoinedMembers: import("../../matrix/api")["getJoinedMembers"]}} api */ async function getViaServers(roomID, api) { const candidates = [] @@ -138,10 +138,10 @@ async function getViaServers(roomID, api) { candidates.push(reg.ooye.server_name) // Candidate 1: Highest joined non-sim non-bot power level user in the room // https://github.com/matrix-org/matrix-react-sdk/blob/552c65db98b59406fb49562e537a2721c8505517/src/utils/permalinks/Permalinks.ts#L172 - const {allCreators, powerLevels} = await getEffectivePower(roomID, [bot], api) + const call = "getEffectivePower" in api ? api.getEffectivePower(roomID, [bot], api) : getEffectivePower(roomID, [bot], api) + const {allCreators, powerLevels} = await call const sorted = allCreators.concat(Object.entries(powerLevels.users ?? {}).sort((a, b) => b[1] - a[1]).map(([mxid]) => mxid)) // Highest... - for (const power of sorted) { - const mxid = power[0] + for (const mxid of sorted) { if (!(mxid in joined)) continue // joined... if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot... const match = mxid.match(/:(.*)/) diff --git a/src/m2d/converters/utils.test.js b/src/m2d/converters/utils.test.js index 650f4201..9c113938 100644 --- a/src/m2d/converters/utils.test.js +++ b/src/m2d/converters/utils.test.js @@ -3,7 +3,7 @@ const e = new Error("Custom error") const {test} = require("supertape") -const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers} = require("./utils") +const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers, roomHasAtLeastVersion} = require("./utils") const util = require("util") /** @param {string[]} mxids */ @@ -88,9 +88,42 @@ test("MatrixStringBuilder: complete code coverage", t => { }) }) +/** + * @param {string[]} [creators] + * @param {{[x: string]: number}} [users] + * @param {string} [roomVersion] + */ +function mockGetEffectivePower(creators = ["@_ooye_bot:cadence.moe"], users = {}, roomVersion = "12") { + return async function getEffectivePower(roomID, mxids) { + return { + allCreators: creators, + powerLevels: {users}, + powers: mxids.reduce((a, mxid) => { + if (creators.includes(mxid) && roomHasAtLeastVersion(roomVersion, 12)) a[mxid] = Infinity + else if (mxid in users) a[mxid] = users[mxid] + else a[mxid] = 0 + return a + }, {}), + roomCreate: { + type: "m.room.create", + state_key: "", + sender: creators[0], + content: { + additional_creators: creators.slice(1), + room_version: roomVersion + }, + room_id: roomID, + origin_server_ts: 0, + event_id: "$create" + }, + tombstone: roomVersion === "12" ? 150 : 100, + } + } +} + test("getViaServers: returns the server name if the room only has sim users", async t => { const result = await getViaServers("!baby", { - getStateEvent: async () => ({}), + getEffectivePower: mockGetEffectivePower(), getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe"]) }) t.deepEqual(result, ["cadence.moe"]) @@ -98,7 +131,7 @@ test("getViaServers: returns the server name if the room only has sim users", as test("getViaServers: also returns the most popular servers in order", async t => { const result = await getViaServers("!baby", { - getStateEvent: async () => ({}), + getEffectivePower: mockGetEffectivePower(), getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid"]) }) t.deepEqual(result, ["cadence.moe", "thecollective.invalid", "selfhosted.invalid"]) @@ -106,20 +139,27 @@ test("getViaServers: also returns the most popular servers in order", async t => test("getViaServers: does not return IP address servers", async t => { const result = await getViaServers("!baby", { - getStateEvent: async () => ({}), + getEffectivePower: mockGetEffectivePower(), getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:45.77.232.172:8443", "@cadence:[::1]:8443", "@cadence:123example.456example.invalid"]) }) t.deepEqual(result, ["cadence.moe", "123example.456example.invalid"]) }) +test("getViaServers: also returns the highest power level user (v12 creator)", async t => { + const result = await getViaServers("!baby", { + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe", "@singleuser:selfhosted.invalid"], { + "@moderator:tractor.invalid": 50 + }), + getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@moderator:tractor.invalid"]) + }) + t.deepEqual(result, ["cadence.moe", "selfhosted.invalid", "thecollective.invalid", "tractor.invalid"]) +}) + test("getViaServers: also returns the highest power level user (100)", async t => { const result = await getViaServers("!baby", { - getStateEvent: async () => ({ - users: { - "@moderator:tractor.invalid": 50, - "@singleuser:selfhosted.invalid": 100, - "@_ooye_bot:cadence.moe": 100 - } + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe"], { + "@moderator:tractor.invalid": 50, + "@singleuser:selfhosted.invalid": 100 }), getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@moderator:tractor.invalid"]) }) @@ -128,11 +168,8 @@ test("getViaServers: also returns the highest power level user (100)", async t = test("getViaServers: also returns the highest power level user (50)", async t => { const result = await getViaServers("!baby", { - getStateEvent: async () => ({ - users: { - "@moderator:tractor.invalid": 50, - "@_ooye_bot:cadence.moe": 100 - } + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe"], { + "@moderator:tractor.invalid": 50 }), getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"]) }) @@ -141,38 +178,23 @@ test("getViaServers: also returns the highest power level user (50)", async t => test("getViaServers: returns at most 4 results", async t => { const result = await getViaServers("!baby", { - getStateEvent: async () => ({ - users: { - "@moderator:tractor.invalid": 50, - "@singleuser:selfhosted.invalid": 100, - "@_ooye_bot:cadence.moe": 100 - } + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe"], { + "@moderator:tractor.invalid": 50, + "@singleuser:selfhosted.invalid": 100 }), getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"]) }) t.deepEqual(result.length, 4) }) -test("getViaServers: returns results even when power levels can't be fetched", async t => { - const result = await getViaServers("!baby", { - getStateEvent: async () => { - throw new Error("event not found or something") - }, - getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"]) - }) - t.deepEqual(result.length, 4) -}) - test("getViaServers: only considers power levels of currently joined members", async t => { const result = await getViaServers("!baby", { - getStateEvent: async () => ({ - users: { - "@moderator:tractor.invalid": 50, - "@former_moderator:missing.invalid": 100, - "@_ooye_bot:cadence.moe": 100 - } + getEffectivePower: mockGetEffectivePower(["@_ooye_bot:cadence.moe", "@former_moderator:missing.invalid"], { + "@moderator:tractor.invalid": 50 }), getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"]) }) t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"]) }) + +module.exports.mockGetEffectivePower = mockGetEffectivePower diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 9fe6ed57..b3269620 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -26,6 +26,8 @@ const utils = sync.require("./converters/utils") const api = sync.require("../matrix/api") /** @type {import("../d2m/actions/create-room")} */ const createRoom = sync.require("../d2m/actions/create-room") +/** @type {import("../matrix/room-upgrade")} */ +const roomUpgrade = require("../matrix/room-upgrade") const {reg} = require("../matrix/read-registration") let lastReportedEvent = 0 @@ -171,9 +173,8 @@ async function onRetryReactionAdd(reactionEvent) { // To stop people injecting misleading messages, the reaction needs to come from either the original sender or a room moderator if (reactionEvent.sender !== event.sender) { // Check if it's a room moderator - const powerLevelsStateContent = await api.getStateEvent(roomID, "m.room.power_levels", "") - const powerLevel = powerLevelsStateContent.users?.[reactionEvent.sender] || 0 - if (powerLevel < 50) return + const {powers: {[reactionEvent.sender]: senderPower}, powerLevels} = await utils.getEffectivePower(roomID, [reactionEvent.sender], api) + if (senderPower < (powerLevels.state_default ?? 50)) return } // Retry @@ -330,6 +331,11 @@ async event => { if (event.state_key[0] !== "@") return const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` + if (event.state_key === bot) { + const upgraded = await roomUpgrade.onBotMembership(event) + if (upgraded) return + } + if (event.content.membership === "invite" && event.state_key === bot) { // We were invited to a room. We should join, and register the invite details for future reference in web. let attemptedApiMessage = "According to unsigned invite data." @@ -342,10 +348,10 @@ async event => { attemptedApiMessage = "According to unsigned invite data. SSS API unavailable: " + e.toString() } } - const name = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.name", "name") - const topic = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.topic", "topic") - const avatar = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.avatar", "url") - const creationType = getFromInviteRoomState(event.unsigned?.invite_room_state, "m.room.create", "type") + const name = getFromInviteRoomState(inviteRoomState, "m.room.name", "name") + const topic = getFromInviteRoomState(inviteRoomState, "m.room.topic", "topic") + const avatar = getFromInviteRoomState(inviteRoomState, "m.room.avatar", "url") + const creationType = getFromInviteRoomState(inviteRoomState, "m.room.create", "type") if (!name) return await api.leaveRoomWithReason(event.room_id, `Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite! (${attemptedApiMessage})`) await api.joinRoom(event.room_id) db.prepare("INSERT OR IGNORE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)").run(event.sender, event.room_id, creationType, name, topic, avatar) @@ -368,18 +374,14 @@ async event => { if (!exists) return // don't cache members in unbridged rooms // Member is here - let powerLevel = 0 - try { - /** @type {Ty.Event.M_Power_Levels} */ - const powerLevelsEvent = await api.getStateEvent(event.room_id, "m.room.power_levels", "") - powerLevel = powerLevelsEvent.users?.[event.state_key] ?? powerLevelsEvent.users_default ?? 0 - } catch (e) {} + let {powers: {[event.state_key]: memberPower}, tombstone} = await utils.getEffectivePower(event.room_id, [event.state_key], api) + if (memberPower === Infinity) memberPower = tombstone // database storage compatibility const displayname = event.content.displayname || null const avatar_url = event.content.avatar_url db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url, power_level) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET displayname = ?, avatar_url = ?, power_level = ?").run( event.room_id, event.state_key, - displayname, avatar_url, powerLevel, - displayname, avatar_url, powerLevel + displayname, avatar_url, memberPower, + displayname, avatar_url, memberPower ) })) @@ -390,11 +392,22 @@ sync.addTemporaryListener(as, "type:m.room.power_levels", guard("m.room.power_le async event => { if (event.state_key !== "") return const existingPower = select("member_cache", "mxid", {room_id: event.room_id}).pluck().all() + const {allCreators} = await utils.getEffectivePower(event.room_id, [], api) const newPower = event.content.users || {} for (const mxid of existingPower) { - db.prepare("UPDATE member_cache SET power_level = ? WHERE room_id = ? AND mxid = ?").run(newPower[mxid] || 0, event.room_id, mxid) + if (!allCreators.includes(mxid)) { + db.prepare("UPDATE member_cache SET power_level = ? WHERE room_id = ? AND mxid = ?").run(newPower[mxid] || 0, event.room_id, mxid) + } } })) +sync.addTemporaryListener(as, "type:m.room.tombstone", guard("m.room.tombstone", +/** + * @param {Ty.Event.StateOuter} event + */ +async event => { + await roomUpgrade.onTombstone(event) +})) + module.exports.stringifyErrorStack = stringifyErrorStack module.exports.sendError = sendError diff --git a/src/matrix/api.js b/src/matrix/api.js index c17d7893..824f13b8 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -122,7 +122,7 @@ async function getEventForTimestamp(roomID, ts) { /** * @param {string} roomID - * @returns {Promise} + * @returns {Promise[]>} */ function getAllState(roomID) { return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state`) @@ -142,7 +142,7 @@ function getStateEvent(roomID, type, key) { * @param {string} roomID * @param {string} type * @param {string} key - * @returns {Promise} the entire state event + * @returns {Promise>} the entire state event */ function getStateEventOuter(roomID, type, key) { return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state/${type}/${key}?format=event`) diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index b0fe9479..ace9c36a 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -69,7 +69,7 @@ function kstateToCreationContent(kstate) { } /** - * @param {import("../types").Event.BaseStateEvent[]} events + * @param {import("../types").Event.StateOuter[]} events * @returns {any} */ function stateToKState(events) { diff --git a/src/matrix/matrix-command-handler.js b/src/matrix/matrix-command-handler.js index f712ece2..601b2dca 100644 --- a/src/matrix/matrix-command-handler.js +++ b/src/matrix/matrix-command-handler.js @@ -123,12 +123,9 @@ const commands = [{ } if (matrixOnlyReason) { // If uploading to Matrix, check if we have permission - const state = await api.getAllState(event.room_id) - const kstate = ks.stateToKState(state) - const powerLevels = kstate["m.room.power_levels/"] - const required = powerLevels.events["im.ponies.room_emotes"] ?? powerLevels.state_default ?? 50 - const have = powerLevels.users[`@${reg.sender_localpart}:${reg.ooye.server_name}`] ?? powerLevels.users_default ?? 0 - if (have < required) { + const {powerLevels, powers: {[mxUtils.bot]: botPower}} = await mxUtils.getEffectivePower(event.room_id, [mxUtils.bot], api) + const requiredPower = powerLevels.events["im.ponies.room_emotes"] ?? powerLevels.state_default ?? 50 + if (botPower < requiredPower) { return api.sendEvent(event.room_id, "m.room.message", { ...ctx, msgtype: "m.text", diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index bb359752..888aa54d 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -72,8 +72,13 @@ async function mreq(method, url, bodyIn, extra = {}) { }, extra) const res = await fetch(baseUrl + url, opts) - /** @type {any} */ - const root = await res.json() + const text = await res.text() + try { + /** @type {any} */ + var root = JSON.parse(text) + } catch (e) { + throw new MatrixServerError(text, {baseUrl, url, ...opts}) + } if (!res.ok || root.errcode) { delete opts.headers?.["Authorization"] diff --git a/src/matrix/room-upgrade.js b/src/matrix/room-upgrade.js new file mode 100644 index 00000000..0b02762a --- /dev/null +++ b/src/matrix/room-upgrade.js @@ -0,0 +1,94 @@ +// @ts-check + +const assert = require("assert/strict") +const Ty = require("../types") +const {Semaphore} = require("@chriscdn/promise-semaphore") +const {tag} = require("@cloudrac3r/html-template-tag") +const {discord, db, sync, as, select, from} = require("../passthrough") + +/** @type {import("./api")}) */ +const api = sync.require("./api") +/** @type {import("../d2m/actions/create-room")}) */ +const createRoom = sync.require("../d2m/actions/create-room") +/** @type {import("../m2d/converters/utils")}) */ +const utils = sync.require("../m2d/converters/utils") + +const roomUpgradeSema = new Semaphore() + +/** + * @param {Ty.Event.StateOuter} event + */ +async function onTombstone(event) { + // Validate + if (event.state_key !== "") return + if (!event.content.replacement_room) return + + // Set up + const oldRoomID = event.room_id + const newRoomID = event.content.replacement_room + const channel = select("channel_room", ["name", "channel_id"], {room_id: oldRoomID}).get() + if (!channel) return + db.prepare("REPLACE INTO room_upgrade_pending (new_room_id, old_room_id) VALUES (?, ?)").run(newRoomID, oldRoomID) + + // Try joining + try { + await api.joinRoom(newRoomID) + } catch (e) { + const message = new utils.MatrixStringBuilder() + message.add( + `You upgraded the bridged room ${channel.name}. To keep bridging, I need you to invite me to the new room: https://matrix.to/#/${newRoomID}`, + tag`You upgraded the bridged room ${channel.name}. To keep bridging, I need you to invite me to the new room: https://matrix.to/#/${newRoomID}` + ) + const privateRoomID = await api.usePrivateChat(event.sender) + await api.sendEvent(privateRoomID, "m.room.message", message.get()) + } + + // Now wait to be invited to/join the room that has the upgrade pending... +} + +/** + * @param {Ty.Event.StateOuter} event + * @returns {Promise} whether to cancel other membership actions + */ +async function onBotMembership(event) { + // Check if an upgrade is pending for this room + const newRoomID = event.room_id + const oldRoomID = select("room_upgrade_pending", "old_room_id", {new_room_id: newRoomID}).pluck().get() + if (!oldRoomID) return + + // Check if is join/invite + if (event.content.membership !== "invite" && event.content.membership !== "join") return + + return await roomUpgradeSema.request(async () => { + // If invited, join + if (event.content.membership === "invite") { + await api.joinRoom(newRoomID) + } + + const channelRow = from("channel_room").join("guild_space", "guild_id").where({room_id: oldRoomID}).select("space_id", "guild_id", "channel_id").get() + assert(channelRow) + + // Remove old room from space + await api.sendState(channelRow.space_id, "m.space.child", oldRoomID, {}) + // await api.sendState(oldRoomID, "m.space.parent", spaceID, {}) // keep this - the room isn't advertised but should still be grouped if opened + + // Remove declaration that old room is bridged (if able) + try { + await api.sendState(oldRoomID, "uk.half-shot.bridge", `moe.cadence.ooye://discord/${channelRow.guild_id}/${channelRow.channel_id}`, {}) + } catch (e) {} + + // Update database + db.transaction(() => { + db.prepare("DELETE FROM room_upgrade_pending WHERE new_room_id = ?").run(newRoomID) + db.prepare("UPDATE channel_room SET room_id = ? WHERE channel_id = ?").run(newRoomID, channelRow.channel_id) + db.prepare("INSERT INTO historical_channel_room (room_id, reference_channel_id, upgraded_timestamp) VALUES (?, ?, ?)").run(newRoomID, channelRow.channel_id, Date.now()) + })() + + // Sync + await createRoom.syncRoom(channelRow.channel_id) + return true + }, event.room_id) +} + +module.exports.onTombstone = onTombstone +module.exports.onBotMembership = onBotMembership diff --git a/src/types.d.ts b/src/types.d.ts index 3b0e5af9..72ed83e3 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -143,21 +143,6 @@ export namespace Event { } } - export type BaseStateEvent = { - type: string - room_id: string - sender: string - content: any - state_key: string - origin_server_ts: number - unsigned?: any - event_id: string - user_id: string - age: number - replaces_state: string - prev_content?: any - } - export type StrippedChildStateEvent = { type: string state_key: string diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 0afbc495..17f27df6 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -10,6 +10,8 @@ const {discord, db, as, sync, select, from} = require("../../passthrough") const auth = sync.require("../auth") /** @type {import("../../matrix/mreq")} */ const mreq = sync.require("../../matrix/mreq") +/** @type {import("../../m2d/converters/utils")}*/ +const utils = sync.require("../../m2d/converters/utils") const {reg} = require("../../matrix/read-registration") /** @@ -87,18 +89,11 @@ as.router.post("/api/link-space", defineEventHandler(async event => { } // Check bridge has PL 100 - const me = `@${reg.sender_localpart}:${reg.ooye.server_name}` - /** @type {Ty.Event.M_Power_Levels?} */ - let powerLevelsStateContent = null - try { - powerLevelsStateContent = await api.getStateEvent(spaceID, "m.room.power_levels", "") - } catch (e) {} - const selfPowerLevel = powerLevelsStateContent?.users?.[me] ?? powerLevelsStateContent?.users_default ?? 0 - if (selfPowerLevel < (powerLevelsStateContent?.state_default ?? 50) || selfPowerLevel < 100) throw createError({status: 400, message: "Bad Request", data: "OOYE needs power level 100 (admin) in the target Matrix space"}) + const {powerLevels, powers: {[utils.bot]: selfPowerLevel, [session.data.mxid]: invitingPowerLevel}} = await utils.getEffectivePower(spaceID, [utils.bot, session.data.mxid], api) + if (selfPowerLevel < (powerLevels?.state_default ?? 50) || selfPowerLevel < 100) throw createError({status: 400, message: "Bad Request", data: "OOYE needs power level 100 (admin) in the target Matrix space"}) // Check inviting user is a moderator in the space - const invitingPowerLevel = powerLevelsStateContent?.users?.[session.data.mxid] ?? powerLevelsStateContent?.users_default ?? 0 - if (invitingPowerLevel < (powerLevelsStateContent?.state_default ?? 50)) throw createError({status: 403, message: "Forbidden", data: `You need to be at least power level 50 (moderator) in the target Matrix space to set up OOYE, but you are currently power level ${invitingPowerLevel}.`}) + if (invitingPowerLevel < (powerLevels?.state_default ?? 50)) throw createError({status: 403, message: "Forbidden", data: `You need to be at least power level 50 (moderator) in the target Matrix space to set up OOYE, but you are currently power level ${invitingPowerLevel}.`}) // Insert database entry db.transaction(() => { @@ -169,14 +164,8 @@ as.router.post("/api/link", defineEventHandler(async event => { } // Check bridge has PL 100 - const me = `@${reg.sender_localpart}:${reg.ooye.server_name}` - /** @type {Ty.Event.M_Power_Levels?} */ - let powerLevelsStateContent = null - try { - powerLevelsStateContent = await api.getStateEvent(parsedBody.matrix, "m.room.power_levels", "") - } catch (e) {} - const selfPowerLevel = powerLevelsStateContent?.users?.[me] ?? powerLevelsStateContent?.users_default ?? 0 - if (selfPowerLevel < (powerLevelsStateContent?.state_default ?? 50) || selfPowerLevel < 100) throw createError({status: 400, message: "Bad Request", data: "OOYE needs power level 100 (admin) in the target Matrix room"}) + const {powerLevels, powers: {[utils.bot]: selfPowerLevel}} = await utils.getEffectivePower(parsedBody.matrix, [utils.bot], api) + if (selfPowerLevel < (powerLevels?.state_default ?? 50) || selfPowerLevel < 100) throw createError({status: 400, message: "Bad Request", data: "OOYE needs power level 100 (admin) in the target Matrix room"}) // Insert database entry, but keep the room's existing properties if they are set const nick = await api.getStateEvent(parsedBody.matrix, "m.room.name", "").then(content => content.name || null).catch(() => null) diff --git a/src/web/routes/link.test.js b/src/web/routes/link.test.js index ffe4e5ee..291ea8e6 100644 --- a/src/web/routes/link.test.js +++ b/src/web/routes/link.test.js @@ -81,63 +81,6 @@ test("web link space: check that OOYE is joined", async t => { t.equal(called, 1) }) -test("web link space: check that OOYE has PL 100 (not missing)", async t => { - let called = 0 - const [error] = await tryToCatch(() => router.test("post", "/api/link-space", { - sessionData: { - managedGuilds: ["665289423482519565"], - mxid: "@cadence:cadence.moe" - }, - body: { - space_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", - guild_id: "665289423482519565" - }, - api: { - async joinRoom(roomID) { - called++ - return roomID - }, - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") - t.equal(type, "m.room.power_levels") - throw new MatrixServerError({errcode: "M_NOT_FOUND", error: "what if I told you that power levels never existed"}) - } - } - })) - t.equal(error.data, "OOYE needs power level 100 (admin) in the target Matrix space") - t.equal(called, 2) -}) - -test("web link space: check that OOYE has PL 100 (not users_default)", async t => { - let called = 0 - const [error] = await tryToCatch(() => router.test("post", "/api/link-space", { - sessionData: { - managedGuilds: ["665289423482519565"], - mxid: "@cadence:cadence.moe" - }, - body: { - space_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", - guild_id: "665289423482519565" - }, - api: { - async joinRoom(roomID) { - called++ - return roomID - }, - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - return {} - } - } - })) - t.equal(error.data, "OOYE needs power level 100 (admin) in the target Matrix space") - t.equal(called, 2) -}) - test("web link space: check that OOYE has PL 100 (not 50)", async t => { let called = 0 const [error] = await tryToCatch(() => router.test("post", "/api/link-space", { @@ -160,11 +103,28 @@ test("web link space: check that OOYE has PL 100 (not 50)", async t => { t.equal(type, "m.room.power_levels") t.equal(key, "") return {users: {"@_ooye_bot:cadence.moe": 50}} + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@creator:cadence.moe", + room_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + content: { + room_version: "11" + } + } } } })) t.equal(error.data, "OOYE needs power level 100 (admin) in the target Matrix space") - t.equal(called, 2) + t.equal(called, 3) }) test("web link space: check that inviting user has PL 50", async t => { @@ -189,11 +149,28 @@ test("web link space: check that inviting user has PL 50", async t => { t.equal(type, "m.room.power_levels") t.equal(key, "") return {users: {"@_ooye_bot:cadence.moe": 100}} + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@creator:cadence.moe", + room_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + content: { + room_version: "11" + } + } } } })) t.equal(error.data, "You need to be at least power level 50 (moderator) in the target Matrix space to set up OOYE, but you are currently power level 0.") - t.equal(called, 2) + t.equal(called, 3) }) test("web link space: successfully adds entry to database and loads page", async t => { @@ -218,10 +195,27 @@ test("web link space: successfully adds entry to database and loads page", async t.equal(type, "m.room.power_levels") t.equal(key, "") return {users: {"@_ooye_bot:cadence.moe": 100, "@cadence:cadence.moe": 50}} + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@creator:cadence.moe", + room_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + content: { + room_version: "11" + } + } } } }) - t.equal(called, 2) + t.equal(called, 3) // check that the entry was added to the database t.equal(select("guild_space", "privacy_level", {guild_id: "665289423482519565", space_id: "!zTMspHVUBhFLLSdmnS:cadence.moe"}).pluck().get(), 0) @@ -441,47 +435,7 @@ test("web link room: check that bridge can join room (uses via for join attempt) t.equal(called, 2) }) -test("web link room: check that bridge has PL 100 in target room (event missing)", async t => { - let called = 0 - const [error] = await tryToCatch(() => router.test("post", "/api/link", { - sessionData: { - managedGuilds: ["665289423482519565"] - }, - body: { - discord: "665310973967597573", - matrix: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - guild_id: "665289423482519565" - }, - api: { - async joinRoom(roomID) { - called++ - return roomID - }, - async *generateFullHierarchy(spaceID) { - called++ - t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") - yield { - room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", - children_state: [], - guest_can_join: false, - num_joined_members: 2 - } - /* c8 ignore next */ - }, - async getStateEvent(roomID, type, key) { - called++ - t.equal(roomID, "!NDbIqNpJyPvfKRnNcr:cadence.moe") - t.equal(type, "m.room.power_levels") - t.equal(key, "") - throw new MatrixServerError({errcode: "M_NOT_FOUND", error: "what if I told you there's no such thing as power levels"}) - } - } - })) - t.equal(error.data, "OOYE needs power level 100 (admin) in the target Matrix room") - t.equal(called, 3) -}) - -test("web link room: check that bridge has PL 100 in target room (users default)", async t => { +test("web link room: check that bridge has PL 100 in target room", async t => { let called = 0 const [error] = await tryToCatch(() => router.test("post", "/api/link", { sessionData: { @@ -514,11 +468,28 @@ test("web link room: check that bridge has PL 100 in target room (users default) t.equal(type, "m.room.power_levels") t.equal(key, "") return {users_default: 50} + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!NDbIqNpJyPvfKRnNcr:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@creator:cadence.moe", + room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + content: { + room_version: "11" + } + } } } })) t.equal(error.data, "OOYE needs power level 100 (admin) in the target Matrix room") - t.equal(called, 3) + t.equal(called, 4) }) test("web link room: successfully calls createRoom", async t => { @@ -568,6 +539,23 @@ test("web link room: successfully calls createRoom", async t => { return {} } }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!NDbIqNpJyPvfKRnNcr:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@creator:cadence.moe", + room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + content: { + room_version: "11" + } + } + }, async sendEvent(roomID, type, content) { called++ t.equal(roomID, "!NDbIqNpJyPvfKRnNcr:cadence.moe") @@ -584,7 +572,7 @@ test("web link room: successfully calls createRoom", async t => { } } }) - t.equal(called, 8) + t.equal(called, 9) }) // ***** diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 9ca3c5a3..76c822f7 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -8,20 +8,20 @@ INSERT INTO guild_active (guild_id, autocreate) VALUES INSERT INTO guild_space (guild_id, space_id, privacy_level) VALUES ('112760669178241024', '!jjmvBegULiLucuWEHU:cadence.moe', 0); -INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom_avatar) VALUES -('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL, NULL), -('687028734322147344', '!fGgIymcYWOqjbSRUdV:cadence.moe', 'slow-news-day', NULL, NULL, NULL), -('497161350934560778', '!CzvdIdUQXgUjDVKxeU:cadence.moe', 'amanda-spam', NULL, NULL, NULL), -('160197704226439168', '!hYnGGlPHlbujVVfktC:cadence.moe', 'the-stanley-parable-channel', 'bots', NULL, NULL), -('1100319550446252084', '!BnKuBPCvyfOkhcUjEu:cadence.moe', 'worm-farm', NULL, NULL, NULL), -('1162005314908999790', '!FuDZhlOAtqswlyxzeR:cadence.moe', 'Hey.', NULL, '1100319550446252084', NULL), -('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL), -('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL), -('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS'), -('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL), -('1160894080998461480', '!TqlyQmifxGUggEmdBN:cadence.moe', 'ooyexperiment', NULL, NULL, NULL), -('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL), -('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL); +INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom_avatar, guild_id) VALUES +('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL, NULL, '112760669178241024'), +('687028734322147344', '!fGgIymcYWOqjbSRUdV:cadence.moe', 'slow-news-day', NULL, NULL, NULL, '112760669178241024'), +('497161350934560778', '!CzvdIdUQXgUjDVKxeU:cadence.moe', 'amanda-spam', NULL, NULL, NULL, '66192955777486848'), +('160197704226439168', '!hYnGGlPHlbujVVfktC:cadence.moe', 'the-stanley-parable-channel', 'bots', NULL, NULL, '112760669178241024'), +('1100319550446252084', '!BnKuBPCvyfOkhcUjEu:cadence.moe', 'worm-farm', NULL, NULL, NULL, '66192955777486848'), +('1162005314908999790', '!FuDZhlOAtqswlyxzeR:cadence.moe', 'Hey.', NULL, '1100319550446252084', NULL, '112760669178241024'), +('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL, '66192955777486848'), +('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL, '112760669178241024'), +('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS', '112760669178241024'), +('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL, '66192955777486848'), +('1160894080998461480', '!TqlyQmifxGUggEmdBN:cadence.moe', 'ooyexperiment', NULL, NULL, NULL, '66192955777486848'), +('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL, '665289423482519565'), +('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL, '665289423482519565'); INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) SELECT channel_id, room_id, 0 FROM channel_room; @@ -177,7 +177,7 @@ INSERT INTO reaction (hashed_event_id, message_id, encoded_emoji) VALUES (5162930312280790092, '1141501302736695317', '%F0%9F%90%88'); INSERT INTO member_power (mxid, room_id, power_level) VALUES -('@test_auto_invite:example.org', '*', 100); +('@test_auto_invite:example.org', '*', 150); INSERT INTO lottie (sticker_id, mxc_url) VALUES ('860171525772279849', 'mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR'); From 4bea696a5a2a7f85102944938c72f807bce7bac0 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 7 Jan 2026 15:38:09 +1300 Subject: [PATCH 054/153] setUserPower should account for room version 12 --- scripts/migrate-from-old-bridge.js | 6 +- scripts/remove-old-bridged-users.js | 3 +- src/d2m/actions/create-room.js | 4 +- src/d2m/actions/register-user.js | 22 ++--- src/d2m/converters/edit-to-changes.js | 4 +- .../message-to-event.embeds.test.js | 2 +- src/d2m/converters/message-to-event.js | 4 +- src/d2m/converters/message-to-event.test.js | 2 +- src/d2m/converters/remove-reaction.js | 4 +- src/d2m/converters/thread-to-announcement.js | 4 +- .../converters/thread-to-announcement.test.js | 2 +- src/discord/interactions/permissions.js | 6 +- src/discord/interactions/permissions.test.js | 84 +++++++++++++++---- src/discord/interactions/reactions.js | 4 +- src/m2d/actions/add-reaction.js | 4 +- src/m2d/actions/redact.js | 4 +- src/m2d/converters/event-to-message.js | 4 +- src/m2d/event-dispatcher.js | 4 +- src/matrix/api.js | 51 ----------- src/matrix/kstate.js | 4 +- src/matrix/matrix-command-handler.js | 4 +- src/matrix/room-upgrade.js | 4 +- src/{m2d/converters => matrix}/utils.js | 64 ++++++++++++-- src/{m2d/converters => matrix}/utils.test.js | 14 ++-- src/web/routes/guild.js | 10 ++- src/web/routes/info.js | 4 +- src/web/routes/link.js | 4 +- src/web/server.js | 4 +- test/test.js | 2 +- 29 files changed, 199 insertions(+), 133 deletions(-) rename src/{m2d/converters => matrix}/utils.js (84%) rename src/{m2d/converters => matrix}/utils.test.js (96%) diff --git a/scripts/migrate-from-old-bridge.js b/scripts/migrate-from-old-bridge.js index 36cf884d..40d6993e 100755 --- a/scripts/migrate-from-old-bridge.js +++ b/scripts/migrate-from-old-bridge.js @@ -37,7 +37,9 @@ const createRoom = sync.require("../d2m/actions/create-room") /** @type {import("../src/matrix/mreq")} */ const mreq = sync.require("../matrix/mreq") /** @type {import("../src/matrix/api")} */ -const api = sync.require("../matrix/api") +const api = sync.require("../src/matrix/api") +/** @type {import("../src/matrix/utils")} */ +const utils = sync.require("../src/matrix/utils") const sema = new Semaphore() @@ -89,7 +91,7 @@ async function migrateGuild(guild) { throw e } } - await api.setUserPower(roomID, newBridgeMxid, 100) + await utils.setUserPower(roomID, newBridgeMxid, 100, api) }) await api.joinRoom(roomID) diff --git a/scripts/remove-old-bridged-users.js b/scripts/remove-old-bridged-users.js index d8910bd0..756a492b 100644 --- a/scripts/remove-old-bridged-users.js +++ b/scripts/remove-old-bridged-users.js @@ -10,6 +10,7 @@ const passthrough = require("../src/passthrough") Object.assign(passthrough, {db, sync}) const api = require("../src/matrix/api") +const utils = require("../src/matrix/utils") const mreq = require("../src/matrix/mreq") const rooms = db.prepare("select room_id from channel_room").pluck().all() @@ -25,7 +26,7 @@ const rooms = db.prepare("select room_id from channel_room").pluck().all() await api.leaveRoom(roomID, mxid) } } - await api.setUserPower(roomID, "@_discord_bot:cadence.moe", 0) + await utils.setUserPower(roomID, "@_discord_bot:cadence.moe", 0, api) await api.leaveRoom(roomID) } catch (e) { if (e.message.includes("Appservice not in room")) { diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index c5fdd603..5c2b76c1 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -17,8 +17,8 @@ const mreq = sync.require("../../matrix/mreq") const ks = sync.require("../../matrix/kstate") /** @type {import("../../discord/utils")} */ const dUtils = sync.require("../../discord/utils") -/** @type {import("../../m2d/converters/utils")} */ -const mUtils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const mUtils = sync.require("../../matrix/utils") /** @type {import("./create-space")} */ const createSpace = sync.require("./create-space") diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index 85e8a08e..966263b5 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -12,7 +12,9 @@ const api = sync.require("../../matrix/api") /** @type {import("../../matrix/file")} */ const file = sync.require("../../matrix/file") /** @type {import("../../discord/utils")} */ -const utils = sync.require("../../discord/utils") +const dUtils = sync.require("../../discord/utils") +/** @type {import("../../matrix/utils")} */ +const mxUtils = sync.require("../../matrix/utils") /** @type {import("../converters/user-to-mxid")} */ const userToMxid = sync.require("../converters/user-to-mxid") /** @type {import("./create-room")} */ @@ -159,8 +161,8 @@ async function memberToStateContent(user, member, guildID) { function memberToPowerLevel(user, member, guild, channel) { if (!member) return 0 - const permissions = utils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites) - const everyonePermissions = utils.getPermissions([], guild.roles, undefined, channel.permission_overwrites) + const permissions = dUtils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites) + const everyonePermissions = dUtils.getPermissions([], guild.roles, undefined, channel.permission_overwrites) /* * PL 100 = Administrator = People who can brick the room. RATIONALE: * - Administrator. @@ -169,7 +171,7 @@ function memberToPowerLevel(user, member, guild, channel) { * - Manage Channels: People who can manage the channel can delete it. * (Setting sim users to PL 100 is safe because even though we can't demote the sims we can use code to make the sims demote themselves.) */ - if (guild.owner_id === user.id || utils.hasSomePermissions(permissions, ["Administrator", "ManageWebhooks", "ManageGuild", "ManageChannels"])) return 100 + if (guild.owner_id === user.id || dUtils.hasSomePermissions(permissions, ["Administrator", "ManageWebhooks", "ManageGuild", "ManageChannels"])) return 100 /* * PL 50 = Moderator = People who can manage people and messages in many ways. RATIONALE: * - Manage Messages: Can moderate by pinning or deleting the conversation. @@ -179,14 +181,14 @@ function memberToPowerLevel(user, member, guild, channel) { * - Mute Members & Deafen Members: Can moderate by silencing disruptive people in ways they can't undo. * - Moderate Members. */ - if (utils.hasSomePermissions(permissions, ["ManageMessages", "ManageNicknames", "ManageThreads", "KickMembers", "BanMembers", "MuteMembers", "DeafenMembers", "ModerateMembers"])) return 50 + if (dUtils.hasSomePermissions(permissions, ["ManageMessages", "ManageNicknames", "ManageThreads", "KickMembers", "BanMembers", "MuteMembers", "DeafenMembers", "ModerateMembers"])) return 50 /* PL 50 = if room is read-only but the user has been specially allowed to send messages */ - const everyoneCanSend = utils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendMessages) - const userCanSend = utils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.SendMessages) + const everyoneCanSend = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendMessages) + const userCanSend = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.SendMessages) if (!everyoneCanSend && userCanSend) return createRoom.READ_ONLY_ROOM_EVENTS_DEFAULT_POWER /* PL 20 = Mention Everyone for technical reasons. */ - const everyoneCanMentionEveryone = utils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.MentionEveryone) - const userCanMentionEveryone = utils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.MentionEveryone) + const everyoneCanMentionEveryone = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.MentionEveryone) + const userCanMentionEveryone = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.MentionEveryone) if (!everyoneCanMentionEveryone && userCanMentionEveryone) return 20 return 0 } @@ -247,7 +249,7 @@ async function _sendSyncUser(roomID, mxid, content, powerLevel, options) { actions.push(api.sendState(roomID, "m.room.member", mxid, content, mxid)) // Update power levels if (powerLevel != null) { - actions.push(api.setUserPower(roomID, mxid, powerLevel)) + actions.push(mxUtils.setUserPower(roomID, mxid, powerLevel, api)) } // Update global profile (if supported by server) if (await supportsMsc4069) { diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index 18adc164..8e4c9a2d 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -6,8 +6,8 @@ const passthrough = require("../../passthrough") const {sync, select, from} = passthrough /** @type {import("./message-to-event")} */ const messageToEvent = sync.require("../converters/message-to-event") -/** @type {import("../../m2d/converters/utils")} */ -const utils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") function eventCanBeEdited(ev) { // Discord does not allow files, images, attachments, or videos to be edited. diff --git a/src/d2m/converters/message-to-event.embeds.test.js b/src/d2m/converters/message-to-event.embeds.test.js index cddd4275..85a08cc0 100644 --- a/src/d2m/converters/message-to-event.embeds.test.js +++ b/src/d2m/converters/message-to-event.embeds.test.js @@ -1,7 +1,7 @@ const {test} = require("supertape") const {messageToEvent} = require("./message-to-event") const data = require("../../../test/data") -const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") +const {mockGetEffectivePower} = require("../../matrix/utils.test") const {db} = require("../../passthrough") test("message2event embeds: nothing but a field", async t => { diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 5360702e..85ee969a 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -14,8 +14,8 @@ const file = sync.require("../../matrix/file") const emojiToKey = sync.require("./emoji-to-key") /** @type {import("../actions/lottie")} */ const lottie = sync.require("../actions/lottie") -/** @type {import("../../m2d/converters/utils")} */ -const mxUtils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const mxUtils = sync.require("../../matrix/utils") /** @type {import("../../discord/utils")} */ const dUtils = sync.require("../../discord/utils") const {reg} = require("../../matrix/read-registration") diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index 4b213e49..05ec5bef 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -2,7 +2,7 @@ const {test} = require("supertape") const {messageToEvent} = require("./message-to-event") const {MatrixServerError} = require("../../matrix/mreq") const data = require("../../../test/data") -const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") +const {mockGetEffectivePower} = require("../../matrix/utils.test") const Ty = require("../../types") /** diff --git a/src/d2m/converters/remove-reaction.js b/src/d2m/converters/remove-reaction.js index caa96d16..4ca22b60 100644 --- a/src/d2m/converters/remove-reaction.js +++ b/src/d2m/converters/remove-reaction.js @@ -5,8 +5,8 @@ const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") const {discord, sync, select} = passthrough -/** @type {import("../../m2d/converters/utils")} */ -const utils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") /** * @typedef ReactionRemoveRequest diff --git a/src/d2m/converters/thread-to-announcement.js b/src/d2m/converters/thread-to-announcement.js index 98b8f124..575b3c58 100644 --- a/src/d2m/converters/thread-to-announcement.js +++ b/src/d2m/converters/thread-to-announcement.js @@ -4,8 +4,8 @@ const assert = require("assert").strict const passthrough = require("../../passthrough") const {discord, sync, db, select} = passthrough -/** @type {import("../../m2d/converters/utils")} */ -const mxUtils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const mxUtils = sync.require("../../matrix/utils") const {reg} = require("../../matrix/read-registration.js") const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) diff --git a/src/d2m/converters/thread-to-announcement.test.js b/src/d2m/converters/thread-to-announcement.test.js index 8d011fd0..3286f62d 100644 --- a/src/d2m/converters/thread-to-announcement.test.js +++ b/src/d2m/converters/thread-to-announcement.test.js @@ -2,7 +2,7 @@ const {test} = require("supertape") const {threadToAnnouncement} = require("./thread-to-announcement") const data = require("../../../test/data") const Ty = require("../../types") -const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") +const {mockGetEffectivePower} = require("../../matrix/utils.test") /** * @param {string} roomID diff --git a/src/discord/interactions/permissions.js b/src/discord/interactions/permissions.js index c780a2ae..036947f4 100644 --- a/src/discord/interactions/permissions.js +++ b/src/discord/interactions/permissions.js @@ -9,8 +9,8 @@ const {InteractionMethods} = require("snowtransfer") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") -/** @type {import("../../m2d/converters/utils")} */ -const utils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") /** * @param {DiscordTypes.APIContextMenuGuildInteraction} interaction @@ -126,7 +126,7 @@ async function* _interactEdit({data, guild_id, message}, {api}) { assert(spaceID) // Do it - await api.setUserPowerCascade(spaceID, mxid, power) + await utils.setUserPowerCascade(spaceID, mxid, power, api) // ACK yield {editOriginalInteractionResponse: { diff --git a/src/discord/interactions/permissions.test.js b/src/discord/interactions/permissions.test.js index ef3fef2d..5a078b52 100644 --- a/src/discord/interactions/permissions.test.js +++ b/src/discord/interactions/permissions.test.js @@ -2,7 +2,7 @@ const {test} = require("supertape") const DiscordTypes = require("discord-api-types/v10") const {select, db} = require("../../passthrough") const {_interact, _interactEdit} = require("./permissions") -const {mockGetEffectivePower} = require("../../m2d/converters/utils.test") +const {mockGetEffectivePower} = require("../../matrix/utils.test") /** * @template T @@ -156,7 +156,7 @@ test("permissions: reports permissions of selected matrix user (admin v11 cannot }) test("permissions: can update user to moderator", async t => { - let called = 0 + let called = [] const msgs = await fromAsync(_interactEdit({ data: { target_id: "1128118177155526666", @@ -168,22 +168,48 @@ test("permissions: can update user to moderator", async t => { guild_id: "112760669178241024" }, { api: { - async setUserPowerCascade(roomID, mxid, power) { - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") // space ID - t.equal(mxid, "@cadence:cadence.moe") - t.equal(power, 50) + async getStateEvent(roomID, type, key) { + called.push("get power levels") + t.equal(type, "m.room.power_levels") + return {} + }, + async getStateEventOuter(roomID, type, key) { + called.push("get room create") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + room_id: roomID, + content: { + room_version: "11" + } + } + }, + async *generateFullHierarchy(spaceID) { + called.push("generate full hierarchy") + }, + async sendState(roomID, type, key, content) { + called.push("set power levels") + t.ok(["!hierarchy", "!jjmvBegULiLucuWEHU:cadence.moe"].includes(roomID), `expected room ID to be in hierarchy, but was ${roomID}`) + t.equal(type, "m.room.power_levels") + t.equal(key, "") + t.deepEqual(content, { + users: {"@cadence:cadence.moe": 50} + }) + return "$updated" } } })) t.equal(msgs.length, 2) t.equal(msgs[0].createInteractionResponse.data.content, "Updating `@cadence:cadence.moe` to **moderator**, please wait...") t.equal(msgs[1].editOriginalInteractionResponse.content, "Updated `@cadence:cadence.moe` to **moderator**.") - t.equal(called, 1) + t.deepEqual(called, ["generate full hierarchy", "get room create", "get power levels", "set power levels"]) }) test("permissions: can update user to default", async t => { - let called = 0 + let called = [] const msgs = await fromAsync(_interactEdit({ data: { target_id: "1128118177155526666", @@ -195,16 +221,44 @@ test("permissions: can update user to default", async t => { guild_id: "112760669178241024" }, { api: { - async setUserPowerCascade(roomID, mxid, power) { - called++ - t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") // space ID - t.equal(mxid, "@cadence:cadence.moe") - t.equal(power, 0) + async getStateEvent(roomID, type, key) { + called.push("get power levels") + t.equal(type, "m.room.power_levels") + return { + users: {"@cadence:cadence.moe": 50} + } + }, + async getStateEventOuter(roomID, type, key) { + called.push("get room create") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + room_id: roomID, + content: { + room_version: "11" + } + } + }, + async *generateFullHierarchy(spaceID) { + called.push("generate full hierarchy") + }, + async sendState(roomID, type, key, content) { + called.push("set power levels") + t.ok(["!hierarchy", "!jjmvBegULiLucuWEHU:cadence.moe"].includes(roomID), `expected room ID to be in hierarchy, but was ${roomID}`) + t.equal(type, "m.room.power_levels") + t.equal(key, "") + t.deepEqual(content, { + users: {} + }) + return "$updated" } } })) t.equal(msgs.length, 2) t.equal(msgs[0].createInteractionResponse.data.content, "Updating `@cadence:cadence.moe` to **default**, please wait...") t.equal(msgs[1].editOriginalInteractionResponse.content, "Updated `@cadence:cadence.moe` to **default**.") - t.equal(called, 1) + t.deepEqual(called, ["generate full hierarchy", "get room create", "get power levels", "set power levels"]) }) diff --git a/src/discord/interactions/reactions.js b/src/discord/interactions/reactions.js index 59bf065a..bd2f8560 100644 --- a/src/discord/interactions/reactions.js +++ b/src/discord/interactions/reactions.js @@ -7,8 +7,8 @@ const {InteractionMethods} = require("snowtransfer") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") -/** @type {import("../../m2d/converters/utils")} */ -const utils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") /** * @param {DiscordTypes.APIMessageApplicationCommandGuildInteraction} interaction diff --git a/src/m2d/actions/add-reaction.js b/src/m2d/actions/add-reaction.js index 268888c3..2b19fb23 100644 --- a/src/m2d/actions/add-reaction.js +++ b/src/m2d/actions/add-reaction.js @@ -5,8 +5,8 @@ const Ty = require("../../types") const passthrough = require("../../passthrough") const {discord, sync, db, select} = passthrough -/** @type {import("../converters/utils")} */ -const utils = sync.require("../converters/utils") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") /** @type {import("../converters/emoji")} */ const emoji = sync.require("../converters/emoji") diff --git a/src/m2d/actions/redact.js b/src/m2d/actions/redact.js index 9b26e8e4..9f99ec13 100644 --- a/src/m2d/actions/redact.js +++ b/src/m2d/actions/redact.js @@ -5,8 +5,8 @@ const Ty = require("../../types") const passthrough = require("../../passthrough") const {discord, sync, db, select, from} = passthrough -/** @type {import("../converters/utils")} */ -const utils = sync.require("../converters/utils") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") /** * @param {Ty.Event.Outer_M_Room_Redaction} event diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index a030ac5f..785243d3 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -14,8 +14,8 @@ const {tag} = require("@cloudrac3r/html-template-tag") const passthrough = require("../../passthrough") const {sync, db, discord, select, from} = passthrough const {reg} = require("../../matrix/read-registration") -/** @type {import("../converters/utils")} */ -const mxUtils = sync.require("../converters/utils") +/** @type {import("../../matrix/utils")} */ +const mxUtils = sync.require("../../matrix/utils") /** @type {import("../../discord/utils")} */ const dUtils = sync.require("../../discord/utils") /** @type {import("../../matrix/file")} */ diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index b3269620..13c0af1a 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -20,8 +20,8 @@ const redact = sync.require("./actions/redact") const updatePins = sync.require("./actions/update-pins") /** @type {import("../matrix/matrix-command-handler")} */ const matrixCommandHandler = sync.require("../matrix/matrix-command-handler") -/** @type {import("./converters/utils")} */ -const utils = sync.require("./converters/utils") +/** @type {import("../matrix/utils")} */ +const utils = sync.require("../matrix/utils") /** @type {import("../matrix/api")}) */ const api = sync.require("../matrix/api") /** @type {import("../d2m/actions/create-room")} */ diff --git a/src/matrix/api.js b/src/matrix/api.js index 824f13b8..01f9c3be 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -358,55 +358,6 @@ async function profileSetAvatarUrl(mxid, avatar_url, inhibitPropagate) { } } -/** - * Set a user's power level within a room. - * @param {string} roomID - * @param {string} mxid - * @param {number} newPower - */ -async function setUserPower(roomID, mxid, newPower) { - assert(roomID[0] === "!") - assert(mxid[0] === "@") - // Yes there's no shortcut https://github.com/matrix-org/matrix-appservice-bridge/blob/2334b0bae28a285a767fe7244dad59f5a5963037/src/components/intent.ts#L352 - const power = await getStateEvent(roomID, "m.room.power_levels", "") - power.users = power.users || {} - - // Check if it has really changed to avoid sending a useless state event - // (Can't diff kstate here because of (a) circular imports (b) kstate has special behaviour diffing power levels) - const oldPowerLevel = power.users?.[mxid] ?? power.users_default ?? 0 - if (oldPowerLevel === newPower) return - - // Bridge bot can't demote equal power users, so need to decide which user will send the event - const botPowerLevel = power.users?.[`@${reg.sender_localpart}:${reg.ooye.server_name}`] ?? power.users_default ?? 0 - const eventSender = oldPowerLevel >= botPowerLevel ? mxid : undefined - - // Update the event content - if (newPower == null || newPower === (power.users_default ?? 0)) { - delete power.users[mxid] - } else { - power.users[mxid] = newPower - } - - await sendState(roomID, "m.room.power_levels", "", power, eventSender) - return power -} - -/** - * Set a user's power level for a whole room hierarchy. - * @param {string} spaceID - * @param {string} mxid - * @param {number} power - */ -async function setUserPowerCascade(spaceID, mxid, power) { - assert(spaceID[0] === "!") - assert(mxid[0] === "@") - const rooms = await getFullHierarchy(spaceID) - await setUserPower(spaceID, mxid, power) - for (const room of rooms) { - await setUserPower(room.room_id, mxid, power) - } -} - async function ping() { // not using mreq so that we can read the status code const res = await fetch(`${mreq.baseUrl}/client/v1/appservice/${reg.id}/ping`, { @@ -579,8 +530,6 @@ module.exports.redactEvent = redactEvent module.exports.sendTyping = sendTyping module.exports.profileSetDisplayname = profileSetDisplayname module.exports.profileSetAvatarUrl = profileSetAvatarUrl -module.exports.setUserPower = setUserPower -module.exports.setUserPowerCascade = setUserPowerCascade module.exports.ping = ping module.exports.getMedia = getMedia module.exports.sendReadReceipt = sendReadReceipt diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index ace9c36a..85a28380 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -10,8 +10,8 @@ const {sync} = passthrough const file = sync.require("./file") /** @type {import("./api")} */ const api = sync.require("./api") -/** @type {import("../m2d/converters/utils")} */ -const utils = sync.require("../m2d/converters/utils") +/** @type {import("./utils")} */ +const utils = sync.require("./utils") /** Mutates the input. Not recursive - can only include or exclude entire state events. */ function kstateStripConditionals(kstate) { diff --git a/src/matrix/matrix-command-handler.js b/src/matrix/matrix-command-handler.js index 601b2dca..e30ae6ff 100644 --- a/src/matrix/matrix-command-handler.js +++ b/src/matrix/matrix-command-handler.js @@ -8,8 +8,8 @@ const sharp = require("sharp") const {discord, sync, db, select} = require("../passthrough") /** @type {import("./api")}) */ const api = sync.require("./api") -/** @type {import("../m2d/converters/utils")} */ -const mxUtils = sync.require("../m2d/converters/utils") +/** @type {import("./utils")} */ +const mxUtils = sync.require("./utils") /** @type {import("../discord/utils")} */ const dUtils = sync.require("../discord/utils") /** @type {import("./kstate")} */ diff --git a/src/matrix/room-upgrade.js b/src/matrix/room-upgrade.js index 0b02762a..7f0d4e61 100644 --- a/src/matrix/room-upgrade.js +++ b/src/matrix/room-upgrade.js @@ -10,8 +10,8 @@ const {discord, db, sync, as, select, from} = require("../passthrough") const api = sync.require("./api") /** @type {import("../d2m/actions/create-room")}) */ const createRoom = sync.require("../d2m/actions/create-room") -/** @type {import("../m2d/converters/utils")}) */ -const utils = sync.require("../m2d/converters/utils") +/** @type {import("./utils")}) */ +const utils = sync.require("./utils") const roomUpgradeSema = new Semaphore() diff --git a/src/m2d/converters/utils.js b/src/matrix/utils.js similarity index 84% rename from src/m2d/converters/utils.js rename to src/matrix/utils.js index ccdef831..3860f6e3 100644 --- a/src/m2d/converters/utils.js +++ b/src/matrix/utils.js @@ -1,11 +1,11 @@ // @ts-check const assert = require("assert").strict -const Ty = require("../../types") -const passthrough = require("../../passthrough") +const Ty = require("../types") +const passthrough = require("../passthrough") const {db} = passthrough -const {reg} = require("../../matrix/read-registration") +const {reg} = require("./read-registration") const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) /** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore @@ -129,7 +129,7 @@ class MatrixStringBuilder { * https://spec.matrix.org/v1.9/appendices/#routing * https://gitdab.com/cadence/out-of-your-element/issues/11 * @param {string} roomID - * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("../../matrix/api")[K]} | {getEffectivePower: (roomID: string, mxids: string[], api: any) => Promise<{powers: Record, allCreators: string[], tombstone: number, roomCreate: Ty.Event.StateOuter, powerLevels: Ty.Event.M_Power_Levels}>, getJoinedMembers: import("../../matrix/api")["getJoinedMembers"]}} api + * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("./api")[K]} | {getEffectivePower: (roomID: string, mxids: string[], api: any) => Promise<{powers: Record, allCreators: string[], tombstone: number, roomCreate: Ty.Event.StateOuter, powerLevels: Ty.Event.M_Power_Levels}>, getJoinedMembers: import("./api")["getJoinedMembers"]}} api */ async function getViaServers(roomID, api) { const candidates = [] @@ -188,7 +188,7 @@ async function getViaServers(roomID, api) { * https://spec.matrix.org/v1.9/appendices/#routing * https://gitdab.com/cadence/out-of-your-element/issues/11 * @param {string} roomID - * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api + * @param {{[K in "getStateEvent" | "getStateEventOuter" | "getJoinedMembers"]: import("./api")[K]}} api * @returns {Promise} */ async function getViaServersQuery(roomID, api) { @@ -273,7 +273,7 @@ function removeCreatorsFromPowerLevels(roomCreateOuter, powerLevels) { * @template {string} T * @param {string} roomID * @param {T[]} mxids - * @param {{[K in "getStateEvent" | "getStateEventOuter"]: import("../../matrix/api")[K]}} api + * @param {{[K in "getStateEvent" | "getStateEventOuter"]: import("./api")[K]}} api * @returns {Promise<{powers: Record, allCreators: string[], tombstone: number, roomCreate: Ty.Event.StateOuter, powerLevels: Ty.Event.M_Power_Levels}>} */ async function getEffectivePower(roomID, mxids, api) { @@ -300,6 +300,56 @@ async function getEffectivePower(roomID, mxids, api) { return {powers, allCreators, tombstone, roomCreate, powerLevels} } +/** + * Set a user's power level within a room. + * @param {string} roomID + * @param {string} mxid + * @param {number} newPower + * @param {{[K in "getStateEvent" | "getStateEventOuter" | "sendState"]: import("./api")[K]}} api + */ +async function setUserPower(roomID, mxid, newPower, api) { + assert(roomID[0] === "!") + assert(mxid[0] === "@") + // Yes there's no shortcut https://github.com/matrix-org/matrix-appservice-bridge/blob/2334b0bae28a285a767fe7244dad59f5a5963037/src/components/intent.ts#L352 + const {powerLevels, powers: {[mxid]: oldPowerLevel, [bot]: botPowerLevel}} = await getEffectivePower(roomID, [mxid, bot], api) + + // Check if it has really changed to avoid sending a useless state event + if (oldPowerLevel === newPower) return + + // Bridge bot can't demote equal power users, so need to decide which user will send the event + const eventSender = oldPowerLevel >= botPowerLevel ? mxid : undefined + + // Update the event content + powerLevels.users ??= {} + if (newPower == null || newPower === (powerLevels.users_default ?? 0)) { + delete powerLevels.users[mxid] + } else { + powerLevels.users[mxid] = newPower + } + + await api.sendState(roomID, "m.room.power_levels", "", powerLevels, eventSender) +} + +/** + * Set a user's power level for a whole room hierarchy. + * @param {string} spaceID + * @param {string} mxid + * @param {number} power + * @param {{[K in "getStateEvent" | "getStateEventOuter" | "sendState" | "generateFullHierarchy"]: import("./api")[K]}} api + */ +async function setUserPowerCascade(spaceID, mxid, power, api) { + assert(spaceID[0] === "!") + assert(mxid[0] === "@") + let seenSpace = false + for await (const room of api.generateFullHierarchy(spaceID)) { + if (room.room_id === spaceID) seenSpace = true + await setUserPower(room.room_id, mxid, power, api) + } + if (!seenSpace) { + await setUserPower(spaceID, mxid, power, api) + } +} + module.exports.bot = bot module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord @@ -311,3 +361,5 @@ module.exports.getViaServersQuery = getViaServersQuery module.exports.roomHasAtLeastVersion = roomHasAtLeastVersion module.exports.removeCreatorsFromPowerLevels = removeCreatorsFromPowerLevels module.exports.getEffectivePower = getEffectivePower +module.exports.setUserPower = setUserPower +module.exports.setUserPowerCascade = setUserPowerCascade diff --git a/src/m2d/converters/utils.test.js b/src/matrix/utils.test.js similarity index 96% rename from src/m2d/converters/utils.test.js rename to src/matrix/utils.test.js index 9c113938..0ecd41ed 100644 --- a/src/m2d/converters/utils.test.js +++ b/src/matrix/utils.test.js @@ -1,7 +1,5 @@ // @ts-check -const e = new Error("Custom error") - const {test} = require("supertape") const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers, roomHasAtLeastVersion} = require("./utils") const util = require("util") @@ -41,8 +39,14 @@ test("event hash: hash is different for different inputs", t => { }) test("MatrixStringBuilder: add, addLine, add same text", t => { + const e = { + stack: "Error: Custom error\n at ./example.test.js:3:11)", + toString() { + return "Error: Custom error" + } + } const gatewayMessage = {t: "MY_MESSAGE", d: {display: "Custom message data"}} - let stackLines = e.stack?.split("\n") + let stackLines = e.stack.split("\n") const builder = new MatrixStringBuilder() builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 Bridged event from Discord not delivered") @@ -63,12 +67,12 @@ test("MatrixStringBuilder: add, addLine, add same text", t => { + "\nError: Custom error" + "\nError trace:" + "\nError: Custom error" - + "\n at ./m2d/converters/utils.test.js:3:11)\n", + + "\n at ./example.test.js:3:11)\n", format: "org.matrix.custom.html", formatted_body: "\u26a0 Bridged event from Discord not delivered" + "
Gateway event: MY_MESSAGE" + "
Error: Custom error" - + "
Error trace
Error: Custom error\n    at ./m2d/converters/utils.test.js:3:11)
" + + "
Error trace
Error: Custom error\n    at ./example.test.js:3:11)
" + `
Original payload
{ display: 'Custom message data' }
` }) }) diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index 8c2d99db..6b80e9df 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -18,7 +18,9 @@ const createSpace = sync.require("../../d2m/actions/create-space") /** @type {import("../auth")} */ const auth = require("../auth") /** @type {import("../../discord/utils")} */ -const utils = sync.require("../../discord/utils") +const dUtils = sync.require("../../discord/utils") +/** @type {import("../../matrix/utils")} */ +const mxUtils = sync.require("../../matrix/utils") const {reg} = require("../../matrix/read-registration") const schema = { @@ -102,8 +104,8 @@ function getChannelRoomsLinks(guild, rooms, roles) { let unlinkedChannels = unlinkedChannelIDs.map(c => discord.channels.get(c)) let removedWrongTypeChannels = filterTo(unlinkedChannels, c => c && [0, 5].includes(c.type)) let removedPrivateChannels = filterTo(unlinkedChannels, c => { - const permissions = utils.getPermissions(roles, guild.roles, botID, c["permission_overwrites"]) - return utils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) + const permissions = dUtils.getPermissions(roles, guild.roles, botID, c["permission_overwrites"]) + return dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) }) unlinkedChannels.sort((a, b) => getPosition(a) - getPosition(b)) @@ -228,7 +230,7 @@ as.router.post("/api/invite", defineEventHandler(async event => { ( parsedBody.permissions === "admin" ? 100 : parsedBody.permissions === "moderator" ? 50 : 0) - if (powerLevel) await api.setUserPowerCascade(spaceID, parsedBody.mxid, powerLevel) + if (powerLevel) await mxUtils.setUserPowerCascade(spaceID, parsedBody.mxid, powerLevel, api) if (parsedBody.guild_id) { setResponseHeader(event, "HX-Refresh", true) diff --git a/src/web/routes/info.js b/src/web/routes/info.js index 9c202fa8..0c3e3b11 100644 --- a/src/web/routes/info.js +++ b/src/web/routes/info.js @@ -4,8 +4,8 @@ const {z} = require("zod") const {defineEventHandler, getValidatedQuery, H3Event} = require("h3") const {as, from, sync, select} = require("../../passthrough") -/** @type {import("../../m2d/converters/utils")} */ -const mUtils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")} */ +const mUtils = sync.require("../../matrix/utils") /** * @param {H3Event} event diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 17f27df6..5193dedb 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -10,8 +10,8 @@ const {discord, db, as, sync, select, from} = require("../../passthrough") const auth = sync.require("../auth") /** @type {import("../../matrix/mreq")} */ const mreq = sync.require("../../matrix/mreq") -/** @type {import("../../m2d/converters/utils")}*/ -const utils = sync.require("../../m2d/converters/utils") +/** @type {import("../../matrix/utils")}*/ +const utils = sync.require("../../matrix/utils") const {reg} = require("../../matrix/read-registration") /** diff --git a/src/web/server.js b/src/web/server.js index 7c8ed3e4..3cb3060c 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -13,8 +13,8 @@ const reg = require("../matrix/read-registration") const {sync, discord, as, select} = require("../passthrough") /** @type {import("./pug-sync")} */ const pugSync = sync.require("./pug-sync") -/** @type {import("../m2d/converters/utils")} */ -const mUtils = sync.require("../m2d/converters/utils") +/** @type {import("../matrix/utils")} */ +const mUtils = sync.require("../matrix/utils") const {id} = require("../../addbot") // Pug diff --git a/test/test.js b/test/test.js index 2c3902a0..6e2c97b4 100644 --- a/test/test.js +++ b/test/test.js @@ -147,6 +147,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/matrix/mreq.test") require("../src/matrix/read-registration.test") require("../src/matrix/txnid.test") + require("../src/matrix/utils.test") require("../src/d2m/actions/create-room.test") require("../src/d2m/actions/create-space.test") require("../src/d2m/actions/register-user.test") @@ -164,7 +165,6 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/m2d/converters/diff-pins.test") require("../src/m2d/converters/event-to-message.test") require("../src/m2d/converters/emoji.test") - require("../src/m2d/converters/utils.test") require("../src/m2d/converters/emoji-sheet.test") require("../src/discord/interactions/invite.test") require("../src/discord/interactions/matrix-info.test") From 420258422bf14ea272bb3a1e8882c6be5e419aea Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 8 Jan 2026 20:09:43 +1300 Subject: [PATCH 055/153] Use channel overwrites when checking embeddability --- src/m2d/actions/send-event.js | 8 +- src/m2d/converters/event-to-message.js | 15 +- src/m2d/converters/event-to-message.test.js | 192 ++++++++++++++------ 3 files changed, 151 insertions(+), 64 deletions(-) diff --git a/src/m2d/actions/send-event.js b/src/m2d/actions/send-event.js index 7ed70c72..ddf82f96 100644 --- a/src/m2d/actions/send-event.js +++ b/src/m2d/actions/send-event.js @@ -69,16 +69,16 @@ async function sendEvent(event) { threadID = channelID channelID = row.thread_parent // it's the thread's parent... get with the times... } - // @ts-ignore - const guildID = discord.channels.get(channelID).guild_id - const guild = discord.guilds.get(guildID) + /** @type {DiscordTypes.APIGuildTextChannel} */ // @ts-ignore + const channel = discord.channels.get(channelID) + const guild = discord.guilds.get(channel.guild_id) assert(guild) const historicalRoomIndex = select("historical_channel_room", "historical_room_index", {room_id: event.room_id}).pluck().get() assert(historicalRoomIndex) // no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it - let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, mxcDownloader: emojiSheet.getAndConvertEmoji}) + let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, channel, {api, snow: discord.snow, mxcDownloader: emojiSheet.getAndConvertEmoji}) messagesToEdit = await Promise.all(messagesToEdit.map(async e => { e.message = await resolvePendingFiles(e.message) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 785243d3..273521ce 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -518,10 +518,11 @@ async function getL1L2ReplyLine(called = false) { /** * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event - * @param {import("discord-api-types/v10").APIGuild} guild + * @param {DiscordTypes.APIGuild} guild + * @param {DiscordTypes.APIGuildTextChannel} channel * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, mxcDownloader: (mxc: string) => Promise}} di simple-as-nails dependency injection for the matrix API */ -async function eventToMessage(event, guild, di) { +async function eventToMessage(event, guild, channel, di) { let displayName = event.sender let avatarURL = undefined const allowedMentionsParse = ["users", "roles"] @@ -918,17 +919,19 @@ async function eventToMessage(event, guild, di) { content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible let offset = 0 - for (const match of [...content.matchAll(/\bhttps?:\/\/[^ )>]*/g)]) { + for (const match of [...content.matchAll(/\bhttps?:\/\/[^ )>\n]+/g)]) { assert(typeof match.index === "number") // Respect sender's angle brackets const alreadySuppressed = content[match.index-1+offset] === "<" && content[match.index+match.length+offset] === ">" if (alreadySuppressed) continue - // Put < > around any surviving matrix.to links + + // Suppress matrix.to links always let shouldSuppress = !!match[0].match(/^https?:\/\/matrix\.to\//) + + // Suppress if regular users don't have permission if (!shouldSuppress && guild?.roles) { - // Suppress if regular users don't have permission - const permissions = dUtils.getPermissions([], guild.roles) + const permissions = dUtils.getPermissions([], guild.roles, undefined, channel.permission_overwrites) const canEmbedLinks = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.EmbedLinks) shouldSuppress = !canEmbedLinks } diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index b298f5b1..5cdf4aff 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -115,7 +115,7 @@ test("event2message: any markdown in body is escaped, except strikethrough", asy unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { snow: { guild: { searchGuildMembers: () => [] @@ -303,7 +303,7 @@ test("event2message: markdown in link text does not attempt to be escaped becaus ) }) -test("event2message: links are escaped if the guild does not have embed links permission (formatted body)", async t => { +test("event2message: embeds are suppressed if the guild does not have embed links permission (formatted body)", async t => { t.deepEqual( await eventToMessage({ content: { @@ -341,7 +341,7 @@ test("event2message: links are escaped if the guild does not have embed links pe ) }) -test("event2message: links are escaped if the guild does not have embed links permission (plaintext body)", async t => { +test("event2message: embeds are suppressed if the guild does not have embed links permission (plaintext body)", async t => { t.deepEqual( await eventToMessage({ content: { @@ -360,6 +360,49 @@ test("event2message: links are escaped if the guild does not have embed links pe name: "@everyone", permissions: DiscordTypes.PermissionFlagsBits.SendMessages }] + }, {}), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "posting one of my favourite songs recently (starts at timestamp) ", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: embeds are suppressed if the channel does not have embed links permission (plaintext body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "posting one of my favourite songs recently (starts at timestamp) https://youtu.be/RhV2X7WQMPA?t=364", + msgtype: "m.text" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + }, { + id: "123", + roles: [{ + id: "123", + name: "@everyone", + permissions: DiscordTypes.PermissionFlagsBits.SendMessages | DiscordTypes.PermissionFlagsBits.EmbedLinks + }] + }, { + permission_overwrites: [{ + id: "123", + type: 0, + deny: String(DiscordTypes.PermissionFlagsBits.EmbedLinks), + allow: "0" + }] }), { ensureJoined: [], @@ -437,6 +480,47 @@ test("event2message: links retain angle brackets (plaintext body)", async t => { ) }) +test("event2message: links don't have angle brackets added by accident", async t => { + t.deepEqual( + await eventToMessage({ + "content": { + "body": "Wanted to automate WG→AWG config enrichment and ended up basically coding a batch INI processor.\nhttps://github.com/Erquint/wgcbp", + "m.mentions": {}, + "msgtype": "m.text" + }, + "origin_server_ts": 1767848218369, + "sender": "@erquint:agiadn.org", + "type": "m.room.message", + "unsigned": { + "membership": "join" + }, + "event_id": "$DxPjyI88VYsJGKuGmhFivFeKn-i5MEBEnAhabmsBaXQ", + "room_id": "!zq94fae5bVKUubZLp7:agiadn.org" + }, {}, {}, { + api: { + async getStateEvent(roomID, type, key) { + return { + displayname: "Erquint" + } + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "Erquint", + content: "Wanted to automate WG→AWG config enrichment and ended up basically coding a batch INI processor.\nhttps://github.com/Erquint/wgcbp", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + test("event2message: basic html is converted to markdown", async t => { t.deepEqual( await eventToMessage({ @@ -1316,7 +1400,7 @@ test("event2message: rich reply to a sim user", async t => { }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { type: "m.room.message", @@ -1366,7 +1450,7 @@ test("event2message: rich reply to a rich reply to a multi-line message should c unsigned: {}, event_id: "$Q5kNrPxGs31LfWOhUul5I03jNjlxKOwRmWVuivaqCHY", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!kLRqKKUQXcibIMtOpl:cadence.moe", "$A0Rj559NKOh2VndCZSTJXcvgi42gZWVfVQt73wA2Hn0", { "type": "m.room.message", @@ -1441,7 +1525,7 @@ test("event2message: rich reply to an already-edited message will quote the new }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$DSQvWxOBB2DYaei6b83-fb33dQGYt5LJd_s8Nl2a43Q", { type: "m.room.message", @@ -1524,7 +1608,7 @@ test("event2message: rich reply to a missing event will quote from formatted_bod }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { async getEvent(roomID, eventID) { called++ @@ -1574,7 +1658,7 @@ test("event2message: rich reply to a missing event without formatted_body will u }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { async getEvent(roomID, eventID) { called++ @@ -1625,7 +1709,7 @@ test("event2message: rich reply to a missing event and no reply fallback will no }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { async getEvent(roomID, eventID) { called++ @@ -1670,7 +1754,7 @@ test("event2message: should avoid using blockquote contents as reply preview in }, event_id: "$BpGx8_vqHyN6UQDARPDU51ftrlRBhleutRSgpAJJ--g", room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { "type": "m.room.message", @@ -1721,7 +1805,7 @@ test("event2message: should suppress embeds for links in reply preview", async t }, event_id: "$0Bs3rbsXaeZmSztGMx1NIsqvOrkXOpIWebN-dqr09i4", room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$qmyjr-ISJtnOM5WTWLI0fT7uSlqRLgpyin2d2NCglCU", { "type": "m.room.message", @@ -1770,7 +1854,7 @@ test("event2message: should include a reply preview when message ends with a blo }, event_id: "$n6sg1X9rLeMzCYufJTRvaLzFeLQ-oEXjCWkHtRxcem4", room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$uXM2I6w-XMtim14-OSZ_8Z2uQ6MDAZLT37eYIiEU6KQ", { type: 'm.room.message', @@ -1859,7 +1943,7 @@ test("event2message: should include a reply preview when replying to a descripti }, event_id: "$qCOlszCawu5hlnF2a2PGyXeGGvtoNJdXyRAEaTF0waA", room_id: "!CzvdIdUQXgUjDVKxeU:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!CzvdIdUQXgUjDVKxeU:cadence.moe", "$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ", { type: "m.room.message", @@ -1944,7 +2028,7 @@ test("event2message: entities are not escaped in main message or reply preview", }, event_id: "$2I7odT9okTdpwDcqOjkJb_A3utdO4V8Cp3LK6-Rvwcs", room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$yIWjZPi6Xk56fBxJwqV4ANs_hYLjnWI2cNKbZ2zwk60", { type: "m.room.message", @@ -1996,7 +2080,7 @@ test("event2message: reply preview converts emoji formatting when replying to a }, event_id: "$bCMLaLiMfoRajaGTgzaxAci-g8hJfkspVJIKwYktnvc", room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$zmO-dtPO6FubBkDxJZ5YmutPIsG1RgV5JJku-9LeGWs", { type: "m.room.message", @@ -2046,7 +2130,7 @@ test("event2message: reply preview can guess custom emoji based on the name if i }, event_id: "$bCMLaLiMfoRajaGTgzaxAci-g8hJfkspVJIKwYktnvc", room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$zmO-dtPO6FubBkDxJZ5YmutPIsG1RgV5JJku-9LeGWs", { type: "m.room.message", @@ -2096,7 +2180,7 @@ test("event2message: reply preview uses emoji title text when replying to an unk }, event_id: "$bCMLaLiMfoRajaGTgzaxAci-g8hJfkspVJIKwYktnvc", room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$zmO-dtPO6FubBkDxJZ5YmutPIsG1RgV5JJku-9LeGWs", { type: "m.room.message", @@ -2146,7 +2230,7 @@ test("event2message: reply preview ignores garbage image", async t => { }, event_id: "$bCMLaLiMfoRajaGTgzaxAci-g8hJfkspVJIKwYktnvc", room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$zmO-dtPO6FubBkDxJZ5YmutPIsG1RgV5JJku-9LeGWs", { type: "m.room.message", @@ -2196,7 +2280,7 @@ test("event2message: reply to empty message doesn't show an extra line or anythi }, event_id: "$bCMLaLiMfoRajaGTgzaxAci-g8hJfkspVJIKwYktnvc", room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$zmO-dtPO6FubBkDxJZ5YmutPIsG1RgV5JJku-9LeGWs", { type: "m.room.message", @@ -2256,7 +2340,7 @@ test("event2message: editing a rich reply to a sim user", async t => { }, "event_id": "$XEgssz13q-a7NLO7UZO2Oepq7tSiDBD7YRfr7Xu_QiA", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: (roomID, eventID) => { assert.ok(eventID === "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04" || eventID === "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8") @@ -2337,7 +2421,7 @@ test("event2message: editing a plaintext body message", async t => { }, "event_id": "$KxGwvVNzNcmlVbiI2m5kX-jMFNi3Jle71-uu1j7P7vM", "room_id": "!BnKuBPCvyfOkhcUjEu:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!BnKuBPCvyfOkhcUjEu:cadence.moe", "$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs", { type: "m.room.message", @@ -2392,7 +2476,7 @@ test("event2message: editing a plaintext message to be longer", async t => { }, "event_id": "$KxGwvVNzNcmlVbiI2m5kX-jMFNi3Jle71-uu1j7P7vM", "room_id": "!BnKuBPCvyfOkhcUjEu:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!BnKuBPCvyfOkhcUjEu:cadence.moe", "$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs", { type: "m.room.message", @@ -2454,7 +2538,7 @@ test("event2message: editing a plaintext message to be shorter", async t => { }, "event_id": "$KxGwvVNzNcmlVbiI2m5kX-jMFNi3Jle71-uu1j7P7vM", "room_id": "!BnKuBPCvyfOkhcUjEu:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!BnKuBPCvyfOkhcUjEu:cadence.moe", "$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt", { type: "m.room.message", @@ -2513,7 +2597,7 @@ test("event2message: editing a formatted body message", async t => { }, "event_id": "$KxGwvVNzNcmlVbiI2m5kX-jMFNi3Jle71-uu1j7P7vM", "room_id": "!BnKuBPCvyfOkhcUjEu:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!BnKuBPCvyfOkhcUjEu:cadence.moe", "$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs", { type: "m.room.message", @@ -2569,7 +2653,7 @@ test("event2message: rich reply to a matrix user's long message with formatting" }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { "type": "m.room.message", @@ -2624,7 +2708,7 @@ test("event2message: rich reply to an image", async t => { }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { type: "m.room.message", @@ -2686,7 +2770,7 @@ test("event2message: rich reply to a spoiler should ensure the spoiler is hidden }, "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { type: "m.room.message", @@ -2737,7 +2821,7 @@ test("event2message: with layered rich replies, the preview should only be the r }, event_id: "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { "type": "m.room.message", @@ -2798,7 +2882,7 @@ test("event2message: if event is a reply and starts with a quote, they should be }, room_id: "!TqlyQmifxGUggEmdBN:cadence.moe", event_id: "$nCvtZeBFedYuEavt4OftloCHc0kaFW2ktHCfIOklhjU", - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$tTYQcke93fwocsc1K6itwUq85EG0RZ0ksCuIglKioks", { sender: "@aflower:syndicated.gay", @@ -2849,7 +2933,7 @@ test("event2message: rich reply to a deleted event", async t => { }, event_id: "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$f-noT-d-Eo_Xgpc05Ww89ErUXku4NwKWYGHLzWKo1kU", { type: "m.room.message", @@ -2907,7 +2991,7 @@ test("event2message: rich reply to a state event with no body", async t => { }, event_id: "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$f-noT-d-Eo_Xgpc05Ww89ErUXku4NwKWYGHLzWKo1kU", { type: "m.room.topic", @@ -2973,7 +3057,7 @@ test("event2message: rich reply with an image", async t => { }, event_id: "$QOxkw7u8vjTrrdKxEUO13JWSixV7UXAZU1freT1SkHc", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { getEvent(roomID, eventID) { called++ @@ -3378,7 +3462,7 @@ test("event2message: mentioning bridged rooms by alias works", async t => { unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { async getAlias(alias) { called++ @@ -3420,7 +3504,7 @@ test("event2message: mentioning bridged rooms by alias works (plaintext body)", unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { async getAlias(alias) { called++ @@ -3462,7 +3546,7 @@ test("event2message: mentioning bridged rooms by alias skips the link when alias unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { async getAlias(alias) { called++ @@ -3639,7 +3723,7 @@ test("event2message: mentioning unknown bridged events can approximate with time unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { async getEvent(roomID, eventID) { called++ @@ -3686,7 +3770,7 @@ test("event2message: mentioning events falls back to original link when server d unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { async getEvent(roomID, eventID) { called++ @@ -3732,7 +3816,7 @@ test("event2message: mentioning events falls back to original link when the chan unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { /* c8 ignore next 3 */ async getEvent() { @@ -3894,7 +3978,7 @@ test("event2message: caches the member if the member is not known", async t => { unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { getStateEvent: async (roomID, type, stateKey) => { called++ @@ -3944,7 +4028,7 @@ test("event2message: does not cache the member if the room is not known", async unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { getStateEvent: async (roomID, type, stateKey) => { called++ @@ -3992,7 +4076,7 @@ test("event2message: skips caching the member if the member does not exist, some unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { getStateEvent: async (roomID, type, stateKey) => { called++ @@ -4037,7 +4121,7 @@ test("event2message: overly long usernames are shifted into the message content" unsigned: { age: 405299 } - }, {}, { + }, {}, {}, { api: { getStateEvent: async (roomID, type, stateKey) => { called++ @@ -4610,7 +4694,7 @@ test("event2message: stickers fetch mimetype from server when mimetype not provi }, event_id: "$mL-eEVWCwOvFtoOiivDP7gepvf-fTYH6_ioK82bWDI0", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, { + }, {}, {}, { api: { async getMedia(mxc, options) { called++ @@ -4653,7 +4737,7 @@ test("event2message: stickers with unknown mimetype are not allowed", async t => }, event_id: "$mL-eEVWCwOvFtoOiivDP7gepvf-fTYH6_ioK82bWDI0", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, { + }, {}, {}, { api: { async getMedia(mxc, options) { called++ @@ -4817,7 +4901,7 @@ test("event2message: guessed @mentions in plaintext may join members to mention" room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" }, { id: "112760669178241024" - }, { + }, {}, { snow: { guild: { async searchGuildMembers(guildID, options) { @@ -4870,7 +4954,7 @@ test("event2message: guessed @mentions in formatted body may join members to men room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" }, { id: "112760669178241024" - }, { + }, {}, { snow: { guild: { async searchGuildMembers(guildID, options) { @@ -4914,7 +4998,7 @@ test("event2message: guessed @mentions feature will not activate on links or cod }, event_id: "$u5gSwSzv_ZQS3eM00mnTBCor8nx_A_AwuQz7e59PZk8", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, { + }, {}, {}, { snow: { guild: { /* c8 ignore next 4 */ @@ -4983,7 +5067,7 @@ test("event2message: @room converts to @everyone and is allowed when the room do }, room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", event_id: "$SiXetU9h9Dg-M9Frcw_C6ahnoXZ3QPZe3MVJR5tcB9A" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { async getStateEvent(roomID, type, key) { called++ @@ -5041,7 +5125,7 @@ test("event2message: @room converts to @everyone but is not allowed when the roo }, room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", event_id: "$SiXetU9h9Dg-M9Frcw_C6ahnoXZ3QPZe3MVJR5tcB9A" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { async getStateEvent(roomID, type, key) { t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") @@ -5098,7 +5182,7 @@ test("event2message: @room converts to @everyone and is allowed if the user has }, room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", event_id: "$SiXetU9h9Dg-M9Frcw_C6ahnoXZ3QPZe3MVJR5tcB9A" - }, data.guild.general, { + }, data.guild.general, data.channel.general, { api: { async getStateEvent(roomID, type, key) { t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") @@ -5222,7 +5306,7 @@ slow()("event2message: unknown emoji at the end is reuploaded as a sprite sheet" }, event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {mxcDownloader: mockGetAndConvertEmoji}) + }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) const testResult = { content: messages.messagesToSend[0].content, fileName: messages.messagesToSend[0].pendingFiles[0].name, @@ -5247,7 +5331,7 @@ slow()("event2message: known emoji from an unreachable server at the end is reup }, event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {mxcDownloader: mockGetAndConvertEmoji}) + }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) const testResult = { content: messages.messagesToSend[0].content, fileName: messages.messagesToSend[0].pendingFiles[0].name, @@ -5272,7 +5356,7 @@ slow()("event2message: known and unknown emojis in the end are reuploaded as a s }, event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {mxcDownloader: mockGetAndConvertEmoji}) + }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) const testResult = { content: messages.messagesToSend[0].content, fileName: messages.messagesToSend[0].pendingFiles[0].name, @@ -5297,7 +5381,7 @@ slow()("event2message: all unknown chess emojis are reuploaded as a sprite sheet }, event_id: "$Me6iE8C8CZyrDEOYYrXKSYRuuh_25Jj9kZaNrf7LKr4", room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {mxcDownloader: mockGetAndConvertEmoji}) + }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) const testResult = { content: messages.messagesToSend[0].content, fileName: messages.messagesToSend[0].pendingFiles[0].name, From 8e0fe29bec2d830036d7f98aab4c1347ef5f77e5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 8 Jan 2026 20:10:21 +1300 Subject: [PATCH 056/153] Web accessibility --- src/web/pug/guild_not_linked.pug | 2 +- src/web/pug/includes/template.pug | 6 +- src/web/routes/guild.test.js | 102 ++++++++++++++++++++++++++---- 3 files changed, 95 insertions(+), 15 deletions(-) diff --git a/src/web/pug/guild_not_linked.pug b/src/web/pug/guild_not_linked.pug index 59de2fb3..61c57e9b 100644 --- a/src/web/pug/guild_not_linked.pug +++ b/src/web/pug/guild_not_linked.pug @@ -4,7 +4,7 @@ mixin space(space) .s-user-card.flex__1 span.s-avatar.s-avatar__32.s-user-card--avatar if space.avatar - img.s-avatar--image(src=mUtils.getPublicUrlForMxc(space.avatar)) + img.s-avatar--image(src=mUtils.getPublicUrlForMxc(space.avatar) alt="") else .s-avatar--letter.bg-silver-400.bar-md(aria-hidden="true")= space.name[0] .s-user-card--info.ai-start diff --git a/src/web/pug/includes/template.pug b/src/web/pug/includes/template.pug index d9f1c309..93aaefc4 100644 --- a/src/web/pug/includes/template.pug +++ b/src/web/pug/includes/template.pug @@ -1,7 +1,7 @@ mixin guild(guild) span.s-avatar.s-avatar__32.s-user-card--avatar if guild.icon - img.s-avatar--image(src=`https://cdn.discordapp.com/icons/${guild.id}/${guild.icon}.png?size=32`) + img.s-avatar--image(src=`https://cdn.discordapp.com/icons/${guild.id}/${guild.icon}.png?size=32` alt="") else .s-avatar--letter.bg-silver-400.bar-md(aria-hidden="true")= guild.name[0] .s-user-card--info.ai-start @@ -82,10 +82,10 @@ html(lang="en") +define-themed-button("matrix", "black") body.themed.theme-system header.s-topbar - .s-topbar--skip-link(href="#content") Skip to main content + a.s-topbar--skip-link(href="#content") Skip to main content .s-topbar--container.wmx9 a.s-topbar--logo(href=rel("/")) - img.s-avatar.s-avatar__32(src=rel("/icon.png")) + img.s-avatar.s-avatar__32(src=rel("/icon.png") alt="") nav.s-topbar--navigation ul.s-topbar--content li.ps-relative.g8 diff --git a/src/web/routes/guild.test.js b/src/web/routes/guild.test.js index ea59173e..bb77c129 100644 --- a/src/web/routes/guild.test.js +++ b/src/web/routes/guild.test.js @@ -89,7 +89,7 @@ test("web guild: unbridged self-service guild shows available spaces", async t = }) t.has(html, `Data Horde`) t.has(html, `
  • here is the space topic
    • `) - t.has(html, ``) + t.has(html, ``) t.notMatch(html, /some room<\/strong>/) t.notMatch(html, /somebody else's space<\/strong>/) }) @@ -190,21 +190,66 @@ test("api invite: can invite with valid nonce", async t => { api: { async getStateEvent(roomID, type, key) { called++ - return {membership: "leave"} + if (type === "m.room.member" && key === "@cadence:cadence.moe") { + return {membership: "leave"} + } else if (type === "m.room.power_levels" && key === "") { + return {} + } else { + t.fail(`unexpected getStateEvent call. roomID: ${roomID}, type: ${type}, key: ${key}`) + } + }, + async getStateEventOuter(roomID, type, key) { + called++ + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + room_id: roomID, + content: { + room_version: "11" + } + } }, async inviteToRoom(roomID, mxidToInvite, mxid) { + called++ t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - called++ }, - async setUserPowerCascade(roomID, mxid, power) { - t.equal(power, 50) // moderator + async *generateFullHierarchy(spaceID) { called++ + yield { + room_id: "!hierarchy", + children_state: [], + guest_can_join: false, + num_joined_members: 2, + } + }, + async sendState(roomID, type, key, content) { + called++ + t.ok(["!hierarchy", "!jjmvBegULiLucuWEHU:cadence.moe"].includes(roomID), `expected room ID to be in hierarchy, but was ${roomID}`) + t.equal(type, "m.room.power_levels") + t.equal(key, "") + t.deepEqual(content, { + users: {"@cadence:cadence.moe": 50} + }) + return "$updated" } } }) ) t.notOk(error) - t.equal(called, 3) + /* + 1. get membership + 2. invite to room + set power: + 3. generate hierarchy + 4-5. calculate powers + 6. send state + 7-8. calculate powers + 9. send state + */ + t.equal(called, 9) // get membership + }) test("api invite: access denied when nonce has been used", async t => { @@ -235,21 +280,56 @@ test("api invite: can invite to a moderated guild", async t => { api: { async getStateEvent(roomID, type, key) { called++ - throw new MatrixServerError({errcode: "M_NOT_FOUND", error: "Event not found or something"}) + if (type === "m.room.member" && key === "@cadence:cadence.moe") { + return {membership: "leave"} + } else if (type === "m.room.power_levels" && key === "") { + return {} + } else { + t.fail(`unexpected getStateEvent call. roomID: ${roomID}, type: ${type}, key: ${key}`) + } + }, + async getStateEventOuter(roomID, type, key) { + called++ + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + room_id: roomID, + content: { + room_version: "11" + } + } }, async inviteToRoom(roomID, mxidToInvite, mxid) { + called++ t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe") - called++ }, - async setUserPowerCascade(roomID, mxid, power) { - t.equal(power, 100) // moderator + async *generateFullHierarchy(spaceID) { called++ + yield { + room_id: "!hierarchy", + children_state: [], + guest_can_join: false, + num_joined_members: 2, + } + }, + async sendState(roomID, type, key, content) { + called++ + t.ok(["!hierarchy", "!jjmvBegULiLucuWEHU:cadence.moe"].includes(roomID), `expected room ID to be in hierarchy, but was ${roomID}`) + t.equal(type, "m.room.power_levels") + t.equal(key, "") + t.deepEqual(content, { + users: {"@cadence:cadence.moe": 100} + }) + return "$updated" } } }) ) t.notOk(error) - t.equal(called, 3) + t.equal(called, 9) }) test("api invite: does not reinvite joined users", async t => { From 29d08df09422578b9aab7c8c153f2df72bfa17a0 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Jan 2026 01:48:59 +1300 Subject: [PATCH 057/153] Fix Discord replies to multipart Matrix messages The second part of the message is part=1 but the reply database lookup wanted part=0. To fix this, now it finds the first available part. --- src/d2m/converters/message-to-event.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 85ee969a..66717ad3 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -281,7 +281,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions // (Still need to do scenarios 1 and 2 part B, and scenario 3.) if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) { - const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id) + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? ORDER BY part ASC").get(message.message_reference.message_id) if (row) { repliedToEventRow = Object.assign(row, {channel_id: row.reference_channel_id}) } else if (message.referenced_message) { @@ -295,7 +295,7 @@ async function messageToEvent(message, guild, options = {}, di) { assert(message.embeds[0].description) const match = message.embeds[0].description.match(/\/channels\/[0-9]*\/[0-9]*\/([0-9]{2,})/) if (match) { - const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1]) + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? ORDER BY part ASC").get(match[1]) if (row) { /* we generate a partial referenced_message based on what PK provided. we don't need everything, since this will only be used for further message-to-event converting. From 8661aa7cfa4c3d006b517e364204723d0052209e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Jan 2026 01:50:16 +1300 Subject: [PATCH 058/153] Remove unused orm function --- src/db/orm.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/db/orm.js b/src/db/orm.js index d14e8ae2..4d9b6f1a 100644 --- a/src/db/orm.js +++ b/src/db/orm.js @@ -104,21 +104,6 @@ class From { return r } - /** - * @template {Col} Select - * @param {string} what - * @param {Select} col - */ - pluckAs(what, col) { - /** @type {Pluck} */ - // @ts-ignore - const r = this - r.cols = [`${what} AS ${col}`] - this.makeColsSafe = false - r.isPluck = true - return r - } - /** * @param {string} sql */ From 0d15865bcd0432b3573991f6f0ac9bf2a25c3d89 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Jan 2026 02:07:08 +1300 Subject: [PATCH 059/153] kstate fixes and code coverage --- src/db/migrations/0029-force-guild-ids.js | 2 + src/matrix/kstate.js | 17 +- src/matrix/kstate.test.js | 184 +++++++++++++++++++++- 3 files changed, 192 insertions(+), 11 deletions(-) diff --git a/src/db/migrations/0029-force-guild-ids.js b/src/db/migrations/0029-force-guild-ids.js index e5757838..354bc6bc 100644 --- a/src/db/migrations/0029-force-guild-ids.js +++ b/src/db/migrations/0029-force-guild-ids.js @@ -14,6 +14,8 @@ const {discord} = require("../../passthrough") const ones = "₀₁₂₃₄₅₆₇₈₉" const tens = "0123456789" +/* c8 ignore start */ + module.exports = async function(db) { /** @type {{name: string, channel_id: string, thread_parent: string | null}[]} */ const rows = db.prepare("SELECT name, channel_id, thread_parent FROM channel_room WHERE guild_id IS NULL").all() diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index 85a28380..37eed393 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -53,11 +53,11 @@ async function kstateToState(kstate) { kstateStripConditionals(kstate) await kstateUploadMxc(kstate) for (const [k, content] of Object.entries(kstate)) { - if (k === "m.room.create/") continue const slashIndex = k.indexOf("/") assert(slashIndex > 0) const type = k.slice(0, slashIndex) const state_key = k.slice(slashIndex + 1) + if (type === "m.room.create") continue events.push({type, state_key, content}) } return events @@ -94,15 +94,16 @@ function diffKState(actual, target) { if (key === "m.room.power_levels/") { // Special handling for power levels, we want to deep merge the actual and target into the final state. if (!(key in actual)) throw new Error(`want to apply a power levels diff, but original power level data is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) - const mixedTarget = mixin({}, actual[key], target[key]) + // if the diff includes users, it needs to be cleaned wrt room version 12 + const cleanedTarget = mixin({}, target[key]) + if (target[key].users && Object.keys(target[key].users).length > 0) { + assert("m.room.create/" in actual, `want to apply a power levels diff, but original m.room.create/ is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) + assert("m.room.create/outer" in actual, `want to apply a power levels diff, but original m.room.create/outer is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) + utils.removeCreatorsFromPowerLevels(actual["m.room.create/outer"], cleanedTarget) + } + const mixedTarget = mixin({}, actual[key], cleanedTarget) if (!isDeepStrictEqual(actual[key], mixedTarget)) { // they differ. use the newly prepared object as the diff. - // if the diff includes users, it needs to be cleaned wrt room version 12 - if (target[key].users && Object.keys(target[key].users).length > 0) { - if (!("m.room.create/" in actual)) throw new Error(`want to apply a power levels diff, but original m.room.create/ is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) - if (!("m.room.create/outer" in actual)) throw new Error(`want to apply a power levels diff, but original m.room.create/outer is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) - utils.removeCreatorsFromPowerLevels(actual["m.room.create/outer"], mixedTarget) - } diff[key] = mixedTarget } diff --git a/src/matrix/kstate.test.js b/src/matrix/kstate.test.js index ff65e9c6..b67a7252 100644 --- a/src/matrix/kstate.test.js +++ b/src/matrix/kstate.test.js @@ -1,5 +1,5 @@ const assert = require("assert") -const {kstateToState, stateToKState, diffKState, kstateStripConditionals, kstateUploadMxc} = require("./kstate") +const {kstateToState, stateToKState, diffKState, kstateStripConditionals, kstateUploadMxc, kstateToCreationContent} = require("./kstate") const {test} = require("supertape") test("kstate strip: strips false conditions", t => { @@ -68,6 +68,8 @@ test("kstateUploadMxc and strip: work together", async t => { test("kstate2state: general", async t => { t.deepEqual(await kstateToState({ + "m.room.create/": {bogus: true}, + "m.room.create/outer": {bogus: true}, "m.room.name/": {name: "test name"}, "m.room.member/@cadence:cadence.moe": {membership: "join"}, "uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"} @@ -98,6 +100,14 @@ test("kstate2state: general", async t => { test("state2kstate: general", t => { t.deepEqual(stateToKState([ + { + type: "m.room.create", + state_key: "", + sender: "@example:matrix.org", + content: { + room_version: "12" + } + }, { type: "m.room.name", state_key: "", @@ -122,7 +132,9 @@ test("state2kstate: general", t => { ]), { "m.room.name/": {name: "test name"}, "m.room.member/@cadence:cadence.moe": {membership: "join"}, - "uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"} + "uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"}, + "m.room.create/": {room_version: "12"}, + "m.room.create/outer": {type: "m.room.create", state_key: "", sender: "@example:matrix.org", content: {room_version: "12"}} }) }) @@ -157,6 +169,17 @@ test("diffKState: detects new properties", t => { test("diffKState: power levels are mixed together", t => { const original = { + "m.room.create/outer": { + type: "m.room.create", + state_key: "", + sender: "@example:matrix.org", + content: { + room_version: "11" + } + }, + "m.room.create/": { + room_version: "11" + }, "m.room.power_levels/": { "ban": 50, "events": { @@ -181,6 +204,9 @@ test("diffKState: power levels are mixed together", t => { "m.room.power_levels/": { "events": { "m.room.avatar": 0 + }, + users: { + "@example:matrix.org": 100 } } }) @@ -201,7 +227,8 @@ test("diffKState: power levels are mixed together", t => { "redact": 50, "state_default": 50, "users": { - "@example:localhost": 100 + "@example:localhost": 100, + "@example:matrix.org": 100 }, "users_default": 0 } @@ -270,3 +297,154 @@ test("diffKState: topic changes if the topic key has changed", t => { } }) }) + +test("diffKState: room v12 creators cannot be introduced into power levels", t => { + const original = { + "m.room.create/outer": { + type: "m.room.create", + state_key: "", + sender: "@example1:matrix.org", + content: { + additional_creators: ["@example2:matrix.org"], + room_version: "12" + } + }, + "m.room.create/": { + room_version: "12" + }, + "m.room.power_levels/": { + "ban": 50, + "events": { + "m.room.name": 100, + "m.room.power_levels": 100 + }, + "events_default": 0, + "invite": 50, + "kick": 50, + "notifications": { + "room": 20 + }, + "redact": 50, + "state_default": 50, + "users": { + "@example:localhost": 100 + }, + "users_default": 0 + } + } + const result = diffKState(original, { + "m.room.create/": { + bogus: true + }, + "m.room.power_levels/": { + events: { + "m.room.avatar": 0 + }, + users: { + "@example1:matrix.org": 100, + "@example2:matrix.org": 100, + "@example3:matrix.org": 100 + } + } + }) + t.deepEqual(result, { + "m.room.power_levels/": { + "ban": 50, + "events": { + "m.room.name": 100, + "m.room.power_levels": 100, + "m.room.avatar": 0 + }, + "events_default": 0, + "invite": 50, + "kick": 50, + "notifications": { + "room": 20 + }, + "redact": 50, + "state_default": 50, + "users": { + "@example:localhost": 100, + "@example3:matrix.org": 100 + }, + "users_default": 0 + } + }) + t.notDeepEqual(original, result) +}) + +test("diffKState: room v12 creators cannot be introduced into power levels - no diff if no changes", t => { + const original = { + "m.room.create/outer": { + type: "m.room.create", + state_key: "", + sender: "@example1:matrix.org", + content: { + additional_creators: ["@example2:matrix.org"], + room_version: "12" + } + }, + "m.room.create/": { + additional_creators: ["@example2:matrix.org"], + room_version: "12" + }, + "m.room.power_levels/": { + "ban": 50, + "events": { + "m.room.name": 100, + "m.room.power_levels": 100 + }, + "events_default": 0, + "invite": 50, + "kick": 50, + "notifications": { + "room": 20 + }, + "redact": 50, + "state_default": 50, + "users": { + "@example:localhost": 100 + }, + "users_default": 0 + } + } + const result = diffKState(original, { + "m.room.power_levels/": { + users: { + "@example1:matrix.org": 100, + "@example2:matrix.org": 100 + } + } + }) + t.deepEqual(result, {}) + t.notDeepEqual(original, result) +}) + +test("kstateToCreationContent: works", t => { + const original = { + "m.room.create/outer": { + type: "m.room.create", + state_key: "", + sender: "@example1:matrix.org", + content: { + additional_creators: ["@example2:matrix.org"], + room_version: "12", + type: "m.space" + } + }, + "m.room.create/": { + additional_creators: ["@example2:matrix.org"], + room_version: "12", + type: "m.space" + } + } + t.deepEqual(kstateToCreationContent(original), { + additional_creators: ["@example2:matrix.org"], + room_version: "12", + type: "m.space" + }) +}) + +test("kstateToCreationContent: works if empty", t => { + t.deepEqual(kstateToCreationContent({}), {}) +}) From 045fdfdf27280c83076b841ec097d505049b5a96 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Jan 2026 03:49:32 +1300 Subject: [PATCH 060/153] General code coverage --- src/d2m/actions/create-room.js | 3 +- src/m2d/converters/emoji.test.js | 12 ++ src/matrix/mreq.js | 30 ++-- src/matrix/utils.js | 5 +- src/matrix/utils.test.js | 218 +++++++++++++++++++++++- src/web/routes/download-discord.js | 4 +- src/web/routes/download-discord.test.js | 65 +++++-- src/web/routes/guild.test.js | 7 + src/web/routes/info.js | 3 +- src/web/routes/link.test.js | 10 +- src/web/routes/password.test.js | 16 ++ test/test.js | 1 + 12 files changed, 331 insertions(+), 43 deletions(-) create mode 100644 src/web/routes/password.test.js diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 5c2b76c1..11a03e5a 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -127,8 +127,9 @@ async function channelToKState(channel, guild, di) { const everyoneCanMentionEveryone = dUtils.hasAllPermissions(everyonePermissions, ["MentionEveryone"]) const spacePowerDetails = await mUtils.getEffectivePower(guildSpaceID, [], di.api) + spacePowerDetails.powerLevels.users ??= {} const spaceCreatorsAndFounders = spacePowerDetails.allCreators - .concat(Object.entries(spacePowerDetails.powerLevels.users ?? {}).filter(([, power]) => power >= spacePowerDetails.tombstone).map(([mxid]) => mxid)) + .concat(Object.entries(spacePowerDetails.powerLevels.users).filter(([, power]) => power >= spacePowerDetails.tombstone).map(([mxid]) => mxid)) const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all() const globalAdminPower = globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {}) diff --git a/src/m2d/converters/emoji.test.js b/src/m2d/converters/emoji.test.js index ad9846ba..fafb1631 100644 --- a/src/m2d/converters/emoji.test.js +++ b/src/m2d/converters/emoji.test.js @@ -50,3 +50,15 @@ test("emoji: spy needs u+fe0f in the middle", async t => { test("emoji: couple needs u+fe0f in the middle", async t => { t.equal(await encodeEmoji("👩‍❤‍👩", null), "%F0%9F%91%A9%E2%80%8D%E2%9D%A4%EF%B8%8F%E2%80%8D%F0%9F%91%A9") }) + +test("emoji: exact known emojis are returned", async t => { + t.equal(await encodeEmoji("mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC", "hippo"), "hippo%3A230201364309868544") +}) + +test("emoji: inexact emojis are guessed by name", async t => { + t.equal(await encodeEmoji("mxc://example.invalid/a", "hippo"), "hippo%3A230201364309868544") +}) + +test("emoji: unknown custom emoji returns null", async t => { + t.equal(await encodeEmoji("mxc://example.invalid/a", "silly"), null) +}) diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index 888aa54d..9085add8 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -18,21 +18,6 @@ class MatrixServerError extends Error { } } -/** - * @param {Response} res - * @param {object} opts - */ -async function makeMatrixServerError(res, opts = {}) { - delete opts.headers?.["Authorization"] - if (res.headers.get("content-type") === "application/json") { - return new MatrixServerError(await res.json(), opts) - } else if (res.headers.get("content-type")?.startsWith("text/")) { - return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, opts) - } else { - return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, opts) - } -} - /** * @param {undefined | string | object | streamWeb.ReadableStream | stream.Readable} body * @returns {Promise} @@ -52,6 +37,21 @@ async function _convertBody(body) { /* c8 ignore start */ +/** + * @param {Response} res + * @param {object} opts + */ +async function makeMatrixServerError(res, opts = {}) { + delete opts.headers?.["Authorization"] + if (res.headers.get("content-type") === "application/json") { + return new MatrixServerError(await res.json(), opts) + } else if (res.headers.get("content-type")?.startsWith("text/")) { + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, opts) + } else { + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, opts) + } +} + /** * @param {string} method * @param {string} url diff --git a/src/matrix/utils.js b/src/matrix/utils.js index 3860f6e3..f299d95f 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -138,14 +138,17 @@ async function getViaServers(roomID, api) { candidates.push(reg.ooye.server_name) // Candidate 1: Highest joined non-sim non-bot power level user in the room // https://github.com/matrix-org/matrix-react-sdk/blob/552c65db98b59406fb49562e537a2721c8505517/src/utils/permalinks/Permalinks.ts#L172 + /* c8 ignore next */ const call = "getEffectivePower" in api ? api.getEffectivePower(roomID, [bot], api) : getEffectivePower(roomID, [bot], api) const {allCreators, powerLevels} = await call - const sorted = allCreators.concat(Object.entries(powerLevels.users ?? {}).sort((a, b) => b[1] - a[1]).map(([mxid]) => mxid)) // Highest... + powerLevels.users ??= {} + const sorted = allCreators.concat(Object.entries(powerLevels.users).sort((a, b) => b[1] - a[1]).map(([mxid]) => mxid)) // Highest... for (const mxid of sorted) { if (!(mxid in joined)) continue // joined... if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot... const match = mxid.match(/:(.*)/) assert(match) + /* c8 ignore next - should be already covered by the userRegex test, but let's be explicit */ if (candidates.includes(match[1])) continue // from a different server candidates.push(match[1]) break diff --git a/src/matrix/utils.test.js b/src/matrix/utils.test.js index 0ecd41ed..842c5130 100644 --- a/src/matrix/utils.test.js +++ b/src/matrix/utils.test.js @@ -1,7 +1,8 @@ // @ts-check +const {select} = require("../passthrough") const {test} = require("supertape") -const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers, roomHasAtLeastVersion} = require("./utils") +const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers, roomHasAtLeastVersion, removeCreatorsFromPowerLevels, setUserPower} = require("./utils") const util = require("util") /** @param {string[]} mxids */ @@ -201,4 +202,219 @@ test("getViaServers: only considers power levels of currently joined members", a t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"]) }) +test("roomHasAtLeastVersion: v9 < v11", t => { + t.equal(roomHasAtLeastVersion("9", 11), false) +}) + +test("roomHasAtLeastVersion: v12 >= v11", t => { + t.equal(roomHasAtLeastVersion("12", 11), true) +}) + +test("roomHasAtLeastVersion: v12 >= v12", t => { + t.equal(roomHasAtLeastVersion("12", 12), true) +}) + +test("roomHasAtLeastVersion: custom versions never match", t => { + t.equal(roomHasAtLeastVersion("moe.cadence.silly", 11), false) +}) + +test("removeCreatorsFromPowerLevels: removes the creator from a v12 room", t => { + t.deepEqual(removeCreatorsFromPowerLevels({ + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + room_id: "!example", + event_id: "$create", + origin_server_ts: 0, + content: { + room_version: "12" + } + }, { + users: { + "@_ooye_bot:cadence.moe": 100 + } + }), { + users: { + } + }) +}) + +test("removeCreatorsFromPowerLevels: removes all creators from a v12 room", t => { + t.deepEqual(removeCreatorsFromPowerLevels({ + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + room_id: "!example", + event_id: "$create", + origin_server_ts: 0, + content: { + additional_creators: ["@cadence:cadence.moe"], + room_version: "12" + } + }, { + users: { + "@_ooye_bot:cadence.moe": 100, + "@cadence:cadence.moe": 100 + } + }), { + users: { + } + }) +}) + +test("removeCreatorsFromPowerLevels: doesn't touch a v11 room", t => { + t.deepEqual(removeCreatorsFromPowerLevels({ + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + room_id: "!example", + event_id: "$create", + origin_server_ts: 0, + content: { + additional_creators: ["@cadence:cadence.moe"], + room_version: "11" + } + }, { + users: { + "@_ooye_bot:cadence.moe": 100, + "@cadence:cadence.moe": 100 + } + }), { + users: { + "@_ooye_bot:cadence.moe": 100, + "@cadence:cadence.moe": 100 + } + }) +}) + +test("set user power: no-op", async t => { + let called = 0 + await setUserPower("!room", "@cadence:cadence.moe", 0, { + async getStateEvent(roomID, type, key) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return {} + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + room_id: "!room", + origin_server_ts: 0, + event_id: "$create", + content: { + room_version: "11" + } + } + }, + /* c8 ignore next 4 */ + async sendState() { + called++ + throw new Error("should not try to send state") + } + }) + t.equal(called, 2) +}) + +test("set user power: bridge bot must promote unprivileged users", async t => { + let called = 0 + await setUserPower("!room", "@cadence:cadence.moe", 100, { + async getStateEvent(roomID, type, key) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return { + users: {"@_ooye_bot:cadence.moe": 100} + } + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + room_id: "!room", + origin_server_ts: 0, + event_id: "$create", + content: { + room_version: "11" + } + } + }, + async sendState(roomID, type, key, content, mxid) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + t.deepEqual(content, { + users: { + "@_ooye_bot:cadence.moe": 100, + "@cadence:cadence.moe": 100 + } + }) + t.equal(mxid, undefined) + return "$sent" + } + }) + t.equal(called, 3) +}) + +test("set user power: privileged users must demote themselves", async t => { + let called = 0 + await setUserPower("!room", "@cadence:cadence.moe", 0, { + async getStateEvent(roomID, type, key) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return { + users: { + "@cadence:cadence.moe": 100, + "@_ooye_bot:cadence.moe": 100 + } + } + }, + async getStateEventOuter(roomID, type, key) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + room_id: "!room", + origin_server_ts: 0, + event_id: "$create", + content: { + room_version: "11" + } + } + }, + async sendState(roomID, type, key, content, mxid) { + called++ + t.equal(roomID, "!room") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + t.deepEqual(content, { + users: {"@_ooye_bot:cadence.moe": 100} + }) + t.equal(mxid, "@cadence:cadence.moe") + return "$sent" + } + }) + t.equal(called, 3) +}) + module.exports.mockGetEffectivePower = mockGetEffectivePower diff --git a/src/web/routes/download-discord.js b/src/web/routes/download-discord.js index bbf33b08..7c8c8e71 100644 --- a/src/web/routes/download-discord.js +++ b/src/web/routes/download-discord.js @@ -31,14 +31,13 @@ function getSnow(event) { /** @type {Map>} */ const cache = new Map() -/** @param {string} url */ +/** @param {string | undefined} url */ function timeUntilExpiry(url) { const params = new URL(url).searchParams const ex = params.get("ex") assert(ex) // refreshed urls from the discord api always include this parameter const time = parseInt(ex, 16)*1000 - Date.now() if (time > 0) return time - return false } function defineMediaProxyHandler(domain) { @@ -71,6 +70,7 @@ function defineMediaProxyHandler(domain) { refreshed = await promise const time = timeUntilExpiry(refreshed) assert(time) // the just-refreshed URL will always be in the future + /* c8 ignore next 3 */ setTimeout(() => { cache.delete(url) }, time).unref() diff --git a/src/web/routes/download-discord.test.js b/src/web/routes/download-discord.test.js index b0b0077e..a801c2c7 100644 --- a/src/web/routes/download-discord.test.js +++ b/src/web/routes/download-discord.test.js @@ -5,20 +5,6 @@ const {test} = require("supertape") const {router} = require("../../../test/web") const {MatrixServerError} = require("../../matrix/mreq") -const snow = { - channel: { - async refreshAttachmentURLs(attachments) { - if (typeof attachments === "string") attachments = [attachments] - return { - refreshed_urls: attachments.map(a => ({ - original: a, - refreshed: a + `?ex=${Math.floor(Date.now() / 1000 + 3600).toString(16)}` - })) - } - } - } -} - test("web download discord: access denied if not a known attachment", async t => { const [error] = await tryToCatch(() => router.test("get", "/download/discordcdn/:channel_id/:attachment_id/:file_name", { @@ -27,7 +13,19 @@ test("web download discord: access denied if not a known attachment", async t => attachment_id: "2", file_name: "image.png" }, - snow + snow: { + channel: { + async refreshAttachmentURLs(attachments) { + if (typeof attachments === "string") attachments = [attachments] + return { + refreshed_urls: attachments.map(a => ({ + original: a, + refreshed: a + `?ex=${Math.floor(Date.now() / 1000 + 3600).toString(16)}` + })) + } + } + } + } }) ) t.ok(error) @@ -42,8 +40,43 @@ test("web download discord: works if a known attachment", async t => { file_name: "image.png" }, event, - snow + snow: { + channel: { + async refreshAttachmentURLs(attachments) { + if (typeof attachments === "string") attachments = [attachments] + return { + refreshed_urls: attachments.map(a => ({ + original: a, + refreshed: a + `?ex=${Math.floor(Date.now() / 1000 + 3600).toString(16)}` + })) + } + } + } + } }) t.equal(event.node.res.statusCode, 302) t.match(event.node.res.getHeader("location"), /https:\/\/cdn.discordapp.com\/attachments\/655216173696286746\/1314358913482621010\/image\.png\?ex=/) }) + +test("web download discord: uses cache", async t => { + let notCalled = true + const event = {} + await router.test("get", "/download/discordcdn/:channel_id/:attachment_id/:file_name", { + params: { + channel_id: "655216173696286746", + attachment_id: "1314358913482621010", + file_name: "image.png" + }, + event, + snow: { + channel: { + // @ts-ignore + async refreshAttachmentURLs(attachments) { + notCalled = false + throw new Error("tried to refresh when it should be in cache") + } + } + } + }) + t.ok(notCalled) +}) diff --git a/src/web/routes/guild.test.js b/src/web/routes/guild.test.js index bb77c129..45869b3e 100644 --- a/src/web/routes/guild.test.js +++ b/src/web/routes/guild.test.js @@ -314,6 +314,13 @@ test("api invite: can invite to a moderated guild", async t => { guest_can_join: false, num_joined_members: 2, } + yield { + room_id: spaceID, + children_state: [], + guest_can_join: false, + num_joined_members: 2, + room_type: "m.space" + } }, async sendState(roomID, type, key, content) { called++ diff --git a/src/web/routes/info.js b/src/web/routes/info.js index 0c3e3b11..e83bf89f 100644 --- a/src/web/routes/info.js +++ b/src/web/routes/info.js @@ -68,8 +68,7 @@ as.router.get("/api/message", defineEventHandler(async event => { } } if (!matrix_author.displayname) matrix_author.displayname = mxid - if (matrix_author.avatar_url) matrix_author.avatar_url = mUtils.getPublicUrlForMxc(matrix_author.avatar_url) - else matrix_author.avatar_url = null + matrix_author.avatar_url = mUtils.getPublicUrlForMxc(matrix_author.avatar_url) || null matrix_author["mxid"] = mxid } diff --git a/src/web/routes/link.test.js b/src/web/routes/link.test.js index 291ea8e6..440bdfc5 100644 --- a/src/web/routes/link.test.js +++ b/src/web/routes/link.test.js @@ -148,7 +148,7 @@ test("web link space: check that inviting user has PL 50", async t => { t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") t.equal(type, "m.room.power_levels") t.equal(key, "") - return {users: {"@_ooye_bot:cadence.moe": 100}} + return {users: {"@_ooye_bot:cadence.moe": 100}, events: {"m.room.tombstone": 150}} }, async getStateEventOuter(roomID, type, key) { called++ @@ -163,7 +163,7 @@ test("web link space: check that inviting user has PL 50", async t => { event_id: "$create", origin_server_ts: 0, content: { - room_version: "11" + room_version: "12" } } } @@ -194,7 +194,7 @@ test("web link space: successfully adds entry to database and loads page", async t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") t.equal(type, "m.room.power_levels") t.equal(key, "") - return {users: {"@_ooye_bot:cadence.moe": 100, "@cadence:cadence.moe": 50}} + return {users: {"@cadence:cadence.moe": 50}} }, async getStateEventOuter(roomID, type, key) { called++ @@ -204,12 +204,12 @@ test("web link space: successfully adds entry to database and loads page", async return { type: "m.room.create", state_key: "", - sender: "@creator:cadence.moe", + sender: "@_ooye_bot:cadence.moe", room_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", event_id: "$create", origin_server_ts: 0, content: { - room_version: "11" + room_version: "12" } } } diff --git a/src/web/routes/password.test.js b/src/web/routes/password.test.js new file mode 100644 index 00000000..aa60bd3a --- /dev/null +++ b/src/web/routes/password.test.js @@ -0,0 +1,16 @@ +// @ts-check + +const tryToCatch = require("try-to-catch") +const {test} = require("supertape") +const {router} = require("../../../test/web") + +test("web password: stores password", async t => { + const event = {} + await router.test("post", "/api/password", { + body: { + password: "password123" + }, + event + }) + t.equal(event.node.res.statusCode, 302) +}) diff --git a/test/test.js b/test/test.js index 6e2c97b4..6470aae6 100644 --- a/test/test.js +++ b/test/test.js @@ -140,6 +140,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/web/routes/info.test") require("../src/web/routes/link.test") require("../src/web/routes/log-in-with-matrix.test") + require("../src/web/routes/password.test") require("../src/discord/utils.test") require("../src/matrix/kstate.test") require("../src/matrix/api.test") From 513e67189e82557c3a9fb84ec946e1fce4128f4a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Jan 2026 13:56:21 +1300 Subject: [PATCH 061/153] Formally remove discord_path from room avatars --- src/types.d.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/types.d.ts b/src/types.d.ts index 72ed83e3..25bed64a 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -276,7 +276,6 @@ export namespace Event { } export type M_Room_Avatar = { - discord_path?: string url?: string } From 505c41a35e260d6a7919a5a8d7e9277423f42c1b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 10 Jan 2026 02:28:18 +1300 Subject: [PATCH 062/153] More code coverage --- package.json | 2 +- src/d2m/converters/edit-to-changes.test.js | 55 +++++++ src/d2m/converters/message-to-event.js | 2 +- src/d2m/converters/message-to-event.test.js | 80 +++++++++ src/d2m/converters/user-to-mxid.js | 7 +- src/d2m/converters/user-to-mxid.test.js | 10 +- src/discord/utils.test.js | 12 ++ src/m2d/event-dispatcher.js | 6 +- src/matrix/matrix-command-handler.js | 2 +- src/matrix/room-upgrade.js | 30 ++-- src/matrix/room-upgrade.test.js | 169 ++++++++++++++++++++ src/web/routes/download-discord.js | 2 + src/web/routes/download-discord.test.js | 47 ++++-- src/web/routes/guild.js | 53 ++++-- src/web/routes/guild.test.js | 73 +++++++-- src/web/routes/link.js | 8 +- src/web/routes/log-in-with-matrix.test.js | 5 +- src/web/routes/oauth.js | 44 +++-- src/web/routes/oauth.test.js | 121 ++++++++++++++ test/data.js | 77 +++++++++ test/ooye-test-data.sql | 25 ++- test/test.js | 3 + test/web.js | 4 +- 23 files changed, 735 insertions(+), 102 deletions(-) create mode 100644 src/matrix/room-upgrade.test.js create mode 100644 src/web/routes/oauth.test.js diff --git a/package.json b/package.json index 4a80f227..05a4220e 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,6 @@ "addbot": "node addbot.js", "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js | tap-dot", "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot", - "cover": "c8 -o test/coverage --skip-full -x db/migrations -x src/matrix/file.js -x src/matrix/api.js -x src/d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow" + "cover": "c8 -o test/coverage --skip-full -x db/migrations -x src/m2d/event-dispatcher.js -x src/matrix/file.js -x src/matrix/api.js -x src/d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow" } } diff --git a/src/d2m/converters/edit-to-changes.test.js b/src/d2m/converters/edit-to-changes.test.js index dfb286c1..b252175e 100644 --- a/src/d2m/converters/edit-to-changes.test.js +++ b/src/d2m/converters/edit-to-changes.test.js @@ -271,6 +271,61 @@ test("edit2changes: promotes the text event when multiple rows have part = 1 (sh ]) }) +test("edit2changes: promotes newly sent event", async t => { + const {eventsToReplace, eventsToRedact, eventsToSend, promotions} = await editToChanges({ + channel_id: "1160894080998461480", + id: "1404133238414376971", + content: "hi", + attachments: [{ + id: "1157854643037163610", + filename: "Screenshot_20231001_034036.jpg", + size: 51981, + url: "https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg?ex=651a1faa&is=6518ce2a&hm=eb5ca80a3fa7add8765bf404aea2028a28a2341e4a62435986bcdcf058da82f3&", + proxy_url: "https://media.discordapp.net/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg?ex=651a1faa&is=6518ce2a&hm=eb5ca80a3fa7add8765bf404aea2028a28a2341e4a62435986bcdcf058da82f3&", + width: 1080, + height: 1170, + content_type: "image/jpeg" + }], + author: { + username: "cadence.worm", + global_name: "Cadence" + } + }, data.guild.general, { + async getEvent(roomID, eventID) { + t.equal(eventID, "$uUKLcTQvik5tgtTGDKuzn0Ci4zcCvSoUcYn2X7mXm9I") + return { + type: "m.room.message", + sender: "@_ooye_cadence.worm:cadence.moe", + content: { + msgtype: "m.text", + body: "hi" + } + } + } + }) + t.deepEqual(eventsToRedact, ["$LhmoWWvYyn5_AHkfb6FaXmLI6ZOC1kloql5P40YDmIk"]) + t.deepEqual(eventsToReplace, []) + t.deepEqual(eventsToSend, [{ + $type: "m.room.message", + body: "Screenshot_20231001_034036.jpg", + external_url: "https://bridge.example.org/download/discordcdn/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg", + filename: "Screenshot_20231001_034036.jpg", + info: { + mimetype: "image/jpeg", + size: 51981, + w: 1080, + h: 1170 + }, + url: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR", + "m.mentions": {}, + msgtype: "m.image" + }]) + t.deepEqual(promotions, [ + {column: "reaction_part", nextEvent: true} + ]) + // assert that the event parts will be consistent in database after this +}) + test("edit2changes: generated embed", async t => { let called = 0 const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.embed_generated_social_media_image, data.guild.general, { diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 66717ad3..4ff1f4d9 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -178,7 +178,7 @@ async function attachmentToEvent(mentions, attachment) { info: { mimetype: attachment.content_type, size: attachment.size, - duration: attachment.duration_secs ? Math.round(attachment.duration_secs * 1000) : undefined + duration: attachment.duration_secs && Math.round(attachment.duration_secs * 1000) } } } else { diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index 05ec5bef..3303b275 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -1356,3 +1356,83 @@ test("message2event: channel links are converted even inside lists (parser post- ]) t.equal(called, 1) }) + +test("message2event: emoji added special message", async t => { + const events = await messageToEvent(data.special_message.emoji_added) + t.deepEqual(events, [ + { + $type: "m.room.message", + msgtype: "m.emote", + body: "added a new emoji, :cx_marvelous: :cx_marvelous:", + format: "org.matrix.custom.html", + formatted_body: `added a new emoji, :cx_marvelous: :cx_marvelous:`, + "m.mentions": {} + } + ]) +}) + +test("message2event: cross-room reply", async t => { + let called = 0 + const events = await messageToEvent({ + type: 19, + message_reference: { + channel_id: "1161864271370666075", + guild_id: "1160893336324931584", + message_id: "1458091145136443547" + }, + referenced_message: { + channel_id: "1161864271370666075", + id: "1458091145136443547", + content: "", + attachments: [{ + filename: "image.png", + id: "1456813607693193478", + size: 104006, + content_type: "image/png", + url: "https://cdn.discordapp.com/attachments/1160893337029586956/1458790740338409605/image.png?ex=696194ff&is=6960437f&hm=923d0ef7d1b249470be49edbc37628cc4ff8a438f0ab12f54c045578135f7050" + }], + author: { + username: "Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆" + } + }, + content: "cross-room reply" + }, {}, {}, {api: { + async getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!mHmhQQPwXNananaOLD:cadence.moe") + t.equal(eventID, "$pgzCQjq_y5sy8RvWOUuoF3obNHjs8iNvt9c-odrOCPY") + return { + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + "body": "image.png", + "info": { + "h": 738, + "mimetype": "image/png", + "org.matrix.msc4230.is_animated": false, + "size": 111189, + "w": 772, + "xyz.amorgan.blurhash": "L255Oa~qRPD$-pxuoJoLIUM{xuxu" + }, + "m.mentions": {}, + "msgtype": "m.image", + "url": "mxc://matrix.org/QbSujQjRLekzPknKlPsXbGDS" + } + } + } + }}) + t.deepEqual(events, [ + { + $type: "m.room.message", + msgtype: "m.text", + body: "> Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆: [Media]\n\ncross-room reply", + format: "org.matrix.custom.html", + formatted_body: `
    In reply to Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆
    [Media]
    cross-room reply`, + "m.mentions": { + user_ids: [ + "@cadence:cadence.moe" + ] + } + } + ]) +}) diff --git a/src/d2m/converters/user-to-mxid.js b/src/d2m/converters/user-to-mxid.js index f7a49b10..12891c09 100644 --- a/src/d2m/converters/user-to-mxid.js +++ b/src/d2m/converters/user-to-mxid.js @@ -108,12 +108,7 @@ function isWebhookUserID(userID) { * @returns {string} */ function webhookAuthorToSimName(author) { - if (SPECIAL_USER_MAPPINGS.has(author.id)) { - const error = new Error("Special users should have followed the other code path.") - // @ts-ignore - error.author = author - throw error - } + assert(!SPECIAL_USER_MAPPINGS.has(author.id), "Special users should have followed the other code path.") // 1. Is sim user already registered? const fakeUserID = webhookAuthorToFakeUserID(author) diff --git a/src/d2m/converters/user-to-mxid.test.js b/src/d2m/converters/user-to-mxid.test.js index 86f151b6..2217a93e 100644 --- a/src/d2m/converters/user-to-mxid.test.js +++ b/src/d2m/converters/user-to-mxid.test.js @@ -2,7 +2,7 @@ const {test} = require("supertape") const tryToCatch = require("try-to-catch") const assert = require("assert") const data = require("../../../test/data") -const {userToSimName} = require("./user-to-mxid") +const {userToSimName, webhookAuthorToSimName} = require("./user-to-mxid") test("user2name: cannot create user for a webhook", async t => { const [error] = await tryToCatch(() => userToSimName({discriminator: "0000"})) @@ -52,3 +52,11 @@ test("user2name: includes ID if requested in config", t => { t.equal(userToSimName({username: "f***", discriminator: "0001", id: "123456"}), "123456_f") reg.ooye.include_user_id_in_mxid = false }) + +test("webhook author: can generate sim names", t => { + t.equal(webhookAuthorToSimName({ + username: "Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆", + avatar: null, + id: "123" + }), "webhook_cadence_maid_of_creation_eye_of_clarity_empress_of_hope") +}) diff --git a/src/discord/utils.test.js b/src/discord/utils.test.js index 79004409..516bd2f6 100644 --- a/src/discord/utils.test.js +++ b/src/discord/utils.test.js @@ -180,3 +180,15 @@ test("isEphemeralMessage: doesn't detect normal message", t => { test("getPublicUrlForCdn: no-op on non-discord URL", t => { t.equal(utils.getPublicUrlForCdn("https://cadence.moe"), "https://cadence.moe") }) + +test("how old: now", t => { + t.equal(utils.howOldUnbridgedMessage(new Date().toISOString(), new Date().toISOString()), "an unbridged message") +}) + +test("how old: hours", t => { + t.equal(utils.howOldUnbridgedMessage("2026-01-01T00:00:00", "2026-01-01T03:10:00"), "a 3-hour-old unbridged message") +}) + +test("how old: days", t => { + t.equal(utils.howOldUnbridgedMessage("2024-01-01", "2025-01-01"), "a 366-day-old unbridged message") +}) diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 13c0af1a..d9108528 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -332,7 +332,7 @@ async event => { const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` if (event.state_key === bot) { - const upgraded = await roomUpgrade.onBotMembership(event) + const upgraded = await roomUpgrade.onBotMembership(event, api, createRoom) if (upgraded) return } @@ -406,7 +406,9 @@ sync.addTemporaryListener(as, "type:m.room.tombstone", guard("m.room.tombstone", * @param {Ty.Event.StateOuter} event */ async event => { - await roomUpgrade.onTombstone(event) + if (event.state_key !== "") return + if (!event.content.replacement_room) return + await roomUpgrade.onTombstone(event, api) })) module.exports.stringifyErrorStack = stringifyErrorStack diff --git a/src/matrix/matrix-command-handler.js b/src/matrix/matrix-command-handler.js index e30ae6ff..721c3bdf 100644 --- a/src/matrix/matrix-command-handler.js +++ b/src/matrix/matrix-command-handler.js @@ -58,7 +58,7 @@ async function addButton(roomID, eventID, key, mxid) { setInterval(() => { const now = Date.now() buttons = buttons.filter(b => now - b.created < 2*60*60*1000) -}, 10*60*1000) +}, 10*60*1000).unref() /** @param {Ty.Event.Outer} event */ function onReactionAdd(event) { diff --git a/src/matrix/room-upgrade.js b/src/matrix/room-upgrade.js index 7f0d4e61..6c344cf1 100644 --- a/src/matrix/room-upgrade.js +++ b/src/matrix/room-upgrade.js @@ -4,12 +4,8 @@ const assert = require("assert/strict") const Ty = require("../types") const {Semaphore} = require("@chriscdn/promise-semaphore") const {tag} = require("@cloudrac3r/html-template-tag") -const {discord, db, sync, as, select, from} = require("../passthrough") +const {db, sync, select, from} = require("../passthrough") -/** @type {import("./api")}) */ -const api = sync.require("./api") -/** @type {import("../d2m/actions/create-room")}) */ -const createRoom = sync.require("../d2m/actions/create-room") /** @type {import("./utils")}) */ const utils = sync.require("./utils") @@ -17,11 +13,12 @@ const roomUpgradeSema = new Semaphore() /** * @param {Ty.Event.StateOuter} event + * @param {import("./api")} api */ -async function onTombstone(event) { - // Validate - if (event.state_key !== "") return - if (!event.content.replacement_room) return +async function onTombstone(event, api) { + // Preconditions (checked by event-dispatcher, enforced here) + assert.equal(event.state_key, "") + assert.ok(event.content.replacement_room) // Set up const oldRoomID = event.room_id @@ -48,13 +45,21 @@ async function onTombstone(event) { /** * @param {Ty.Event.StateOuter} event + * @param {import("./api")} api + * @param {import("../d2m/actions/create-room")} createRoom * @returns {Promise} whether to cancel other membership actions */ -async function onBotMembership(event) { +async function onBotMembership(event, api, createRoom) { + // Preconditions (checked by event-dispatcher, enforced here) + assert.equal(event.type, "m.room.member") + assert.equal(event.state_key, utils.bot) + // Check if an upgrade is pending for this room const newRoomID = event.room_id const oldRoomID = select("room_upgrade_pending", "old_room_id", {new_room_id: newRoomID}).pluck().get() if (!oldRoomID) return + const channelRow = from("channel_room").join("guild_space", "guild_id").where({room_id: oldRoomID}).select("space_id", "guild_id", "channel_id").get() + assert(channelRow) // this could only fail if the channel was unbridged or something between upgrade and joining // Check if is join/invite if (event.content.membership !== "invite" && event.content.membership !== "join") return @@ -65,9 +70,6 @@ async function onBotMembership(event) { await api.joinRoom(newRoomID) } - const channelRow = from("channel_room").join("guild_space", "guild_id").where({room_id: oldRoomID}).select("space_id", "guild_id", "channel_id").get() - assert(channelRow) - // Remove old room from space await api.sendState(channelRow.space_id, "m.space.child", oldRoomID, {}) // await api.sendState(oldRoomID, "m.space.parent", spaceID, {}) // keep this - the room isn't advertised but should still be grouped if opened @@ -75,7 +77,7 @@ async function onBotMembership(event) { // Remove declaration that old room is bridged (if able) try { await api.sendState(oldRoomID, "uk.half-shot.bridge", `moe.cadence.ooye://discord/${channelRow.guild_id}/${channelRow.channel_id}`, {}) - } catch (e) {} + } catch (e) { /* c8 ignore next */ } // Update database db.transaction(() => { diff --git a/src/matrix/room-upgrade.test.js b/src/matrix/room-upgrade.test.js new file mode 100644 index 00000000..3de1a8f3 --- /dev/null +++ b/src/matrix/room-upgrade.test.js @@ -0,0 +1,169 @@ +const {test} = require("supertape") +const {select} = require("../passthrough") +const {onTombstone, onBotMembership} = require("./room-upgrade") + +test("join upgraded room: only cares about upgrades in progress", async t => { + let called = 0 + await onBotMembership({ + type: "m.room.member", + state_key: "@_ooye_bot:cadence.moe", + room_id: "!JBxeGYnzQwLnaooOLD:cadence.moe", + content: { + membership: "invite" + } + }, { + /* c8 ignore next 4 */ + async joinRoom(roomID) { + called++ + throw new Error("should not join this room") + } + }) + t.equal(called, 0) +}) + +test("tombstone: only cares about bridged rooms", async t => { + let called = 0 + await onTombstone({ + event_id: "$tombstone", + type: "m.room.tombstone", + state_key: "", + sender: "@cadence:cadence.moe", + origin_server_ts: 0, + room_id: "!imaginary:cadence.moe", + content: { + body: "This room has been replaced", + replacement_room: "!JBxeGYnzQwLnaooNEW:cadence.moe" + } + }, { + /* c8 ignore next 4 */ + async joinRoom(roomID) { + called++ + throw new Error("should not join this room") + } + }) + t.equal(called, 0) +}) + +test("tombstone: joins new room and stores upgrade in database", async t => { + let called = 0 + await onTombstone({ + event_id: "$tombstone", + type: "m.room.tombstone", + state_key: "", + sender: "@cadence:cadence.moe", + origin_server_ts: 0, + room_id: "!JBxeGYnzQwLnaooOLD:cadence.moe", + content: { + body: "This room has been replaced", + replacement_room: "!JBxeGYnzQwLnaooNEW:cadence.moe" + } + }, { + async joinRoom(roomID) { + called++ + t.equal(roomID, "!JBxeGYnzQwLnaooNEW:cadence.moe") + return roomID + } + }) + t.equal(called, 1) + t.ok(select("room_upgrade_pending", ["old_room_id", "new_room_id"], {new_room_id: "!JBxeGYnzQwLnaooNEW:cadence.moe", old_room_id: "!JBxeGYnzQwLnaooOLD:cadence.moe"}).get()) +}) + +test("tombstone: requests invite from upgrader if can't join room", async t => { + let called = 0 + await onTombstone({ + event_id: "$tombstone", + type: "m.room.tombstone", + state_key: "", + sender: "@cadence:cadence.moe", + origin_server_ts: 0, + room_id: "!JBxeGYnzQwLnaooOLD:cadence.moe", + content: { + body: "This room has been replaced", + replacement_room: "!JBxeGYnzQwLnaooNEW:cadence.moe" + } + }, { + async joinRoom(roomID) { + called++ + t.equal(roomID, "!JBxeGYnzQwLnaooNEW:cadence.moe") + throw new Error("access denied or something") + }, + async usePrivateChat(sender) { + called++ + t.equal(sender, "@cadence:cadence.moe") + return "!private" + }, + async sendEvent(roomID, type, content) { + called++ + t.equal(roomID, "!private") + t.equal(type, "m.room.message") + t.deepEqual(content, { + msgtype: "m.text", + body: "You upgraded the bridged room winners. To keep bridging, I need you to invite me to the new room: https://matrix.to/#/!JBxeGYnzQwLnaooNEW:cadence.moe", + format: "org.matrix.custom.html", + formatted_body: `You upgraded the bridged room winners. To keep bridging, I need you to invite me to the new room: https://matrix.to/#/!JBxeGYnzQwLnaooNEW:cadence.moe` + }) + } + }) + t.equal(called, 3) +}) + +test("join upgraded room: only cares about invites/joins", async t => { + let called = 0 + await onBotMembership({ + type: "m.room.member", + state_key: "@_ooye_bot:cadence.moe", + room_id: "!JBxeGYnzQwLnaooNEW:cadence.moe", + content: { + membership: "leave" + } + }, { + /* c8 ignore next 4 */ + async joinRoom(roomID) { + called++ + throw new Error("should not join this room") + } + }) + t.equal(called, 0) +}) + +test("join upgraded room: joins invited room, updates database", async t => { + let called = 0 + await onBotMembership({ + type: "m.room.member", + state_key: "@_ooye_bot:cadence.moe", + room_id: "!JBxeGYnzQwLnaooNEW:cadence.moe", + content: { + membership: "invite" + } + }, { + async joinRoom(roomID) { + called++ + t.equal(roomID, "!JBxeGYnzQwLnaooNEW:cadence.moe") + return roomID + }, + async sendState(roomID, type, key, content) { + called++ + if (type === "m.space.child") { + t.equal(roomID, "!CvQMeeqXIkgedUpkzv:cadence.moe") // space + t.equal(key, "!JBxeGYnzQwLnaooOLD:cadence.moe") + t.deepEqual(content, {}) + return "$child" + } else if (type === "uk.half-shot.bridge") { + t.equal(roomID, "!JBxeGYnzQwLnaooOLD:cadence.moe") + t.equal(key, "moe.cadence.ooye://discord/1345641201902288987/598707048112193536") + t.deepEqual(content, {}) + return "$bridge" + } + /* c8 ignore next */ + throw new Error(`unexpected sendState: ${roomID} - ${type}/${key}`) + } + }, { + async syncRoom(channelID) { + called++ + t.equal(channelID, "598707048112193536") + } + }) + t.equal(called, 4) + t.equal(select("channel_room", "room_id", {channel_id: "598707048112193536"}).pluck().get(), "!JBxeGYnzQwLnaooNEW:cadence.moe") + t.equal(select("historical_channel_room", "historical_room_index", {reference_channel_id: "598707048112193536"}).pluck().all().length, 2) +}) diff --git a/src/web/routes/download-discord.js b/src/web/routes/download-discord.js index 7c8c8e71..3c58a75b 100644 --- a/src/web/routes/download-discord.js +++ b/src/web/routes/download-discord.js @@ -83,3 +83,5 @@ function defineMediaProxyHandler(domain) { as.router.get(`/download/discordcdn/:channel_id/:attachment_id/:file_name`, defineMediaProxyHandler("cdn.discordapp.com")) as.router.get(`/download/discordmedia/:channel_id/:attachment_id/:file_name`, defineMediaProxyHandler("media.discordapp.net")) + +module.exports._cache = cache diff --git a/src/web/routes/download-discord.test.js b/src/web/routes/download-discord.test.js index a801c2c7..e4f4ab4d 100644 --- a/src/web/routes/download-discord.test.js +++ b/src/web/routes/download-discord.test.js @@ -1,9 +1,10 @@ // @ts-check +const assert = require("assert").strict const tryToCatch = require("try-to-catch") const {test} = require("supertape") const {router} = require("../../../test/web") -const {MatrixServerError} = require("../../matrix/mreq") +const {_cache} = require("./download-discord") test("web download discord: access denied if not a known attachment", async t => { const [error] = await tryToCatch(() => @@ -12,19 +13,6 @@ test("web download discord: access denied if not a known attachment", async t => channel_id: "1", attachment_id: "2", file_name: "image.png" - }, - snow: { - channel: { - async refreshAttachmentURLs(attachments) { - if (typeof attachments === "string") attachments = [attachments] - return { - refreshed_urls: attachments.map(a => ({ - original: a, - refreshed: a + `?ex=${Math.floor(Date.now() / 1000 + 3600).toString(16)}` - })) - } - } - } } }) ) @@ -43,7 +31,7 @@ test("web download discord: works if a known attachment", async t => { snow: { channel: { async refreshAttachmentURLs(attachments) { - if (typeof attachments === "string") attachments = [attachments] + assert(Array.isArray(attachments)) return { refreshed_urls: attachments.map(a => ({ original: a, @@ -70,7 +58,7 @@ test("web download discord: uses cache", async t => { event, snow: { channel: { - // @ts-ignore + /* c8 ignore next 4 */ async refreshAttachmentURLs(attachments) { notCalled = false throw new Error("tried to refresh when it should be in cache") @@ -80,3 +68,30 @@ test("web download discord: uses cache", async t => { }) t.ok(notCalled) }) + +test("web download discord: refreshes when cache has expired", async t => { + _cache.set(`https://cdn.discordapp.com/attachments/655216173696286746/1314358913482621010/image.png`, Promise.resolve(`https://cdn.discordapp.com/blah?ex=${Math.floor(new Date("2026-01-01").getTime() / 1000 + 3600).toString(16)}`)) + let called = 0 + await router.test("get", "/download/discordcdn/:channel_id/:attachment_id/:file_name", { + params: { + channel_id: "655216173696286746", + attachment_id: "1314358913482621010", + file_name: "image.png" + }, + snow: { + channel: { + async refreshAttachmentURLs(attachments) { + called++ + assert(Array.isArray(attachments)) + return { + refreshed_urls: attachments.map(a => ({ + original: a, + refreshed: a + `?ex=${Math.floor(Date.now() / 1000 + 3600).toString(16)}` + })) + } + } + } + } + }) + t.equal(called, 1) +}) diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index 6b80e9df..9037d40e 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -73,23 +73,47 @@ function filterTo(xs, fn) { return filtered } +/** + * @param {{type: number, parent_id?: string, position?: number}} channel + * @param {Map} channels + */ +function getPosition(channel, channels) { + let position = 0 + + // Categories always appear below un-categorised channels. Their contents can be ordered. + // So categories, and things in them, will have their position multiplied by a big number. The category's big number. The regular position small number sorts within the category. + // Categories are size 2000. + let foundCategory = channel + while (foundCategory.parent_id) { + foundCategory = channels.get(foundCategory.parent_id) + } + if (foundCategory.type === DiscordTypes.ChannelType.GuildCategory) position = (foundCategory.position + 1) * 2000 + + // Categories always appear above what they contain. + if (channel.type === DiscordTypes.ChannelType.GuildCategory) position -= 0.5 + + // Within a category, voice channels are always sorted to the bottom. The text/voice split is size 1000 each. + if ([DiscordTypes.ChannelType.GuildVoice, DiscordTypes.ChannelType.GuildStageVoice].includes(channel.type)) position += 1000 + + // Channels are manually ordered within the text/voice split. + if (typeof channel.position === "number") position += channel.position + + // Threads appear below their channel. + if ([DiscordTypes.ChannelType.PublicThread, DiscordTypes.ChannelType.PrivateThread, DiscordTypes.ChannelType.AnnouncementThread].includes(channel.type)) { + position += 0.5 + let parent = channels.get(channel.parent_id) + if (parent && parent["position"]) position += parent["position"] + } + + return position +} + /** * @param {DiscordTypes.APIGuild} guild * @param {Ty.R.Hierarchy[]} rooms * @param {string[]} roles */ function getChannelRoomsLinks(guild, rooms, roles) { - function getPosition(channel) { - let position = 0 - let looking = channel - while (looking.parent_id) { - looking = discord.channels.get(looking.parent_id) - position = looking.position * 1000 - } - if (channel.position) position += channel.position - return position - } - let channelIDs = discord.guildChannelMap.get(guild.id) assert(channelIDs) @@ -97,7 +121,7 @@ function getChannelRoomsLinks(guild, rooms, roles) { let linkedChannelsWithDetails = linkedChannels.map(c => ({channel: discord.channels.get(c.channel_id), ...c})) let removedUncachedChannels = filterTo(linkedChannelsWithDetails, c => c.channel) let linkedChannelIDs = linkedChannelsWithDetails.map(c => c.channel_id) - linkedChannelsWithDetails.sort((a, b) => getPosition(a.channel) - getPosition(b.channel)) + linkedChannelsWithDetails.sort((a, b) => getPosition(a.channel, discord.channels) - getPosition(b.channel, discord.channels)) let unlinkedChannelIDs = channelIDs.filter(c => !linkedChannelIDs.includes(c)) /** @type {DiscordTypes.APIGuildChannel[]} */ // @ts-ignore @@ -107,7 +131,7 @@ function getChannelRoomsLinks(guild, rooms, roles) { const permissions = dUtils.getPermissions(roles, guild.roles, botID, c["permission_overwrites"]) return dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) }) - unlinkedChannels.sort((a, b) => getPosition(a) - getPosition(b)) + unlinkedChannels.sort((a, b) => getPosition(a, discord.channels) - getPosition(b, discord.channels)) let linkedRoomIDs = linkedChannels.map(c => c.room_id) let unlinkedRooms = [...rooms] @@ -239,3 +263,6 @@ as.router.post("/api/invite", defineEventHandler(async event => { return sendRedirect(event, "/ok?msg=User has been invited.", 302) } })) + +module.exports._getPosition = getPosition +module.exports._filterTo = filterTo diff --git a/src/web/routes/guild.test.js b/src/web/routes/guild.test.js index 45869b3e..e0b26dff 100644 --- a/src/web/routes/guild.test.js +++ b/src/web/routes/guild.test.js @@ -1,8 +1,10 @@ // @ts-check +const DiscordTypes = require("discord-api-types/v10") const tryToCatch = require("try-to-catch") const {router, test} = require("../../../test/web") const {MatrixServerError} = require("../../matrix/mreq") +const {_getPosition, _filterTo} = require("./guild") let nonce @@ -101,12 +103,6 @@ test("web guild: can view bridged guild when logged in with discord", async t => managedGuilds: ["112760669178241024"] }, api: { - async getStateEvent(roomID, type, key) { - return {} - }, - async getMembers(roomID, membership) { - return {chunk: []} - }, async getFullHierarchy(roomID) { return [] } @@ -121,12 +117,6 @@ test("web guild: can view bridged guild when logged in with matrix", async t => mxid: "@cadence:cadence.moe" }, api: { - async getStateEvent(roomID, type, key) { - return {} - }, - async getMembers(roomID, membership) { - return {chunk: []} - }, async getFullHierarchy(roomID) { return [] } @@ -191,12 +181,12 @@ test("api invite: can invite with valid nonce", async t => { async getStateEvent(roomID, type, key) { called++ if (type === "m.room.member" && key === "@cadence:cadence.moe") { - return {membership: "leave"} + throw new Error("event not found") } else if (type === "m.room.power_levels" && key === "") { return {} - } else { - t.fail(`unexpected getStateEvent call. roomID: ${roomID}, type: ${type}, key: ${key}`) } + /* c8 ignore next */ + t.fail(`unexpected getStateEvent call. roomID: ${roomID}, type: ${type}, key: ${key}`) }, async getStateEventOuter(roomID, type, key) { called++ @@ -284,9 +274,9 @@ test("api invite: can invite to a moderated guild", async t => { return {membership: "leave"} } else if (type === "m.room.power_levels" && key === "") { return {} - } else { - t.fail(`unexpected getStateEvent call. roomID: ${roomID}, type: ${type}, key: ${key}`) } + /* c8 ignore next */ + t.fail(`unexpected getStateEvent call. roomID: ${roomID}, type: ${type}, key: ${key}`) }, async getStateEventOuter(roomID, type, key) { called++ @@ -362,3 +352,52 @@ test("api invite: does not reinvite joined users", async t => { t.notOk(error) t.equal(called, 1) }) + + +test("position sorting: sorts like discord does", t => { + const channelsList = [{ + type: DiscordTypes.ChannelType.GuildText, + id: "first", + position: 0 + }, { + type: DiscordTypes.ChannelType.PublicThread, + id: "thread", + parent_id: "first", + }, { + type: DiscordTypes.ChannelType.GuildText, + id: "second", + position: 1 + }, { + type: DiscordTypes.ChannelType.GuildVoice, + id: "voice", + position: 0 + }, { + type: DiscordTypes.ChannelType.GuildCategory, + id: "category", + position: 0 + }, { + type: DiscordTypes.ChannelType.GuildText, + id: "category-first", + parent_id: "category", + position: 0 + }, { + type: DiscordTypes.ChannelType.GuildText, + id: "category-second", + parent_id: "category", + position: 1 + }, { + type: DiscordTypes.ChannelType.PublicThread, + id: "category-second-thread", + parent_id: "category-second", + }].reverse() + const channels = new Map(channelsList.map(c => [c.id, c])) + const sortedChannelIDs = [...channels.values()].sort((a, b) => _getPosition(a, channels) - _getPosition(b, channels)).map(c => c.id) + t.deepEqual(sortedChannelIDs, ["first", "thread", "second", "voice", "category", "category-first", "category-second", "category-second-thread"]) +}) + +test("filterTo: works", t => { + const fruit = ["apple", "banana", "apricot"] + const rest = _filterTo(fruit, f => f[0] === "b") + t.deepEqual(fruit, ["banana"]) + t.deepEqual(rest, ["apple", "apricot"]) +}) diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 5193dedb..ce80fd48 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -70,16 +70,14 @@ as.router.post("/api/link-space", defineEventHandler(async event => { // Check space ID if (!session.data.mxid) throw createError({status: 403, message: "Forbidden", data: "Can't link with your Matrix space if you aren't logged in to Matrix"}) const spaceID = parsedBody.space_id - const inviteType = select("invite", "type", {mxid: session.data.mxid, room_id: spaceID}).pluck().get() - if (inviteType !== "m.space") throw createError({status: 403, message: "Forbidden", data: "You personally must invite OOYE to that space on Matrix"}) + const inviteRow = select("invite", ["mxid", "type"], {mxid: session.data.mxid, room_id: spaceID}).get() + if (!inviteRow || inviteRow.type !== "m.space") throw createError({status: 403, message: "Forbidden", data: "You personally must invite OOYE to that space on Matrix"}) // Check they are not already bridged const existing = select("guild_space", "guild_id", {}, "WHERE guild_id = ? OR space_id = ?").get(guildID, spaceID) if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) - const inviteSender = select("invite", "mxid", {mxid: session.data.mxid, room_id: spaceID}).pluck().get() - const inviteSenderServer = inviteSender?.match(/:(.*)/)?.[1] - const via = [inviteSenderServer || ""] + const via = [inviteRow.mxid.match(/:(.*)/)[1]] // Check space exists and bridge is joined try { diff --git a/src/web/routes/log-in-with-matrix.test.js b/src/web/routes/log-in-with-matrix.test.js index a4030555..830556e3 100644 --- a/src/web/routes/log-in-with-matrix.test.js +++ b/src/web/routes/log-in-with-matrix.test.js @@ -39,7 +39,8 @@ test("log in with matrix: sends message to log in", async t => { let called = 0 await router.test("post", "/api/log-in-with-matrix", { body: { - mxid: "@cadence:cadence.moe" + mxid: "@cadence:cadence.moe", + next: "https://bridge.cadence.moe/guild?guild_id=123" }, api: { async usePrivateChat(mxid) { @@ -51,7 +52,7 @@ test("log in with matrix: sends message to log in", async t => { called++ t.equal(roomID, "!created:cadence.moe") t.equal(type, "m.room.message") - token = content.body.match(/log-in-with-matrix\?token=([a-f0-9-]+)/)[1] + token = content.body.match(/log-in-with-matrix\?token=([a-f0-9-]+)&next=/)[1] t.ok(token, "log in token not issued") return "" } diff --git a/src/web/routes/oauth.js b/src/web/routes/oauth.js index fe352306..f4bb61f5 100644 --- a/src/web/routes/oauth.js +++ b/src/web/routes/oauth.js @@ -2,12 +2,12 @@ const {z} = require("zod") const {randomUUID} = require("crypto") -const {defineEventHandler, getValidatedQuery, sendRedirect, createError} = require("h3") +const {defineEventHandler, getValidatedQuery, sendRedirect, createError, H3Event} = require("h3") const {SnowTransfer, tokenless} = require("snowtransfer") const DiscordTypes = require("discord-api-types/v10") const getRelativePath = require("get-relative-path") -const {discord, as, db, sync} = require("../../passthrough") +const {as, db, sync} = require("../../passthrough") const {id, permissions} = require("../../../addbot") /** @type {import("../auth")} */ const auth = sync.require("../auth") @@ -33,6 +33,24 @@ const schema = { }) } +/** + * @param {H3Event} event + * @returns {(string) => {user: {getGuilds: () => Promise}}} + */ +function getClient(event) { + /* c8 ignore next */ + return event.context.getClient || (accessToken => new SnowTransfer(`Bearer ${accessToken}`)) +} + +/** + * @param {H3Event} event + * @returns {typeof tokenless.getOauth2Token} + */ +function getOauth2Token(event) { + /* c8 ignore next */ + return event.context.getOauth2Token || tokenless.getOauth2Token +} + as.router.get("/oauth", defineEventHandler(async event => { const session = await auth.useSession(event) let scope = "guilds" @@ -61,21 +79,15 @@ as.router.get("/oauth", defineEventHandler(async event => { if (!savedState) throw createError({status: 400, message: "Missing state", data: "Missing saved state parameter. Please try again, and make sure you have cookies enabled."}) if (savedState != parsedQuery.data.state) return tryAgain() - const oauthResult = await tokenless.getOauth2Token(id, redirect_uri, reg.ooye.discord_client_secret, parsedQuery.data.code) - const parsedToken = schema.token.safeParse(oauthResult) - if (!parsedToken.success) { - throw createError({status: 502, message: "Invalid token response", data: `Discord completed OAuth, but returned this instead of an OAuth access token: ${JSON.stringify(oauthResult)}`}) - } + const oauthResult = await getOauth2Token(event)(id, redirect_uri, reg.ooye.discord_client_secret, parsedQuery.data.code) + const parsedToken = schema.token.parse(oauthResult) - const userID = Buffer.from(parsedToken.data.access_token.split(".")[0], "base64").toString() - const client = new SnowTransfer(`Bearer ${parsedToken.data.access_token}`) - try { - const guilds = await client.user.getGuilds() - var managedGuilds = guilds.filter(g => BigInt(g.permissions) & DiscordTypes.PermissionFlagsBits.ManageGuild).map(g => g.id) - await session.update({managedGuilds, userID, state: undefined}) - } catch (e) { - throw createError({status: 502, message: "API call failed", data: e.message}) - } + const userID = Buffer.from(parsedToken.access_token.split(".")[0], "base64").toString() + const client = getClient(event)(parsedToken.access_token) + + const guilds = await client.user.getGuilds() + var managedGuilds = guilds.filter(g => BigInt(g.permissions) & DiscordTypes.PermissionFlagsBits.ManageGuild).map(g => g.id) + await session.update({managedGuilds, userID, state: undefined}) // Set auto-create for the guild // @ts-ignore diff --git a/src/web/routes/oauth.test.js b/src/web/routes/oauth.test.js new file mode 100644 index 00000000..2f3a791e --- /dev/null +++ b/src/web/routes/oauth.test.js @@ -0,0 +1,121 @@ +// @ts-check + +const DiscordTypes = require("discord-api-types/v10") +const tryToCatch = require("try-to-catch") +const assert = require("assert/strict") +const {router, test} = require("../../../test/web") + +test("web oauth: redirects to Discord on first visit (add easy)", async t => { + let event = {} + await router.test("get", "/oauth?action=add", { + event, + sessionData: { + password: "password123" + } + }) + t.equal(event.node.res.statusCode, 302) + t.match(event.node.res.getHeader("location"), /^https:\/\/discord.com\/oauth2\/authorize\?client_id=684280192553844747&scope=bot\+guilds&permissions=2251801424568320&response_type=code&redirect_uri=https:\/\/bridge\.example\.org\/oauth&state=/) +}) + +test("web oauth: redirects to Discord on first visit (add self service)", async t => { + let event = {} + await router.test("get", "/oauth?action=add-self-service", { + event, + sessionData: { + password: "password123" + } + }) + t.equal(event.node.res.statusCode, 302) + t.match(event.node.res.getHeader("location"), /^https:\/\/discord.com\/oauth2\/authorize\?client_id=684280192553844747&scope=bot\+guilds&permissions=2251801424568320&response_type=code&redirect_uri=https:\/\/bridge\.example\.org\/oauth&state=/) +}) + +test("web oauth: advises user about cookies if state is missing", async t => { + let event = {} + const [e] = await tryToCatch(() => router.test("get", "/oauth?state=693551d5-47c5-49e2-a433-3600abe3c15c&code=DISCORD_CODE&guild_id=9", { + event + })) + t.equal(e.message, "Missing state") +}) + +test("web oauth: redirects to Discord again if state doesn't match", async t => { + let event = {} + await router.test("get", "/oauth?state=693551d5-47c5-49e2-a433-3600abe3c15c&code=DISCORD_CODE", { + event, + sessionData: { + state: "438aa253-1311-4483-9aa2-c251e29e72c9", + password: "password123" + } + }) + t.equal(event.node.res.statusCode, 302) + t.match(event.node.res.getHeader("location"), /^https:\/\/discord\.com\/oauth2\/authorize/) +}) + +test("web oauth: uses returned state, logs in", async t => { + let event = {} + await router.test("get", "/oauth?state=693551d5-47c5-49e2-a433-3600abe3c15c&code=DISCORD_CODE", { + event, + sessionData: { + state: "693551d5-47c5-49e2-a433-3600abe3c15c", + selfService: false, + password: "password123" + }, + getOauth2Token() { + return { + token_type: "Bearer", + access_token: "6qrZcUqja7812RVdnEKjpzOL4CvHBFG", + expires_in: 604800, + refresh_token: "D43f5y0ahjqew82jZ4NViEr2YafMKhue", + scope: "bot+guilds" + } + }, + getClient(accessToken) { + return { + user: { + async getGuilds() { + return [{ + id: "9", + permissions: DiscordTypes.PermissionFlagsBits.ManageGuild + }] + } + } + } + } + }) + t.equal(event.node.res.statusCode, 302) + t.equal(event.node.res.getHeader("location"), "./") +}) + +test("web oauth: uses returned state, adds managed guild", async t => { + let event = {} + await router.test("get", "/oauth?state=693551d5-47c5-49e2-a433-3600abe3c15c&code=DISCORD_CODE&guild_id=9", { + event, + sessionData: { + state: "693551d5-47c5-49e2-a433-3600abe3c15c", + selfService: false, + password: "password123" + }, + getOauth2Token() { + return { + token_type: "Bearer", + access_token: "6qrZcUqja7812RVdnEKjpzOL4CvHBFG", + expires_in: 604800, + refresh_token: "D43f5y0ahjqew82jZ4NViEr2YafMKhue", + scope: "bot+guilds" + } + }, + getClient(accessToken) { + return { + user: { + async getGuilds() { + return [{ + id: "9", + permissions: DiscordTypes.PermissionFlagsBits.ManageGuild + }] + } + } + } + } + }) + t.equal(event.node.res.statusCode, 302) + t.equal(event.node.res.getHeader("location"), "guild?guild_id=9") +}) diff --git a/test/data.js b/test/data.js index 387ad6a8..e80b4367 100644 --- a/test/data.js +++ b/test/data.js @@ -180,6 +180,39 @@ module.exports = { afk_timeout: 300, id: "112760669178241024", icon: "a_f83622e09ead74f0c5c527fe241f8f8c", + /** @type {DiscordTypes.APIGuildMember[]} */ // @ts-ignore + members: [{ + user: { + username: 'Matrix Bridge', + public_flags: 0, + primary_guild: null, + id: '684280192553844747', + global_name: null, + display_name_styles: null, + display_name: null, + discriminator: '5728', + collectibles: null, + bot: true, + avatar_decoration_data: null, + avatar: '48ae3c24f2a6ec5c60c41bdabd904018' + }, + roles: [ + '703457691342995528', + '289671295359254529', + '1040735082610167858', + '114526764860047367' + ], + premium_since: null, + pending: false, + nick: 'Mother', + mute: false, + joined_at: '2020-04-25T04:09:43.253000+00:00', + flags: 0, + deaf: false, + communication_disabled_until: null, + banner: null, + avatar: null + }], emojis: [ { roles: [], @@ -5433,6 +5466,50 @@ module.exports = { } }, special_message: { + emoji_added: { + type: 63, + content: '<:cx_marvelous:1437322787994992650>', + mentions: [], + mention_roles: [], + attachments: [], + embeds: [], + timestamp: '2025-11-10T06:07:36.930000+00:00', + edited_timestamp: null, + flags: 0, + components: [], + id: '1437322788439457794', + channel_id: '1100319550446252084', + author: { + id: '772659086046658620', + username: 'cadence.worm', + avatar: '466df0c98b1af1e1388f595b4c1ad1b9', + discriminator: '0', + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: 'cadence', + avatar_decoration_data: null, + collectibles: null, + display_name_styles: null, + banner_color: null, + clan: { + identity_guild_id: '532245108070809601', + identity_enabled: true, + tag: 'doll', + badge: 'dba08126b4e810a0e096cc7cd5bc37f0' + }, + primary_guild: { + identity_guild_id: '532245108070809601', + identity_enabled: true, + tag: 'doll', + badge: 'dba08126b4e810a0e096cc7cd5bc37f0' + } + }, + pinned: false, + mention_everyone: false, + tts: false + }, thread_name_change: { id: "1142391602799710298", type: 4, diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 76c822f7..36ec0b61 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -3,10 +3,12 @@ BEGIN TRANSACTION; INSERT INTO guild_active (guild_id, autocreate) VALUES ('112760669178241024', 1), ('66192955777486848', 1), -('665289423482519565', 0); +('665289423482519565', 0), +('1345641201902288987', 1); INSERT INTO guild_space (guild_id, space_id, privacy_level) VALUES -('112760669178241024', '!jjmvBegULiLucuWEHU:cadence.moe', 0); +('112760669178241024', '!jjmvBegULiLucuWEHU:cadence.moe', 0), +('1345641201902288987', '!CvQMeeqXIkgedUpkzv:cadence.moe', 0); INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom_avatar, guild_id) VALUES ('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL, NULL, '112760669178241024'), @@ -21,7 +23,8 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom ('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL, '66192955777486848'), ('1160894080998461480', '!TqlyQmifxGUggEmdBN:cadence.moe', 'ooyexperiment', NULL, NULL, NULL, '66192955777486848'), ('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL, '665289423482519565'), -('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL, '665289423482519565'); +('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL, '665289423482519565'), +('598707048112193536', '!JBxeGYnzQwLnaooOLD:cadence.moe', 'winners', NULL, NULL, NULL, '1345641201902288987'); INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) SELECT channel_id, room_id, 0 FROM channel_room; @@ -78,7 +81,8 @@ WITH a (message_id, channel_id) AS (VALUES ('1339000288144658482', '176333891320283136'), ('1381212840957972480', '112760669178241024'), ('1401760355339862066', '112760669178241024'), -('1439351590262800565', '1438284564815548418')) +('1439351590262800565', '1438284564815548418'), +('1404133238414376971', '112760669178241024')) SELECT message_id, max(historical_room_index) as historical_room_index FROM a INNER JOIN historical_channel_room ON historical_channel_room.reference_channel_id = a.channel_id GROUP BY message_id; INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES @@ -124,7 +128,9 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part ('$AfrB8hzXkDMvuoWjSZkDdFYomjInWH7jMBPkwQMN8AI', 'm.room.message', 'm.text', '1381212840957972480', 0, 1, 1), ('$43baKEhJfD-RlsFQi0LB16Zxd8yMqp0HSVL00TDQOqM', 'm.room.message', 'm.image', '1381212840957972480', 1, 0, 1), ('$7P2O_VTQNHvavX5zNJ35DV-dbJB1Ag80tGQP_JzGdhk', 'm.room.message', 'm.text', '1401760355339862066', 0, 0, 0), -('$ielAnR6geu0P1Tl5UXfrbxlIf-SV9jrNprxrGXP3v7M', 'm.room.message', 'm.image', '1439351590262800565', 0, 0, 0); +('$ielAnR6geu0P1Tl5UXfrbxlIf-SV9jrNprxrGXP3v7M', 'm.room.message', 'm.image', '1439351590262800565', 0, 0, 0), +('$uUKLcTQvik5tgtTGDKuzn0Ci4zcCvSoUcYn2X7mXm9I', 'm.room.message', 'm.text', '1404133238414376971', 0, 1, 1), +('$LhmoWWvYyn5_AHkfb6FaXmLI6ZOC1kloql5P40YDmIk', 'm.room.message', 'm.notice', '1404133238414376971', 1, 0, 1); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), @@ -155,7 +161,8 @@ INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES ('551636841284108289', 'ae_botrac4r', 0, 'mxc://cadence.moe/skqfuItqxNmBYekzmVKyoLzs'), ('975572106295259148', 'brillillillilliant_move', 0, 'mxc://cadence.moe/scfRIDOGKWFDEBjVXocWYQHik'), ('606664341298872324', 'online', 0, 'mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ'), -('288858540888686602', 'upstinky', 0, 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO'); +('288858540888686602', 'upstinky', 0, 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO'), +('1437322787994992650', 'cx_marvelous', 0, 'mxc://cadence.moe/TPZdosVUjTIopsLijkygIbti'); INSERT INTO member_cache (room_id, mxid, displayname, avatar_url, power_level) VALUES ('!jjmvBegULiLucuWEHU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', NULL, 50), @@ -200,4 +207,10 @@ INSERT INTO direct (mxid, room_id) VALUES ('@user1:example.org', '!existing:cadence.moe'), ('@user2:example.org', '!existing:cadence.moe'); +-- for cross-room reply test, in 'updates' room +UPDATE historical_channel_room SET room_id = '!mHmhQQPwXNananaOLD:cadence.moe' WHERE room_id = '!mHmhQQPwXNananMUqq:cadence.moe'; +INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) VALUES ('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 1767922455991); +INSERT INTO message_room (message_id, historical_room_index) SELECT '1458091145136443547', historical_room_index FROM historical_channel_room WHERE room_id = '!mHmhQQPwXNananaOLD:cadence.moe'; +INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES ('$pgzCQjq_y5sy8RvWOUuoF3obNHjs8iNvt9c-odrOCPY', 'm.room.message', 'm.image', '1458091145136443547', 0, 0, 0); + COMMIT; diff --git a/test/test.js b/test/test.js index 6470aae6..be7febf2 100644 --- a/test/test.js +++ b/test/test.js @@ -29,6 +29,7 @@ reg.namespaces = { reg.ooye.bridge_origin = "https://bridge.example.org" reg.ooye.time_zone = "Pacific/Auckland" reg.ooye.max_file_size = 5000000 +reg.ooye.web_password = "password123" const sync = new HeatSync({watchFS: false}) @@ -140,6 +141,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/web/routes/info.test") require("../src/web/routes/link.test") require("../src/web/routes/log-in-with-matrix.test") + require("../src/web/routes/oauth.test") require("../src/web/routes/password.test") require("../src/discord/utils.test") require("../src/matrix/kstate.test") @@ -147,6 +149,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/matrix/file.test") require("../src/matrix/mreq.test") require("../src/matrix/read-registration.test") + require("../src/matrix/room-upgrade.test") require("../src/matrix/txnid.test") require("../src/matrix/utils.test") require("../src/d2m/actions/create-room.test") diff --git a/test/web.js b/test/web.js index 09af95bb..463c6b1b 100644 --- a/test/web.js +++ b/test/web.js @@ -51,7 +51,7 @@ class Router { /** * @param {string} method * @param {string} inputUrl - * @param {{event?: any, params?: any, body?: any, sessionData?: any, api?: Partial, snow?: {[k in keyof SnowTransfer]?: Partial}, createRoom?: Partial, createSpace?: Partial, headers?: any}} [options] + * @param {{event?: any, params?: any, body?: any, sessionData?: any, getOauth2Token?: any, getClient?: (string) => {user: {getGuilds: () => Promise}}, api?: Partial, snow?: {[k in keyof SnowTransfer]?: Partial}, createRoom?: Partial, createSpace?: Partial, headers?: any}} [options] */ async test(method, inputUrl, options = {}) { const url = new URL(inputUrl, "http://a") @@ -87,6 +87,8 @@ class Router { snow: options.snow, createRoom: options.createRoom, createSpace: options.createSpace, + getOauth2Token: options.getOauth2Token, + getClient: options.getClient, sessions: { h3: { id: "h3", From 3d3671e05a13d4a3a21006f35afe0ca50db1fcca Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 10 Jan 2026 13:19:16 +1300 Subject: [PATCH 063/153] skip "cannot invite user that is joined" --- scripts/migrate-from-old-bridge.js | 10 +--------- src/d2m/actions/register-pk-user.js | 12 ++---------- src/d2m/actions/register-user.js | 12 ++---------- src/d2m/actions/register-webhook-user.js | 12 ++---------- src/matrix/api.js | 14 +++++++++++--- 5 files changed, 18 insertions(+), 42 deletions(-) diff --git a/scripts/migrate-from-old-bridge.js b/scripts/migrate-from-old-bridge.js index 40d6993e..1842c16b 100755 --- a/scripts/migrate-from-old-bridge.js +++ b/scripts/migrate-from-old-bridge.js @@ -82,15 +82,7 @@ async function migrateGuild(guild) { // Step 2: (Using old bridge access token) Join the new bridge to the old rooms and give it PL 100 console.log(`-- Joining channel ${channel.name}...`) await mreq.withAccessToken(oldAT, async () => { - try { - await api.inviteToRoom(roomID, newBridgeMxid) - } catch (e) { - if (e.message.includes("is already in the room")) { - // Great! - } else { - throw e - } - } + await api.inviteToRoom(roomID, newBridgeMxid) await utils.setUserPower(roomID, newBridgeMxid, 100, api) }) await api.joinRoom(roomID) diff --git a/src/d2m/actions/register-pk-user.js b/src/d2m/actions/register-pk-user.js index 3c914b6d..6ecd0775 100644 --- a/src/d2m/actions/register-pk-user.js +++ b/src/d2m/actions/register-pk-user.js @@ -87,16 +87,8 @@ async function ensureSimJoined(pkMessage, roomID) { // Ensure joined const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() if (!existing) { - try { - await api.inviteToRoom(roomID, mxid) - await api.joinRoom(roomID, mxid) - } catch (e) { - if (e.message.includes("is already in the room.")) { - // Sweet! - } else { - throw e - } - } + await api.inviteToRoom(roomID, mxid) + await api.joinRoom(roomID, mxid) db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid) } return mxid diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index 966263b5..35da4f5c 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -86,16 +86,8 @@ async function ensureSimJoined(user, roomID) { // Ensure joined const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() if (!existing) { - try { - await api.inviteToRoom(roomID, mxid) - await api.joinRoom(roomID, mxid) - } catch (e) { - if (e.message.includes("is already in the room.")) { - // Sweet! - } else { - throw e - } - } + await api.inviteToRoom(roomID, mxid) + await api.joinRoom(roomID, mxid) db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid) } return mxid diff --git a/src/d2m/actions/register-webhook-user.js b/src/d2m/actions/register-webhook-user.js index 309a120b..145eeb8f 100644 --- a/src/d2m/actions/register-webhook-user.js +++ b/src/d2m/actions/register-webhook-user.js @@ -77,16 +77,8 @@ async function ensureSimJoined(fakeUserID, author, roomID) { // Ensure joined const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() if (!existing) { - try { - await api.inviteToRoom(roomID, mxid) - await api.joinRoom(roomID, mxid) - } catch (e) { - if (e.message.includes("is already in the room.")) { - // Sweet! - } else { - throw e - } - } + await api.inviteToRoom(roomID, mxid) + await api.joinRoom(roomID, mxid) db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid) } return mxid diff --git a/src/matrix/api.js b/src/matrix/api.js index 01f9c3be..a7b30f1a 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -79,9 +79,17 @@ async function joinRoom(roomIDOrAlias, mxid, via) { } async function inviteToRoom(roomID, mxidToInvite, mxid) { - await mreq.mreq("POST", path(`/client/v3/rooms/${roomID}/invite`, mxid), { - user_id: mxidToInvite - }) + try { + await mreq.mreq("POST", path(`/client/v3/rooms/${roomID}/invite`, mxid), { + user_id: mxidToInvite + }) + } catch (e) { + if (e.message.includes("is already in the room.") || e.message.includes("cannot invite user that is joined")) { + // Sweet! + } else { + throw e + } + } } async function leaveRoom(roomID, mxid) { From 536ab56048d890d7d5e8a84d5507ee70ca2555fb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Jan 2026 22:19:54 +1300 Subject: [PATCH 064/153] Better text spoilers Remove spoiler content from plaintext body Don't bridge embeds if their link is spoilered (deliberately imprecise) --- package-lock.json | 37 +++++++------------ package.json | 3 +- .../message-to-event.embeds.test.js | 15 ++++++++ src/d2m/converters/message-to-event.js | 17 ++++++++- src/d2m/converters/message-to-event.test.js | 14 +++++++ 5 files changed, 60 insertions(+), 26 deletions(-) diff --git a/package-lock.json b/package-lock.json index d58d72b5..bf857a5a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "AGPL-3.0-or-later", "dependencies": { "@chriscdn/promise-semaphore": "^3.0.1", - "@cloudrac3r/discord-markdown": "^2.6.7", + "@cloudrac3r/discord-markdown": "^2.6.10", "@cloudrac3r/giframe": "^0.4.3", "@cloudrac3r/html-template-tag": "^5.0.1", "@cloudrac3r/in-your-element": "^1.1.1", @@ -242,9 +242,9 @@ } }, "node_modules/@cloudrac3r/discord-markdown": { - "version": "2.6.8", - "resolved": "https://registry.npmjs.org/@cloudrac3r/discord-markdown/-/discord-markdown-2.6.8.tgz", - "integrity": "sha512-ZrSimHqmLqXR+W3U1n6ge6poAjmQaMzXyWrTkT36znrgKhfuQAYxLBtKTf7m+cmr3VlaDVM2P+iPdSeTeaM0qg==", + "version": "2.6.10", + "resolved": "https://registry.npmjs.org/@cloudrac3r/discord-markdown/-/discord-markdown-2.6.10.tgz", + "integrity": "sha512-E+F9UYDUHP2kHDCciX63SBzgsUnHpu2Pp/h98x9Zo+vKuzXjCQ5PcFNdUlH6M18bvHDZPoIsKVmjnON8UYaAPQ==", "license": "MIT", "dependencies": { "simple-markdown": "^0.7.3" @@ -1227,26 +1227,15 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/prop-types": { - "version": "15.7.11", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", - "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==" - }, "node_modules/@types/react": { - "version": "18.2.55", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.55.tgz", - "integrity": "sha512-Y2Tz5P4yz23brwm2d7jNon39qoAtMMmalOQv6+fEFt1mT+FcM3D841wDpoUvFXhaYenuROCy3FZYqdTjM7qVyA==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.8.tgz", + "integrity": "sha512-3MbSL37jEchWZz2p2mjntRZtPt837ij10ApxKfgmXCTuHWagYg7iA5bqPw6C8BMPfwidlvfPI/fxOc42HLhcyg==", + "license": "MIT", "dependencies": { - "@types/prop-types": "*", - "@types/scheduler": "*", - "csstype": "^3.0.2" + "csstype": "^3.2.2" } }, - "node_modules/@types/scheduler": { - "version": "0.16.8", - "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz", - "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==" - }, "node_modules/acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", @@ -1668,9 +1657,10 @@ } }, "node_modules/csstype": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", - "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "license": "MIT" }, "node_modules/data-uri-to-buffer": { "version": "2.0.2", @@ -2825,6 +2815,7 @@ "version": "0.7.3", "resolved": "https://registry.npmjs.org/simple-markdown/-/simple-markdown-0.7.3.tgz", "integrity": "sha512-uGXIc13NGpqfPeFJIt/7SHHxd6HekEJYtsdoCM06mEBPL9fQH/pSD7LRM6PZ7CKchpSvxKL4tvwMamqAaNDAyg==", + "license": "MIT", "dependencies": { "@types/react": ">=16.0.0" } diff --git a/package.json b/package.json index 05a4220e..64e6f77c 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ }, "dependencies": { "@chriscdn/promise-semaphore": "^3.0.1", - "@cloudrac3r/discord-markdown": "^2.6.7", + "@cloudrac3r/discord-markdown": "^2.6.10", "@cloudrac3r/giframe": "^0.4.3", "@cloudrac3r/html-template-tag": "^5.0.1", "@cloudrac3r/in-your-element": "^1.1.1", @@ -64,6 +64,7 @@ "scripts": { "start": "node --enable-source-maps start.js", "setup": "node --enable-source-maps scripts/setup.js", + "build": "mkdir -p dist/out-of-your-element && cp -R src dist/out-of-your-element && cp -R docs dist/out-of-your-element && npx tsdown", "addbot": "node addbot.js", "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js | tap-dot", "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot", diff --git a/src/d2m/converters/message-to-event.embeds.test.js b/src/d2m/converters/message-to-event.embeds.test.js index 85a08cc0..48959930 100644 --- a/src/d2m/converters/message-to-event.embeds.test.js +++ b/src/d2m/converters/message-to-event.embeds.test.js @@ -312,6 +312,21 @@ test("message2event embeds: youtube video", async t => { }]) }) +test("message2event embeds: embed not bridged if its link was spoilered", async t => { + const events = await messageToEvent({ + ...data.message_with_embeds.youtube_video, + content: "||https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E\n\n\nJutomi I'm gonna make these sounds in your walls tonight||" + }, data.guild.general) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "[spoiler]", + format: "org.matrix.custom.html", + formatted_body: `https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E


    Jutomi I'm gonna make these sounds in your walls tonight    `, + "m.mentions": {} + }]) +}) + test("message2event embeds: tenor gif should show a video link without a provider", async t => { const events = await messageToEvent(data.message_with_embeds.tenor_gif, data.guild.general, {}, {}) t.deepEqual(events, [{ diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 4ff1f4d9..73fd691c 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -26,8 +26,9 @@ const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) * @param {DiscordTypes.APIMessage} message * @param {DiscordTypes.APIGuild} guild * @param {boolean} useHTML + * @param {string[]} spoilers */ -function getDiscordParseCallbacks(message, guild, useHTML) { +function getDiscordParseCallbacks(message, guild, useHTML, spoilers = []) { return { /** @param {{id: string, type: "discordUser"}} node */ user: node => { @@ -90,6 +91,10 @@ function getDiscordParseCallbacks(message, guild, useHTML) { here: () => { if (message.mention_everyone) return "@room" return "@here" + }, + spoiler: node => { + spoilers.push(node.raw) + return useHTML } } } @@ -392,6 +397,7 @@ async function messageToEvent(message, guild, options = {}, di) { return content.replace(/https:\/\/(cdn|media)\.discordapp\.(?:com|net)\/attachments\/([0-9]+)\/([0-9]+)\/([-A-Za-z0-9_.,]+)/g, url => dUtils.getPublicUrlForCdn(url)) } + const spoilers = [] /** * Translate links and emojis and mentions and stuff. Give back the text and HTML so they can be combined into bigger events. * @param {string} content Partial or complete Discord message content @@ -431,7 +437,7 @@ async function messageToEvent(message, guild, options = {}, di) { } let html = await markdown.toHtmlWithPostParser(content, transformParsedVia, { - discordCallback: getDiscordParseCallbacks(message, guild, true), + discordCallback: getDiscordParseCallbacks(message, guild, true, spoilers), ...customOptions }, customParser, customHtmlOutput) @@ -692,6 +698,13 @@ async function messageToEvent(message, guild, options = {}, di) { continue // If discord creates an embed preview for a discord channel link, don't copy that embed } + if (embed.url && spoilers.some(sp => sp.match(/\bhttps?:\/\/[a-z]/))) { + // If the original message had spoilered URLs, don't generate any embeds for links. + // This logic is the same as the Discord desktop client. It doesn't match specific embeds to specific spoilered text, it's all or nothing. + // It's not easy to do much better because posting a link like youtu.be generates an embed.url with youtube.com/watch, so you can't match up the text without making at least that a special case. + continue + } + // Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once const rep = new mxUtils.MatrixStringBuilder() diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index 3303b275..aba80d29 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -862,6 +862,20 @@ test("message2event: advanced written @mentions for matrix users", async t => { t.equal(called, 1, "should only look up the member list once") }) +test("message2event: spoilers are removed from plaintext body", async t => { + const events = await messageToEvent({ + content: "||**beatrice**||" + }) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "[spoiler]", + format: "org.matrix.custom.html", + formatted_body: `beatrice` + }]) +}) + test("message2event: very large attachment is linked instead of being uploaded", async t => { const events = await messageToEvent({ content: "hey", From fcd4eb4e51ea93df039a164221209a13f49f9105 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Jan 2026 22:20:11 +1300 Subject: [PATCH 065/153] Refactor cross-room event detection --- src/d2m/converters/message-to-event.js | 74 ++++++++++++--------- src/d2m/converters/message-to-event.test.js | 4 +- 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 73fd691c..1c921231 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -286,8 +286,8 @@ async function messageToEvent(message, guild, options = {}, di) { // Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions // (Still need to do scenarios 1 and 2 part B, and scenario 3.) if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) { - const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? ORDER BY part ASC").get(message.message_reference.message_id) - if (row) { + const row = await getHistoricalEventRow(message.message_reference?.message_id) + if (row && "event_id" in row) { repliedToEventRow = Object.assign(row, {channel_id: row.reference_channel_id}) } else if (message.referenced_message) { repliedToUnknownEvent = true @@ -300,8 +300,8 @@ async function messageToEvent(message, guild, options = {}, di) { assert(message.embeds[0].description) const match = message.embeds[0].description.match(/\/channels\/[0-9]*\/[0-9]*\/([0-9]{2,})/) if (match) { - const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? ORDER BY part ASC").get(match[1]) - if (row) { + const row = await getHistoricalEventRow(match[1]) + if (row && "event_id" in row) { /* we generate a partial referenced_message based on what PK provided. we don't need everything, since this will only be used for further message-to-event converting. the following properties are necessary: @@ -346,6 +346,34 @@ async function messageToEvent(message, guild, options = {}, di) { return promise } + /** + * @param {string} messageID + * @param {string} [timestampChannelID] + */ + async function getHistoricalEventRow(messageID, timestampChannelID) { + /** @type {{room_id: string} | {event_id: string, room_id: string, reference_channel_id: string, source: number} | null} */ + let row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("event_id", "room_id", "reference_channel_id", "source").where({message_id: messageID}).and("ORDER BY part ASC").get() + if (!row && timestampChannelID) { + const ts = dUtils.snowflakeToTimestampExact(messageID) + const oldestRow = from("historical_channel_room").selectUnsafe("max(upgraded_timestamp)", "room_id") + .where({reference_channel_id: timestampChannelID}).and("and upgraded_timestamp < ?").get(ts) + if (oldestRow?.room_id) { + row = {room_id: oldestRow.room_id} + try { + const {event_id} = await di.api.getEventForTimestamp(oldestRow.room_id, ts) + row = { + event_id, + room_id: oldestRow.room_id, + reference_channel_id: oldestRow.reference_channel_id, + source: 1 + } + } catch (e) {} + } + } + return row + } + /** * Translate Discord message links to Matrix event links. * If OOYE has handled this message in the past, this is an instant database lookup. @@ -358,30 +386,11 @@ async function messageToEvent(message, guild, options = {}, di) { assert(typeof match.index === "number") const [_, channelID, messageID] = match const result = await (async () => { - const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") - .select("event_id", "room_id").where({message_id: messageID}).get() - // const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() - if (row) { - const via = await getViaServersMemo(row.room_id) - return `https://matrix.to/#/${row.room_id}/${row.event_id}?${via}` - } - - const ts = dUtils.snowflakeToTimestampExact(messageID) - const oldestRow = from("historical_channel_room").selectUnsafe("max(upgraded_timestamp)", "room_id") - .where({reference_channel_id: channelID}).and("and upgraded_timestamp < ?").get(ts) - if (oldestRow?.room_id) { - const via = await getViaServersMemo(oldestRow.room_id) - try { - const {event_id} = await di.api.getEventForTimestamp(oldestRow.room_id, ts) - return `https://matrix.to/#/${oldestRow.room_id}/${event_id}?${via}` - } catch (e) { - // M_NOT_FOUND: Unable to find event from in direction Direction.FORWARDS - // not supported in Conduit and descendants - return `[unknown event, timestamp resolution failed, in room: https://matrix.to/#/${oldestRow.room_id}?${via}]` - } - } - - return `${match[0]} [event is from another server]` + const row = await getHistoricalEventRow(messageID, channelID) + if (!row) return `${match[0]} [event is from another server]` + const via = await getViaServersMemo(row.room_id) + if (!("event_id" in row)) return `[unknown event in https://matrix.to/#/${row.room_id}?${via}]` + return `https://matrix.to/#/${row.room_id}/${row.event_id}?${via}` })() content = content.slice(0, match.index + offset) + result + content.slice(match.index + match[0].length + offset) @@ -561,17 +570,16 @@ async function messageToEvent(message, guild, options = {}, di) { // Forwarded content appears first if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_snapshots?.length) { // Forwarded notice - const event = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") - .select("event_id", "room_id").where({message_id: message.message_reference.message_id}).get() + const row = await getHistoricalEventRow(message.message_reference.message_id, message.message_reference.channel_id) const room = select("channel_room", ["room_id", "name", "nick"], {channel_id: message.message_reference.channel_id}).get() const forwardedNotice = new mxUtils.MatrixStringBuilder() if (room) { const roomName = room && (room.nick || room.name) - if (event) { - const via = await getViaServersMemo(event.room_id) + if ("event_id" in row) { + const via = await getViaServersMemo(row.room_id) forwardedNotice.addLine( `[🔀 Forwarded from #${roomName}]`, - tag`🔀 Forwarded from ${roomName} [jump to event]` + tag`🔀 Forwarded from ${roomName} [jump to event]` ) } else { const via = await getViaServersMemo(room.room_id) diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index aba80d29..f7769d33 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -287,10 +287,10 @@ test("message2event: message timestamp failed to fetch", async t => { "m.mentions": {}, msgtype: "m.text", body: "Me: I'll scroll up to find a certain message I'll send\n_scrolls up and clicks message links for god knows how long_\n_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n_stumbles upon:_ " - + "[unknown event, timestamp resolution failed, in room: https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org]", + + "[unknown event in https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org]", format: "org.matrix.custom.html", formatted_body: "Me: I'll scroll up to find a certain message I'll send
    scrolls up and clicks message links for god knows how long
    completely forgets what they were looking for and simply begins scrolling up to find some fun moments
    stumbles upon: " - + '[unknown event, timestamp resolution failed, in room: https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org]' + + '[unknown event in https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org]' }]) t.equal(called, 2, "getEventForTimestamp and getJoinedMembers should be called once each") }) From c8b0f23db32f7a4b916a1bdad876ac706076cdb8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Jan 2026 22:57:52 +1300 Subject: [PATCH 066/153] Retrigger m->d reactions and removals --- src/d2m/actions/retrigger.js | 40 ++++++++++++++++++++------------- src/m2d/actions/add-reaction.js | 21 ++++++++++++----- src/m2d/actions/redact.js | 11 +++++++-- src/m2d/event-dispatcher.js | 4 ++++ 4 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/d2m/actions/retrigger.js b/src/d2m/actions/retrigger.js index aa79a798..7ff04263 100644 --- a/src/d2m/actions/retrigger.js +++ b/src/d2m/actions/retrigger.js @@ -20,31 +20,39 @@ const emitter = new EventEmitter() * (or before the it has finished being bridged to an event). * In this case, wait until the original message has finished bridging, then retrigger the passed function. * @template {(...args: any[]) => Promise} T - * @param {string} messageID + * @param {string} inputID * @param {T} fn * @param {Parameters} rest * @returns {boolean} false if the event was found and the function will be ignored, true if the event was not found and the function will be retriggered */ -function eventNotFoundThenRetrigger(messageID, fn, ...rest) { - if (!paused.has(messageID)) { - const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get() - if (eventID) { - debugRetrigger(`[retrigger] OK mid <-> eid = ${messageID} <-> ${eventID}`) - return false // event was found so don't retrigger +function eventNotFoundThenRetrigger(inputID, fn, ...rest) { + if (!paused.has(inputID)) { + if (inputID.match(/^[0-9]+$/)) { + const eventID = select("event_message", "event_id", {message_id: inputID}).pluck().get() + if (eventID) { + debugRetrigger(`[retrigger] OK mid <-> eid = ${inputID} <-> ${eventID}`) + return false // event was found so don't retrigger + } + } else if (inputID.match(/^\$/)) { + const messageID = select("event_message", "message_id", {event_id: inputID}).pluck().get() + if (messageID) { + debugRetrigger(`[retrigger] OK eid <-> mid = ${inputID} <-> ${messageID}`) + return false // message was found so don't retrigger + } } } - debugRetrigger(`[retrigger] WAIT mid = ${messageID}`) - emitter.once(messageID, () => { - debugRetrigger(`[retrigger] TRIGGER mid = ${messageID}`) + debugRetrigger(`[retrigger] WAIT id = ${inputID}`) + emitter.once(inputID, () => { + debugRetrigger(`[retrigger] TRIGGER id = ${inputID}`) fn(...rest) }) // if the event never arrives, don't trigger the callback, just clean up setTimeout(() => { - if (emitter.listeners(messageID).length) { - debugRetrigger(`[retrigger] EXPIRE mid = ${messageID}`) + if (emitter.listeners(inputID).length) { + debugRetrigger(`[retrigger] EXPIRE id = ${inputID}`) } - emitter.removeAllListeners(messageID) + emitter.removeAllListeners(inputID) }, 60 * 1000) // 1 minute return true // event was not found, then retrigger } @@ -58,11 +66,11 @@ function eventNotFoundThenRetrigger(messageID, fn, ...rest) { */ async function pauseChanges(messageID, promise) { try { - debugRetrigger(`[retrigger] PAUSE mid = ${messageID}`) + debugRetrigger(`[retrigger] PAUSE id = ${messageID}`) paused.add(messageID) return await promise } finally { - debugRetrigger(`[retrigger] RESUME mid = ${messageID}`) + debugRetrigger(`[retrigger] RESUME id = ${messageID}`) paused.delete(messageID) messageFinishedBridging(messageID) } @@ -74,7 +82,7 @@ async function pauseChanges(messageID, promise) { */ function messageFinishedBridging(messageID) { if (emitter.listeners(messageID).length) { - debugRetrigger(`[retrigger] EMIT mid = ${messageID}`) + debugRetrigger(`[retrigger] EMIT id = ${messageID}`) } emitter.emit(messageID) } diff --git a/src/m2d/actions/add-reaction.js b/src/m2d/actions/add-reaction.js index 2b19fb23..9ee92769 100644 --- a/src/m2d/actions/add-reaction.js +++ b/src/m2d/actions/add-reaction.js @@ -4,20 +4,27 @@ const assert = require("assert").strict const Ty = require("../../types") const passthrough = require("../../passthrough") -const {discord, sync, db, select} = passthrough +const {discord, as, sync, db, select, from} = passthrough /** @type {import("../../matrix/utils")} */ const utils = sync.require("../../matrix/utils") /** @type {import("../converters/emoji")} */ const emoji = sync.require("../converters/emoji") +/** @type {import("../../d2m/actions/retrigger")} */ +const retrigger = sync.require("../../d2m/actions/retrigger") /** * @param {Ty.Event.Outer} event */ async function addReaction(event) { - const channelID = select("historical_channel_room", "reference_channel_id", {room_id: event.room_id}).pluck().get() - if (!channelID) return // We just assume the bridge has already been created - const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id}, "ORDER BY reaction_part").pluck().get() - if (!messageID) return // Nothing can be done if the parent message was never bridged. + // Wait until the corresponding channel and message have already been bridged + if (retrigger.eventNotFoundThenRetrigger(event.content["m.relates_to"].event_id, as.emit.bind(as, "type:m.reaction", event))) return + + // These will exist because it passed retrigger + const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") + .select("message_id", "reference_channel_id").where({event_id: event.content["m.relates_to"].event_id}).and("ORDER BY reaction_part ASC").get() + assert(row) + const messageID = row.message_id + const channelID = row.reference_channel_id const key = event.content["m.relates_to"].key const discordPreferredEncoding = await emoji.encodeEmoji(key, event.content.shortcode) @@ -35,6 +42,10 @@ async function addReaction(event) { // happens if a matrix user tries to add on to a super reaction return } + if (e.message?.includes("Unknown Message")) { + // happens under a race condition where a message is deleted after it passes the database check above + return + } throw e } diff --git a/src/m2d/actions/redact.js b/src/m2d/actions/redact.js index 9f99ec13..022157de 100644 --- a/src/m2d/actions/redact.js +++ b/src/m2d/actions/redact.js @@ -4,9 +4,11 @@ const DiscordTypes = require("discord-api-types/v10") const Ty = require("../../types") const passthrough = require("../../passthrough") -const {discord, sync, db, select, from} = passthrough +const {discord, as, sync, db, select, from} = passthrough /** @type {import("../../matrix/utils")} */ const utils = sync.require("../../matrix/utils") +/** @type {import("../../d2m/actions/retrigger")} */ +const retrigger = sync.require("../../d2m/actions/retrigger") /** * @param {Ty.Event.Outer_M_Room_Redaction} event @@ -52,13 +54,18 @@ async function removeReaction(event) { * @param {Ty.Event.Outer_M_Room_Redaction} event */ async function handle(event) { + // If this is for removing a reaction, try it + await removeReaction(event) + + // Or, it might be for removing a message or suppressing embeds. But to do that, the message needs to be bridged first. + if (retrigger.eventNotFoundThenRetrigger(event.redacts, as.emit.bind(as, "type:m.room.redaction", event))) return + const row = select("event_message", ["event_type", "event_subtype", "part"], {event_id: event.redacts}).get() if (row && row.event_type === "m.room.message" && row.event_subtype === "m.notice" && row.part === 1) { await suppressEmbeds(event) } else { await deleteMessage(event) } - await removeReaction(event) } module.exports.handle = handle diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index d9108528..e1f6922a 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -28,6 +28,8 @@ const api = sync.require("../matrix/api") const createRoom = sync.require("../d2m/actions/create-room") /** @type {import("../matrix/room-upgrade")} */ const roomUpgrade = require("../matrix/room-upgrade") +/** @type {import("../d2m/actions/retrigger")} */ +const retrigger = sync.require("../d2m/actions/retrigger") const {reg} = require("../matrix/read-registration") let lastReportedEvent = 0 @@ -201,6 +203,7 @@ async event => { // @ts-ignore await matrixCommandHandler.execute(event) } + retrigger.messageFinishedBridging(event.event_id) await api.ackEvent(event) })) @@ -211,6 +214,7 @@ sync.addTemporaryListener(as, "type:m.sticker", guard("m.sticker", async event => { if (utils.eventSenderIsFromDiscord(event.sender)) return const messageResponses = await sendEvent.sendEvent(event) + retrigger.messageFinishedBridging(event.event_id) await api.ackEvent(event) })) From c7e8d4d58be1f62b2457fb4cc28df19db64e8696 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Jan 2026 23:01:58 +1300 Subject: [PATCH 067/153] Storage is 15% more efficient now --- docs/developer-orientation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer-orientation.md b/docs/developer-orientation.md index 94a420ca..056fe7e5 100644 --- a/docs/developer-orientation.md +++ b/docs/developer-orientation.md @@ -9,7 +9,7 @@ Using WeatherStack as a thin layer between the bridge application and the Discord API lets us control exactly what data is cached in memory. Only necessary information is cached. For example, member data, user data, message content, and past edits are never stored in memory. This keeps the memory usage low and also prevents it ballooning in size over the bridge's runtime. -The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs and URLs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 300,000 messages (representing a year of conversations) requires 47.3 MB of storage space in the SQLite database. +The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs and URLs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 300,000 messages (representing a year of conversations) requires 40.6 MB of storage space in the SQLite database. Only necessary data and columns are queried from the database. We only contact the homeserver API if the database doesn't contain what we need. From 03e9fc4b9cb6a83a34d4f0b88ee423b0c6ad031b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Jan 2026 23:08:00 +1300 Subject: [PATCH 068/153] Room upgrades technical information --- docs/room-upgrades.md | 80 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 docs/room-upgrades.md diff --git a/docs/room-upgrades.md b/docs/room-upgrades.md new file mode 100644 index 00000000..49308fd0 --- /dev/null +++ b/docs/room-upgrades.md @@ -0,0 +1,80 @@ +# Room upgrades + +"Upgrading" a room is supposed to create a new room and then tries to set it up exactly like the old one. So it copies name, topic, power levels, space membership, etc. The old room is marked as old with an `m.room.tombstone` event, its power levels are adjusted to make it harder to send messages, and a hyperlink to the new room is added. + +## What happens? + +A room upgrade is triggered by a POST request to `/_matrix/client/v3/rooms/{roomId}/upgrade`. The upgrade process is done by the server, and involves multiple events across multiple rooms. Since this is server-specific, what will _actually_ happen depends on the server's implementation, but the spec says it does this: + +1. Checks that the user has permission to send `m.room.tombstone` events in the room. +2. Creates a replacement room with a `m.room.create` event containing a predecessor field, the applicable `room_version`, and a `type` field which is copied from the predecessor room. If no `type` is set on the previous room, no `type` is specified on the new room’s create event either. +3. Replicates transferable state events to the new room. The exact details for what is transferred is left as an implementation detail, however the recommended state events to transfer are: + * `m.room.server_acl` + * `m.room.encryption` + * `m.room.name` + * `m.room.avatar` + * `m.room.topic` + * `m.room.guest_access` + * `m.room.history_visibility` + * `m.room.join_rules` + * `m.room.power_levels` + + (Membership can't be transferred by the server.) + +4. Moves any local aliases to the new room. +5. Sends a `m.room.tombstone` event to the old room to indicate that it is not intended to be used any further. +6. If possible, the power levels in the old room should also be modified to prevent sending of events and inviting new users. For example, setting `events_default` and `invite` to the greater of `50` and `users_default + 1`. + +### Synapse additionally: + +1. Copies its `m.space.child` events (if it was a space). + * This is good for OOYE, because it automatically tries to join new rooms when they're added to a registered space. +2. Copies bans. +3. Un/publishes to the public room directory as applicable. +4. Copies user tags and push rules. + +Conduwuit does not do those! + +### Element additionally: + +1. May invite all users from the old room to the new room, depending on if the checkbox is checked in the dialog. +2. Update parent spaces to remove the old room and add the new room. + +Cinny does not do those! The new room is totally detached! The hyperlink from the old room (and the moved alias by server) is the only way to find it! + +* This is probably still okay for OOYE? Since the join rules are preserved, and if they were `restricted`, OOYE is able to join via the tombstone hyperlink. Then, after it joins, it's already PL 100 since the power levels are preserved. It's very bad if the join rules were `invite`, but OOYE never sets this join rule - it's either `restricted` or `public`. + +### Other clients + +Nheko doesn't support room upgrades at all. Cinyy, NeoChat and FluffyChat just call the API and don't do anything. FluffyChat invites all joined/invited users to the new room if the join rule is restricted. + +### Notable things that don't happen at all: + +* Add `m.space.parent` pointing to the space it was in (if it was a room in a space). + +## What should OOYE do? + +### Ideal case (Element, Synapse) + +The new room is added to the space and OOYE autojoins it. It already has the correct power levels and join rules. + +OOYE still needs to do this: + +1. Un/set `m.room.parent` in the rooms. +2. Update `channel_room` and `historical_channel_room` tables. + +### Not ideal case (everyone else) + +OOYE should: + +1. Join the room by following the hyperlink from the tombstone, if able + * If not able, somebody messed with the join rules. Send a PM to the user who upgraded - the new room's creator - asking for an invite. +2. Wait for join. +3. Un/set `m.space.child` events on the space. +4. Un/set `m.room.parent` in the rooms. +5. Update `channel_room` and `historical_channel_room` tables. +6. Un/publish to the room directory. + +### It's actually fine to do all the steps always + +Even by blindly following the entire list, each step is a no-op or atomic, so it doesn't matter if Element is also trying to do them. From a488c153518aaf1c92a6bf5a7e5e704967bdc107 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 14 Jan 2026 15:07:16 +1300 Subject: [PATCH 069/153] Fix confusion between eventsToSend/unchangedEvents --- src/d2m/converters/edit-to-changes.js | 37 +++++++++++++++++++-------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index 8e4c9a2d..ff2a96f4 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -75,13 +75,25 @@ async function editToChanges(message, guild, api) { + The events must have the same subtype. Events will therefore be divided into four categories: */ - /** 1. Events that are matched, and should be edited by sending another m.replace event */ + /** + * 1. Events that are matched, and should be edited by sending another m.replace event + * @type {{old: typeof oldEventRows[0], newFallbackContent: typeof newFallbackContent[0], newInnerContent: typeof newInnerContent[0]}[]} + */ let eventsToReplace = [] - /** 2. Events that are present in the old version only, and should be blanked or redacted */ + /** + * 2. Events that are present in the old version only, and should be blanked or redacted + * @type {{old: typeof oldEventRows[0]}[]} + */ let eventsToRedact = [] - /** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */ + /** + * 3. Events that are present in the new version only, and should be sent as new, with references back to the context + * @type {typeof newInnerContent} + */ let eventsToSend = [] - /** 4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. */ + /** + * 4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. + * @type {(typeof eventsToRedact[0] | typeof eventsToReplace[0])[]} + */ let unchangedEvents = [] function shift() { @@ -124,15 +136,14 @@ async function editToChanges(message, guild, api) { eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice") unchangedEvents.push(...eventsToReplace.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToReplace to unchangedEvents. eventsToReplace = eventsToReplace.filter(e => e.old.event_subtype === "m.notice") - unchangedEvents.push(...eventsToSend.filter(e => e.msgtype !== "m.notice")) // Move them from eventsToSend to unchangedEvents. - eventsToSend = eventsToSend.filter(e => e.msgtype === "m.notice") + eventsToSend = eventsToSend.filter(e => e.msgtype === "m.notice") // Don't send new events that aren't the embed. // Don't post new generated embeds for messages if it's been a while since the message was sent. Detached embeds look weird. const messageTooOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 120 * 1000 // older than 2 minutes ago // Don't post new generated embeds for messages if the setting was disabled. const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 if (messageTooOld || !embedsEnabled) { - eventsToSend = eventsToSend.filter(e => e.msgtype !== "m.notice") + eventsToSend = [] } } @@ -196,10 +207,14 @@ async function editToChanges(message, guild, api) { } // Removing unnecessary properties before returning - eventsToRedact = eventsToRedact.map(e => e.old.event_id) - eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)})) - - return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions} + return { + roomID, + eventsToReplace: eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)})), + eventsToRedact: eventsToRedact.map(e => e.old.event_id), + eventsToSend, + senderMxid, + promotions + } } /** From fba50e95051b3209359e28694cac52b4eb230867 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 14 Jan 2026 15:11:22 +1300 Subject: [PATCH 070/153] Fix getPermissions misunderstanding @everyone --- src/d2m/actions/create-room.js | 2 +- src/d2m/actions/register-user.js | 4 ++-- src/d2m/event-dispatcher.js | 4 ++-- src/discord/utils.js | 11 +++++------ src/discord/utils.test.js | 4 ++-- src/m2d/converters/event-to-message.js | 4 ++-- src/matrix/matrix-command-handler.js | 4 ++-- src/web/pug/guild.pug | 2 +- src/web/routes/guild.js | 2 +- 9 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 11a03e5a..fc2d35f5 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -122,7 +122,7 @@ async function channelToKState(channel, guild, di) { join_rules = {join_rule: PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel]} } - const everyonePermissions = dUtils.getPermissions([], guild.roles, undefined, channel.permission_overwrites) + const everyonePermissions = dUtils.getPermissions(guild.id, [], guild.roles, undefined, channel.permission_overwrites) const everyoneCanSend = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendMessages) const everyoneCanMentionEveryone = dUtils.hasAllPermissions(everyonePermissions, ["MentionEveryone"]) diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index 35da4f5c..d20bfb8a 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -153,8 +153,8 @@ async function memberToStateContent(user, member, guildID) { function memberToPowerLevel(user, member, guild, channel) { if (!member) return 0 - const permissions = dUtils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites) - const everyonePermissions = dUtils.getPermissions([], guild.roles, undefined, channel.permission_overwrites) + const permissions = dUtils.getPermissions(guild.id, member.roles, guild.roles, user.id, channel.permission_overwrites) + const everyonePermissions = dUtils.getPermissions(guild.id, [], guild.roles, undefined, channel.permission_overwrites) /* * PL 100 = Administrator = People who can brick the room. RATIONALE: * - Administrator. diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index bf7efb9a..834cf9d7 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -87,7 +87,7 @@ module.exports = { const member = guild.members.find(m => m.user?.id === client.user.id) if (!member) return if (!("permission_overwrites" in channel)) continue - const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const permissions = dUtils.getPermissions(guild.id, member.roles, guild.roles, client.user.id, channel.permission_overwrites) if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look back in this channel /** More recent messages come first. */ @@ -146,7 +146,7 @@ module.exports = { const lastPin = updatePins.convertTimestamp(channel.last_pin_timestamp) // Permissions check - const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const permissions = dUtils.getPermissions(guild.id, member.roles, guild.roles, client.user.id, channel.permission_overwrites) if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look up the pins in this channel const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get() diff --git a/src/discord/utils.js b/src/discord/utils.js index 963f0b8f..441a55ce 100644 --- a/src/discord/utils.js +++ b/src/discord/utils.js @@ -15,19 +15,18 @@ require("xxhash-wasm")().then(h => hasher = h) const EPOCH = 1420070400000 /** + * @param {string} guildID * @param {string[]} userRoles * @param {DiscordTypes.APIGuild["roles"]} guildRoles * @param {string} [userID] * @param {DiscordTypes.APIGuildChannel["permission_overwrites"]} [channelOverwrites] */ -function getPermissions(userRoles, guildRoles, userID, channelOverwrites) { +function getPermissions(guildID, userRoles, guildRoles, userID, channelOverwrites) { let allowed = BigInt(0) - let everyoneID // Guild allows for (const role of guildRoles) { - if (role.name === "@everyone") { + if (role.id === guildID) { allowed |= BigInt(role.permissions) - everyoneID = role.id } if (userRoles.includes(role.id)) { allowed |= BigInt(role.permissions) @@ -38,9 +37,9 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) { /** @type {((overwrite: Required) => any)[]} */ const actions = [ // Channel @everyone deny - overwrite => overwrite.id === everyoneID && (allowed &= ~BigInt(overwrite.deny)), + overwrite => overwrite.id === guildID && (allowed &= ~BigInt(overwrite.deny)), // Channel @everyone allow - overwrite => overwrite.id === everyoneID && (allowed |= BigInt(overwrite.allow)), + overwrite => overwrite.id === guildID && (allowed |= BigInt(overwrite.allow)), // Role deny overwrite => userRoles.includes(overwrite.id) && (allowed &= ~BigInt(overwrite.deny)), // Role allow diff --git a/src/discord/utils.test.js b/src/discord/utils.test.js index 516bd2f6..3a3e1772 100644 --- a/src/discord/utils.test.js +++ b/src/discord/utils.test.js @@ -79,7 +79,7 @@ test("getPermissions: channel overwrite to allow role works", t => { { type: 0, id: "1168988246680801360", deny: "0", allow: "1024" }, { type: 1, id: "353373325575323648", deny: "0", allow: "1024" } ] - const permissions = utils.getPermissions(userRoles, guildRoles, userID, overwrites) + const permissions = utils.getPermissions("1154868424724463687", userRoles, guildRoles, userID, overwrites) const want = BigInt(1 << 10 | 1 << 16) t.equal((permissions & want), want) }) @@ -140,7 +140,7 @@ test("getPermissions: channel overwrite to allow user works", t => { { type: 0, id: "1168988246680801360", deny: "0", allow: "1024" }, { type: 1, id: "353373325575323648", deny: "0", allow: "1024" } ] - const permissions = utils.getPermissions(userRoles, guildRoles, userID, overwrites) + const permissions = utils.getPermissions("1154868424724463687", userRoles, guildRoles, userID, overwrites) const want = BigInt(1 << 10 | 1 << 16) t.equal((permissions & want), want) }) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 273521ce..b9f80f35 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -868,7 +868,7 @@ async function eventToMessage(event, guild, channel, di) { let shouldSuppress = inBody !== -1 && event.content.body[inBody-1] === "<" if (!shouldSuppress && guild?.roles) { // Suppress if regular users don't have permission - const permissions = dUtils.getPermissions([], guild.roles) + const permissions = dUtils.getPermissions(guild.id, [], guild.roles) const canEmbedLinks = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.EmbedLinks) shouldSuppress = !canEmbedLinks } @@ -931,7 +931,7 @@ async function eventToMessage(event, guild, channel, di) { // Suppress if regular users don't have permission if (!shouldSuppress && guild?.roles) { - const permissions = dUtils.getPermissions([], guild.roles, undefined, channel.permission_overwrites) + const permissions = dUtils.getPermissions(guild.id, [], guild.roles, undefined, channel.permission_overwrites) const canEmbedLinks = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.EmbedLinks) shouldSuppress = !canEmbedLinks } diff --git a/src/matrix/matrix-command-handler.js b/src/matrix/matrix-command-handler.js index 721c3bdf..c1c69f15 100644 --- a/src/matrix/matrix-command-handler.js +++ b/src/matrix/matrix-command-handler.js @@ -114,7 +114,7 @@ const commands = [{ const guild = discord.guilds.get(guildID) assert(guild) const slots = getSlotCount(guild.premium_tier) - const permissions = dUtils.getPermissions([], guild.roles) + const permissions = dUtils.getPermissions(guild.id, [], guild.roles) if (guild.emojis.length >= slots) { matrixOnlyReason = "CAPACITY" } else if (!(permissions & 0x40000000n)) { // MANAGE_GUILD_EXPRESSIONS (apparently CREATE_GUILD_EXPRESSIONS isn't good enough...) @@ -250,7 +250,7 @@ const commands = [{ const guild = discord.guilds.get(guildID) assert(guild) - const permissions = dUtils.getPermissions([], guild.roles) + const permissions = dUtils.getPermissions(guild.id, [], guild.roles) if (!(permissions & 0x800000000n)) { // CREATE_PUBLIC_THREADS return api.sendEvent(event.room_id, "m.room.message", { ...ctx, diff --git a/src/web/pug/guild.pug b/src/web/pug/guild.pug index c09ca7e6..c219e29d 100644 --- a/src/web/pug/guild.pug +++ b/src/web/pug/guild.pug @@ -13,7 +13,7 @@ mixin badge-private mixin discord(channel, radio=false) //- Previously, we passed guild.roles as the second parameter, but this doesn't quite match Discord's behaviour. See issue #42 for why this was changed. //- Basically we just want to assign badges based on the channel overwrites, without considering the guild's base permissions. /shrug - - let permissions = dUtils.getPermissions([], [{id: guild_id, name: "@everyone", permissions: 1<<10 | 1<<11}], null, channel.permission_overwrites) + - let permissions = dUtils.getPermissions(guild_id, [], [{id: guild_id, name: "@everyone", permissions: 1<<10 | 1<<11}], null, channel.permission_overwrites) .s-user-card.s-user-card__small if !dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) != icons.Icons.IconLock diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index 9037d40e..47b710bb 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -128,7 +128,7 @@ function getChannelRoomsLinks(guild, rooms, roles) { let unlinkedChannels = unlinkedChannelIDs.map(c => discord.channels.get(c)) let removedWrongTypeChannels = filterTo(unlinkedChannels, c => c && [0, 5].includes(c.type)) let removedPrivateChannels = filterTo(unlinkedChannels, c => { - const permissions = dUtils.getPermissions(roles, guild.roles, botID, c["permission_overwrites"]) + const permissions = dUtils.getPermissions(guild.id, roles, guild.roles, botID, c["permission_overwrites"]) return dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) }) unlinkedChannels.sort((a, b) => getPosition(a, discord.channels) - getPosition(b, discord.channels)) From c472388acc8c5474d4c41e3db273c5204233463a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 15 Jan 2026 14:17:03 +1300 Subject: [PATCH 071/153] Room create event must be included to diff --- src/d2m/actions/create-room.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index fc2d35f5..f9137184 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -273,7 +273,7 @@ async function postApplyPowerLevels(kstate, callback) { const originalPowerLevels = await api.getStateEvent(roomID, "m.room.power_levels", "") const powerLevelsDiff = ks.diffKState( - {"m.room.power_levels/": originalPowerLevels}, + {"m.room.power_levels/": originalPowerLevels, "m.room.create/": roomCreate.content, "m.room.create/outer": roomCreate}, {"m.room.power_levels/": powerLevelContent} ) await ks.applyKStateDiffToRoom(roomID, powerLevelsDiff) From 1741bc0fa7fcba35bcc094c1f82a9142a52331b0 Mon Sep 17 00:00:00 2001 From: abdul <32655037-CamperThumper@users.noreply.gitlab.com> Date: Sat, 17 Jan 2026 12:15:04 +0300 Subject: [PATCH 072/153] Fix webp animated emojis https://discord.com/developers/docs/reference#image-formatting-cdn-endpoints > WebP and AVIF formats must be requested as WebP. --- src/d2m/actions/expression.js | 2 +- src/matrix/file.js | 6 +++--- test/ooye-test-data.sql | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/d2m/actions/expression.js b/src/d2m/actions/expression.js index fd75aa5b..1c95dda1 100644 --- a/src/d2m/actions/expression.js +++ b/src/d2m/actions/expression.js @@ -24,7 +24,7 @@ async function emojisToState(emojis) { file.uploadDiscordFileToMxc(file.emoji(emoji.id, emoji.animated)).then(url => { result.images[emoji.name] = { info: { - mimetype: emoji.animated ? "image/gif" : "image/png" + mimetype: "image/webp" }, url } diff --git a/src/matrix/file.js b/src/matrix/file.js index 2070a568..7bc1fec5 100644 --- a/src/matrix/file.js +++ b/src/matrix/file.js @@ -103,9 +103,9 @@ function memberAvatar(guildID, user, member) { } function emoji(emojiID, animated) { - const base = `/emojis/${emojiID}` - if (animated) return base + ".gif" - else return base + ".png" + const base = `/emojis/${emojiID}.webp` + if (animated) return base + "?animated=true" + else return base } const stickerFormat = new Map([ diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 36ec0b61..ef3dc5f7 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -145,7 +145,7 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/emojis/230201364309868544.png', 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), ('https://cdn.discordapp.com/emojis/393635038903926784.gif', 'mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc'), ('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'), -('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'), +('https://cdn.discordapp.com/emojis/1125827250609201255.webp', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'), ('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX'), ('https://cdn.discordapp.com/emojis/288858540888686602.png', 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO'), ('https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4', 'mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU'), From 92a60955bca0a1e24c42179b3bc051695be60298 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 18 Jan 2026 02:39:17 +1300 Subject: [PATCH 073/153] Improve Matrix info and add alternative access React with red question mark to get a DM of the author info, like PK. --- src/d2m/event-dispatcher.js | 13 +++++- src/discord/interactions/matrix-info.js | 43 ++++++++++++++++++-- src/discord/interactions/matrix-info.test.js | 20 +++++++-- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 834cf9d7..0a619ef8 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -34,6 +34,8 @@ const retrigger = sync.require("./actions/retrigger") const setPresence = sync.require("./actions/set-presence") /** @type {import("../m2d/event-dispatcher")} */ const matrixEventDispatcher = sync.require("../m2d/event-dispatcher") +/** @type {import("../discord/interactions/matrix-info")} */ +const matrixInfoInteraction = sync.require("../discord/interactions/matrix-info") const {Semaphore} = require("@chriscdn/promise-semaphore") const checkMissedPinsSema = new Semaphore() @@ -299,7 +301,16 @@ module.exports = { */ async MESSAGE_REACTION_ADD(client, data) { if (data.user_id === client.user.id) return // m2d reactions are added by the discord bot user - do not reflect them back to matrix. - await addReaction.addReaction(data) + if (data.emoji.name === "❓" && select("event_message", "message_id", {message_id: data.message_id, source: 0})) { + const guild_id = data.guild_id ?? client.channels.get(data.channel_id)["guild_id"] + await Promise.all([ + client.snow.channel.deleteReaction(data.channel_id, data.message_id, data.emoji.name).catch(() => {}), + // @ts-ignore - this is all you need for it to do a matrix-side lookup + matrixInfoInteraction.dm({guild_id, data: {target_id: data.message_id}, member: {user: {id: data.user_id}}}) + ]) + } else { + await addReaction.addReaction(data) + } }, /** diff --git a/src/discord/interactions/matrix-info.js b/src/discord/interactions/matrix-info.js index e35cde1d..ca7da5ba 100644 --- a/src/discord/interactions/matrix-info.js +++ b/src/discord/interactions/matrix-info.js @@ -7,12 +7,18 @@ const assert = require("assert").strict /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") + +/** @type {import("../../web/routes/guild")} */ +const webGuild = sync.require("../../web/routes/guild") + /** * @param {DiscordTypes.APIMessageApplicationCommandGuildInteraction} interaction * @param {{api: typeof api}} di * @returns {Promise} */ -async function _interact({guild_id, channel, data}, {api}) { +async function _interact({guild_id, data}, {api}) { const message = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") .select("source", "reference_channel_id", "room_id", "event_id").where({message_id: data.target_id, part: 0}).get() @@ -47,11 +53,32 @@ async function _interact({guild_id, channel, data}, {api}) { // from Matrix const event = await api.getEvent(message.room_id, message.event_id) + const via = await utils.getViaServersQuery(message.room_id, api) + const inChannels = discord.guildChannelMap.get(guild_id) + .map(cid => discord.channels.get(cid)) + .sort((a, b) => webGuild._getPosition(a, discord.channels) - webGuild._getPosition(b, discord.channels)) + .filter(channel => from("channel_room").join("member_cache", "room_id").select("mxid").where({channel_id: channel.id, mxid: event.sender}).get()) + const matrixMember = select("member_cache", ["displayname", "avatar_url"], {room_id: message.room_id, mxid: event.sender}).get() + const name = matrixMember?.displayname || event.sender return { type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, data: { - content: `Bridged [${event.sender}]()'s message in [${roomName}]() on Matrix to https://discord.com/channels/${guild_id}/${channel_id}/${data.target_id} on Discord.` - + idInfo, + embeds: [{ + author: { + name, + url: `https://matrix.to/#/${event.sender}`, + icon_url: utils.getPublicUrlForMxc(matrixMember.avatar_url) + }, + description: `This Matrix message was delivered to Discord by **Out Of Your Element**.\n[View on Matrix →]()\n\n**User ID**: [${event.sender}]()`, + color: 0x0dbd8b, + fields: [{ + name: "In Channels", + value: inChannels.map(c => `<#${c.id}>`).join(" • ") + }, { + name: "\u200b", + value: idInfo + }] + }], flags: DiscordTypes.MessageFlags.Ephemeral } } @@ -64,5 +91,15 @@ async function interact(interaction) { await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, await _interact(interaction, {api})) } +/** @param {DiscordTypes.APIMessageApplicationCommandGuildInteraction} interaction */ +async function dm(interaction) { + const channel = await discord.snow.user.createDirectMessageChannel(interaction.member.user.id) + const response = await _interact(interaction, {api}) + assert(response.type === DiscordTypes.InteractionResponseType.ChannelMessageWithSource) + response.data.flags &= 0 // not ephemeral + await discord.snow.channel.createMessage(channel.id, response.data) +} + module.exports.interact = interact module.exports._interact = _interact +module.exports.dm = dm diff --git a/src/discord/interactions/matrix-info.test.js b/src/discord/interactions/matrix-info.test.js index 5d0206e7..f4557006 100644 --- a/src/discord/interactions/matrix-info.test.js +++ b/src/discord/interactions/matrix-info.test.js @@ -60,13 +60,27 @@ test("matrix info: shows info for matrix source message", async t => { }, sender: "@cadence:cadence.moe" } + }, + async getJoinedMembers(roomID) { + return { + joined: {} + } + }, + async getStateEventOuter(roomID, type, key) { + return { + content: { + room_version: "11" + } + } + }, + async getStateEvent(roomID, type, key) { + return {} } } }) t.equal( - msg.data.content, - "Bridged [@cadence:cadence.moe]()'s message in [main]() on Matrix to https://discord.com/channels/112760669178241024/112760669178241024/1128118177155526666 on Discord." - + "\n-# Room ID: `!kLRqKKUQXcibIMtOpl:cadence.moe`" + msg.data.embeds[0].fields[1].value, + "\n-# Room ID: `!kLRqKKUQXcibIMtOpl:cadence.moe`" + "\n-# Event ID: `$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4`" ) t.equal(called, 1) From 014a87ed9e74d939d6400b1833870e4dbed50bf6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 18 Jan 2026 02:53:39 +1300 Subject: [PATCH 074/153] Force Matrix m.notices to be unchanged events --- src/d2m/converters/edit-to-changes.js | 19 +++++++-------- src/discord/utils.js | 21 +++++++++++++++++ src/discord/utils.test.js | 7 ++++++ src/web/routes/guild.js | 33 +++++---------------------- src/web/routes/guild.test.js | 9 +------- 5 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index ff2a96f4..a3952096 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -6,8 +6,10 @@ const passthrough = require("../../passthrough") const {sync, select, from} = passthrough /** @type {import("./message-to-event")} */ const messageToEvent = sync.require("../converters/message-to-event") +/** @type {import("../../discord/utils")} */ +const dUtils = sync.require("../../discord/utils") /** @type {import("../../matrix/utils")} */ -const utils = sync.require("../../matrix/utils") +const mxUtils = sync.require("../../matrix/utils") function eventCanBeEdited(ev) { // Discord does not allow files, images, attachments, or videos to be edited. @@ -56,7 +58,7 @@ async function editToChanges(message, guild, api) { // Should be a system generated embed. We want the embed to be sent by the same user who sent the message, so that the messages get grouped in most clients. const eventID = oldEventRows[0].event_id // a calling function should have already checked that there is at least one message to edit const event = await api.getEvent(roomID, eventID) - if (utils.eventSenderIsFromDiscord(event.sender)) { + if (mxUtils.eventSenderIsFromDiscord(event.sender)) { senderMxid = event.sender } } @@ -132,14 +134,14 @@ async function editToChanges(message, guild, api) { // If this is a generated embed update, only allow the embeds to be updated, since the system only sends data about events. Ignore changes to other things. // This also prevents Matrix events that were re-subtyped during conversion (e.g. large image -> text link) from being mistakenly included. if (isGeneratedEmbed) { - unchangedEvents.push(...eventsToRedact.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToRedact to unchangedEvents. - eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice") - unchangedEvents.push(...eventsToReplace.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToReplace to unchangedEvents. - eventsToReplace = eventsToReplace.filter(e => e.old.event_subtype === "m.notice") + unchangedEvents = unchangedEvents.concat( + dUtils.filterTo(eventsToRedact, e => e.old.event_subtype === "m.notice" && e.old.source === 1), // Move everything except embeds from eventsToRedact to unchangedEvents. + dUtils.filterTo(eventsToReplace, e => e.old.event_subtype === "m.notice" && e.old.source === 1) // Move everything except embeds from eventsToReplace to unchangedEvents. + ) eventsToSend = eventsToSend.filter(e => e.msgtype === "m.notice") // Don't send new events that aren't the embed. // Don't post new generated embeds for messages if it's been a while since the message was sent. Detached embeds look weird. - const messageTooOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 120 * 1000 // older than 2 minutes ago + const messageTooOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 30 * 1000 // older than 30 seconds ago // Don't post new generated embeds for messages if the setting was disabled. const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 if (messageTooOld || !embedsEnabled) { @@ -150,8 +152,7 @@ async function editToChanges(message, guild, api) { // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! // (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.) // So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed. - unchangedEvents.push(...eventsToReplace.filter(ev => !eventCanBeEdited(ev))) // Move them from eventsToRedact to unchangedEvents. - eventsToReplace = eventsToReplace.filter(eventCanBeEdited) + unchangedEvents = unchangedEvents.concat(dUtils.filterTo(eventsToReplace, ev => eventCanBeEdited(ev))) // Move them from eventsToReplace to unchangedEvents. // Now, everything in eventsToReplace has the potential to have changed, but did it actually? // (Example: if a URL preview was generated or updated, the message text won't have changed.) diff --git a/src/discord/utils.js b/src/discord/utils.js index 441a55ce..a51b155b 100644 --- a/src/discord/utils.js +++ b/src/discord/utils.js @@ -153,6 +153,26 @@ function howOldUnbridgedMessage(oldTimestamp, newTimestamp) { return dateDisplay } +/** + * Modifies the input, removing items that don't pass the filter. Returns the items that didn't pass. + * @param {T[]} xs + * @param {(x: T, i?: number) => any} fn + * @template T + * @returns T[] + */ +function filterTo(xs, fn) { + /** @type {T[]} */ + const filtered = [] + for (let i = xs.length-1; i >= 0; i--) { + const x = xs[i] + if (!fn(x, i)) { + filtered.unshift(x) + xs.splice(i, 1) + } + } + return filtered +} + module.exports.getPermissions = getPermissions module.exports.hasPermission = hasPermission module.exports.hasSomePermissions = hasSomePermissions @@ -163,3 +183,4 @@ module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact module.exports.getPublicUrlForCdn = getPublicUrlForCdn module.exports.howOldUnbridgedMessage = howOldUnbridgedMessage +module.exports.filterTo = filterTo diff --git a/src/discord/utils.test.js b/src/discord/utils.test.js index 3a3e1772..88e51c92 100644 --- a/src/discord/utils.test.js +++ b/src/discord/utils.test.js @@ -192,3 +192,10 @@ test("how old: hours", t => { test("how old: days", t => { t.equal(utils.howOldUnbridgedMessage("2024-01-01", "2025-01-01"), "a 366-day-old unbridged message") }) + +test("filterTo: works", t => { + const fruit = ["apple", "banana", "apricot"] + const rest = utils.filterTo(fruit, f => f[0] === "b") + t.deepEqual(fruit, ["banana"]) + t.deepEqual(rest, ["apple", "apricot"]) +}) diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index 47b710bb..0af37e2c 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -53,26 +53,6 @@ function getAPI(event) { /** @type {LRUCache} nonce to guild id */ const validNonce = new LRUCache({max: 200}) -/** - * Modifies the input, removing items that don't pass the filter. Returns the items that didn't pass. - * @param {T[]} xs - * @param {(x: T, i?: number) => any} fn - * @template T - * @returns T[] - */ -function filterTo(xs, fn) { - /** @type {T[]} */ - const filtered = [] - for (let i = xs.length-1; i >= 0; i--) { - const x = xs[i] - if (!fn(x, i)) { - filtered.unshift(x) - xs.splice(i, 1) - } - } - return filtered -} - /** * @param {{type: number, parent_id?: string, position?: number}} channel * @param {Map} channels @@ -119,15 +99,15 @@ function getChannelRoomsLinks(guild, rooms, roles) { let linkedChannels = select("channel_room", ["channel_id", "room_id", "name", "nick"], {channel_id: channelIDs}).all() let linkedChannelsWithDetails = linkedChannels.map(c => ({channel: discord.channels.get(c.channel_id), ...c})) - let removedUncachedChannels = filterTo(linkedChannelsWithDetails, c => c.channel) + let removedUncachedChannels = dUtils.filterTo(linkedChannelsWithDetails, c => c.channel) let linkedChannelIDs = linkedChannelsWithDetails.map(c => c.channel_id) linkedChannelsWithDetails.sort((a, b) => getPosition(a.channel, discord.channels) - getPosition(b.channel, discord.channels)) let unlinkedChannelIDs = channelIDs.filter(c => !linkedChannelIDs.includes(c)) /** @type {DiscordTypes.APIGuildChannel[]} */ // @ts-ignore let unlinkedChannels = unlinkedChannelIDs.map(c => discord.channels.get(c)) - let removedWrongTypeChannels = filterTo(unlinkedChannels, c => c && [0, 5].includes(c.type)) - let removedPrivateChannels = filterTo(unlinkedChannels, c => { + let removedWrongTypeChannels = dUtils.filterTo(unlinkedChannels, c => c && [0, 5].includes(c.type)) + let removedPrivateChannels = dUtils.filterTo(unlinkedChannels, c => { const permissions = dUtils.getPermissions(guild.id, roles, guild.roles, botID, c["permission_overwrites"]) return dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) }) @@ -135,11 +115,11 @@ function getChannelRoomsLinks(guild, rooms, roles) { let linkedRoomIDs = linkedChannels.map(c => c.room_id) let unlinkedRooms = [...rooms] - let removedLinkedRooms = filterTo(unlinkedRooms, r => !linkedRoomIDs.includes(r.room_id)) - let removedWrongTypeRooms = filterTo(unlinkedRooms, r => !r.room_type) + let removedLinkedRooms = dUtils.filterTo(unlinkedRooms, r => !linkedRoomIDs.includes(r.room_id)) + let removedWrongTypeRooms = dUtils.filterTo(unlinkedRooms, r => !r.room_type) // https://discord.com/developers/docs/topics/threads#active-archived-threads // need to filter out linked archived threads from unlinkedRooms, will just do that by comparing against the name - let removedArchivedThreadRooms = filterTo(unlinkedRooms, r => r.name && !r.name.match(/^\[(🔒)?⛓️\]/)) + let removedArchivedThreadRooms = dUtils.filterTo(unlinkedRooms, r => r.name && !r.name.match(/^\[(🔒)?⛓️\]/)) return { linkedChannelsWithDetails, unlinkedChannels, unlinkedRooms, @@ -265,4 +245,3 @@ as.router.post("/api/invite", defineEventHandler(async event => { })) module.exports._getPosition = getPosition -module.exports._filterTo = filterTo diff --git a/src/web/routes/guild.test.js b/src/web/routes/guild.test.js index e0b26dff..aa17548e 100644 --- a/src/web/routes/guild.test.js +++ b/src/web/routes/guild.test.js @@ -4,7 +4,7 @@ const DiscordTypes = require("discord-api-types/v10") const tryToCatch = require("try-to-catch") const {router, test} = require("../../../test/web") const {MatrixServerError} = require("../../matrix/mreq") -const {_getPosition, _filterTo} = require("./guild") +const {_getPosition} = require("./guild") let nonce @@ -394,10 +394,3 @@ test("position sorting: sorts like discord does", t => { const sortedChannelIDs = [...channels.values()].sort((a, b) => _getPosition(a, channels) - _getPosition(b, channels)).map(c => c.id) t.deepEqual(sortedChannelIDs, ["first", "thread", "second", "voice", "category", "category-first", "category-second", "category-second-thread"]) }) - -test("filterTo: works", t => { - const fruit = ["apple", "banana", "apricot"] - const rest = _filterTo(fruit, f => f[0] === "b") - t.deepEqual(fruit, ["banana"]) - t.deepEqual(rest, ["apple", "apricot"]) -}) From 756e8e27ad0299a6bcdd94d693eac203b805265d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 21 Jan 2026 01:59:54 +1300 Subject: [PATCH 075/153] Make registration more consistent --- src/d2m/converters/user-to-mxid.test.js | 15 +++++++-------- test/test.js | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/d2m/converters/user-to-mxid.test.js b/src/d2m/converters/user-to-mxid.test.js index 2217a93e..387d4726 100644 --- a/src/d2m/converters/user-to-mxid.test.js +++ b/src/d2m/converters/user-to-mxid.test.js @@ -45,14 +45,6 @@ test("user2name: works on special user", t => { t.equal(userToSimName(data.user.clyde_ai), "clyde_ai") }) -test("user2name: includes ID if requested in config", t => { - const {reg} = require("../../matrix/read-registration") - reg.ooye.include_user_id_in_mxid = true - t.equal(userToSimName({username: "Harry Styles!", discriminator: "0001", id: "123456"}), "123456_harry_styles") - t.equal(userToSimName({username: "f***", discriminator: "0001", id: "123456"}), "123456_f") - reg.ooye.include_user_id_in_mxid = false -}) - test("webhook author: can generate sim names", t => { t.equal(webhookAuthorToSimName({ username: "Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆", @@ -60,3 +52,10 @@ test("webhook author: can generate sim names", t => { id: "123" }), "webhook_cadence_maid_of_creation_eye_of_clarity_empress_of_hope") }) + +test("user2name: includes ID if requested in config", t => { + const {reg} = require("../../matrix/read-registration") + reg.ooye.include_user_id_in_mxid = true + t.equal(userToSimName({username: "Harry Styles!", discriminator: "0001", id: "123456"}), "123456_harry_styles") + t.equal(userToSimName({username: "f***", discriminator: "0001", id: "123456"}), "123456_f") +}) diff --git a/test/test.js b/test/test.js index be7febf2..5ae9f67e 100644 --- a/test/test.js +++ b/test/test.js @@ -30,6 +30,7 @@ reg.ooye.bridge_origin = "https://bridge.example.org" reg.ooye.time_zone = "Pacific/Auckland" reg.ooye.max_file_size = 5000000 reg.ooye.web_password = "password123" +reg.ooye.include_user_id_in_mxid = false const sync = new HeatSync({watchFS: false}) From 5e4b99a5523327c5820a3fa57b9f89140b240a48 Mon Sep 17 00:00:00 2001 From: Rory& Date: Tue, 20 Jan 2026 11:21:12 +0100 Subject: [PATCH 076/153] Remove reply fallback for same-room replies (and update tests accordingly) --- src/d2m/converters/edit-to-changes.test.js | 7 +--- src/d2m/converters/message-to-event.js | 12 +++---- .../converters/message-to-event.pk.test.js | 18 ++-------- src/d2m/converters/message-to-event.test.js | 33 ++++--------------- 4 files changed, 15 insertions(+), 55 deletions(-) diff --git a/src/d2m/converters/edit-to-changes.test.js b/src/d2m/converters/edit-to-changes.test.js index b252175e..d6877020 100644 --- a/src/d2m/converters/edit-to-changes.test.js +++ b/src/d2m/converters/edit-to-changes.test.js @@ -181,12 +181,7 @@ test("edit2changes: edit of reply to skull webp attachment with content", async newContent: { $type: "m.room.message", msgtype: "m.text", - body: "> Extremity: Image\n\n* Edit", - format: "org.matrix.custom.html", - formatted_body: - '
    In reply to Extremity' - + '
    Image
    ' - + '* Edit', + body: "* Edit", "m.mentions": {}, "m.new_content": { msgtype: "m.text", diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 1c921231..11e82a5d 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -508,15 +508,13 @@ async function messageToEvent(message, guild, options = {}, di) { // Generate a reply pointing to the Matrix event we found const latestRoomID = select("channel_room", "room_id", {channel_id: repliedToEventRow.channel_id}).pluck().get() // native replies don't work across room upgrades, so make sure the old and new message are in the same room if (latestRoomID !== repliedToEventRow.room_id) repliedToEventInDifferentRoom = true - html = - (latestRoomID === repliedToEventRow.room_id ? "" : "") - + `
    In reply to ${repliedToUserHtml}` + html = repliedToEventInDifferentRoom ? + (`
    In reply to ${repliedToUserHtml}` + `
    ${repliedToHtml}
    ` - + (latestRoomID === repliedToEventRow.room_id ? "" : "") - + html - body = (`${repliedToDisplayName}: ` // scenario 1 part B for mentions + + html) : html + body = repliedToEventInDifferentRoom ? ((`${repliedToDisplayName}: ` // scenario 1 part B for mentions + repliedToBody).split("\n").map(line => "> " + line).join("\n") - + "\n\n" + body + + "\n\n" + body) : body } else { // repliedToUnknownEvent // This reply can't point to the Matrix event because it isn't bridged, we need to indicate this. assert(message.referenced_message) diff --git a/src/d2m/converters/message-to-event.pk.test.js b/src/d2m/converters/message-to-event.pk.test.js index ce83d544..1323280c 100644 --- a/src/d2m/converters/message-to-event.pk.test.js +++ b/src/d2m/converters/message-to-event.pk.test.js @@ -50,11 +50,7 @@ test("message2event: pk reply to matrix is converted to native matrix reply", as ] }, msgtype: "m.text", - body: "> cadence [they]: now for my next experiment:\n\nthis is a reply", - format: "org.matrix.custom.html", - formatted_body: '
    In reply to cadence [they]
    ' - + "now for my next experiment:
    " - + "this is a reply", + body: "this is a reply", "m.relates_to": { "m.in_reply_to": { event_id: "$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU" @@ -80,11 +76,7 @@ test("message2event: pk reply to discord is converted to native matrix reply", a $type: "m.room.message", msgtype: "m.text", "m.mentions": {}, - body: "> wing: some text\n\nthis is a reply", - format: "org.matrix.custom.html", - formatted_body: '
    In reply to wing
    ' - + "some text
    " - + "this is a reply", + body: "this is a reply", "m.relates_to": { "m.in_reply_to": { event_id: "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA" @@ -120,11 +112,7 @@ test("message2event: pk reply to matrix attachment is converted to native matrix "m.mentions": { user_ids: ["@ampflower:matrix.org"] }, - body: "> Ampflower 🌺: [Media]\n\nCat nod", - format: "org.matrix.custom.html", - formatted_body: '
    In reply to Ampflower 🌺
    ' - + "[Media]
    " - + "Cat nod", + body: "Cat nod", "m.relates_to": { "m.in_reply_to": { event_id: "$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y" diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index f7769d33..a527ad85 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -423,12 +423,7 @@ test("message2event: reply to skull webp attachment with content", async t => { }, "m.mentions": {}, msgtype: "m.text", - body: "> Extremity: Image\n\nReply", - format: "org.matrix.custom.html", - formatted_body: - '
    In reply to Extremity' - + '
    Image
    ' - + 'Reply' + body: "Reply" }, { $type: "m.room.message", "m.mentions": {}, @@ -472,12 +467,7 @@ test("message2event: simple reply to matrix user", async t => { ] }, msgtype: "m.text", - body: "> cadence: so can you reply to my webhook uwu\n\nReply", - format: "org.matrix.custom.html", - formatted_body: - '
    In reply to cadence' - + '
    so can you reply to my webhook uwu
    ' - + 'Reply' + body: "Reply" }]) }) @@ -539,12 +529,7 @@ test("message2event: reply to matrix user with mention", async t => { ] }, msgtype: "m.text", - body: "> okay 🤍 yay 🤍: @extremity: you owe me $30\n\nkys", - format: "org.matrix.custom.html", - formatted_body: - '
    In reply to okay 🤍 yay 🤍' - + '
    @extremity you owe me $30
    ' - + 'kys' + body: "kys" }]) }) @@ -656,9 +641,7 @@ test("message2event: simple reply in thread to a matrix user's reply", async t = user_ids: ["@cadence:cadence.moe"] }, msgtype: "m.text", - body: "> cadence [they]: What about them?\n\nWell, they don't seem to...", - format: "org.matrix.custom.html", - formatted_body: "
    In reply to cadence [they]
    What about them?
    Well, they don't seem to...", + body: "Well, they don't seem to..." }]) }) @@ -695,9 +678,7 @@ test("message2event: infinidoge's reply to ami's matrix smalltext reply to infin user_ids: ["@ami:the-apothecary.club"] }, msgtype: "m.text", - body: `> Ami (she/her): let me guess they got a lot of bug reports like "empty chest with no loot?"\n\nMost likely`, - format: "org.matrix.custom.html", - formatted_body: `
    In reply to Ami (she/her)
    let me guess they got a lot of bug reports like "empty chest with no loot?"
    Most likely`, + body: `Most likely` }]) }) @@ -734,9 +715,7 @@ test("message2event: infinidoge's reply to ami's matrix smalltext singleline rep user_ids: ["@ami:the-apothecary.club"] }, msgtype: "m.text", - body: `> Ami (she/her): let me guess they got a lot of bug reports like "empty chest with no loot?"\n\nMost likely`, - format: "org.matrix.custom.html", - formatted_body: `
    In reply to Ami (she/her)
    let me guess they got a lot of bug reports like "empty chest with no loot?"
    Most likely`, + body: `Most likely` }]) }) From b5596b2459664019fcadd113438efa40f4de5345 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 21 Jan 2026 13:50:16 +1300 Subject: [PATCH 077/153] Fetch referenced_message for reply fallback --- src/d2m/converters/message-to-event.js | 96 ++++++++++++++------------ 1 file changed, 53 insertions(+), 43 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 11e82a5d..4e521767 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -480,51 +480,61 @@ async function messageToEvent(message, guild, options = {}, di) { } // Fallback body/formatted_body for replies + // Generate a fallback if native replies are unsupported, which is in the following situations: + // 1. The replied-to event is in a different room to where the reply will be sent (i.e. a room upgrade occurred between) + // 2. The replied-to message has no corresponding Matrix event (repliedToUnknownEvent is true) // This branch is optional - do NOT change anything apart from the reply fallback, since it may not be run if ((repliedToEventRow || repliedToUnknownEvent) && options.includeReplyFallback !== false) { - let repliedToDisplayName - let repliedToUserHtml - if (repliedToEventRow?.source === 0 && repliedToEventSenderMxid) { - const match = repliedToEventSenderMxid.match(/^@([^:]*)/) - assert(match) - repliedToDisplayName = message.referenced_message?.author.username || match[1] || "a Matrix user" // grab the localpart as the display name, whatever - repliedToUserHtml = `${repliedToDisplayName}` - } else { - repliedToDisplayName = message.referenced_message?.author.global_name || message.referenced_message?.author.username || "a Discord user" - repliedToUserHtml = repliedToDisplayName - } - let repliedToContent = message.referenced_message?.content - if (repliedToContent?.match(/^(-# )?> (-# )?<:L1:/)) { - // If the Discord user is replying to a Matrix user's reply, the fallback is going to contain the emojis and stuff from the bridged rep of the Matrix user's reply quote. - // Need to remove that previous reply rep from this fallback body. The fallbody body should only contain the Matrix user's actual message. - // ┌──────A─────┐ A reply rep starting with >quote or -#smalltext >quote. Match until the end of the line. - // ┆ ┆┌─B─┐ There may be up to 2 reply rep lines in a row if it was created in the old format. Match all lines. - repliedToContent = repliedToContent.replace(/^((-# )?> .*\n){1,2}/, "") - } - if (repliedToContent == "") repliedToContent = "[Media]" - else if (!repliedToContent) repliedToContent = "[Replied-to message content wasn't provided by Discord]" - const {body: repliedToBody, html: repliedToHtml} = await transformContent(repliedToContent) - if (repliedToEventRow) { - // Generate a reply pointing to the Matrix event we found - const latestRoomID = select("channel_room", "room_id", {channel_id: repliedToEventRow.channel_id}).pluck().get() // native replies don't work across room upgrades, so make sure the old and new message are in the same room - if (latestRoomID !== repliedToEventRow.room_id) repliedToEventInDifferentRoom = true - html = repliedToEventInDifferentRoom ? - (`
    In reply to ${repliedToUserHtml}` - + `
    ${repliedToHtml}
    ` - + html) : html - body = repliedToEventInDifferentRoom ? ((`${repliedToDisplayName}: ` // scenario 1 part B for mentions - + repliedToBody).split("\n").map(line => "> " + line).join("\n") - + "\n\n" + body) : body - } else { // repliedToUnknownEvent - // This reply can't point to the Matrix event because it isn't bridged, we need to indicate this. - assert(message.referenced_message) - const dateDisplay = dUtils.howOldUnbridgedMessage(message.referenced_message.timestamp, message.timestamp) - html = `
    In reply to ${dateDisplay} from ${repliedToDisplayName}:` - + `
    ${repliedToHtml}
    ` - + html - body = (`In reply to ${dateDisplay}:\n${repliedToDisplayName}: ` - + repliedToBody).split("\n").map(line => "> " + line).join("\n") - + "\n\n" + body + const latestRoomID = repliedToEventRow ? select("channel_room", "room_id", {channel_id: repliedToEventRow.channel_id}).pluck().get() : null + if (latestRoomID !== repliedToEventRow?.room_id) repliedToEventInDifferentRoom = true + + // check that condition 1 or 2 is met + if (repliedToEventInDifferentRoom || repliedToUnknownEvent) { + let referenced = message.referenced_message + if (!referenced) { // backend couldn't be bothered to dereference the message, have to do it ourselves + referenced = await discord.snow.channel.getChannelMessage(message.message_reference.channel_id, message.message_reference.message_id) + } + + // Username + let repliedToDisplayName + let repliedToUserHtml + if (repliedToEventRow?.source === 0 && repliedToEventSenderMxid) { + const match = repliedToEventSenderMxid.match(/^@([^:]*)/) + assert(match) + repliedToDisplayName = referenced.author.username || match[1] || "a Matrix user" // grab the localpart as the display name, whatever + repliedToUserHtml = `${repliedToDisplayName}` + } else { + repliedToDisplayName = referenced.author.global_name || referenced.author.username || "a Discord user" + repliedToUserHtml = repliedToDisplayName + } + + // Content + let repliedToContent = referenced.content + if (repliedToContent?.match(/^(-# )?> (-# )?<:L1:/)) { + // If the Discord user is replying to a Matrix user's reply, the fallback is going to contain the emojis and stuff from the bridged rep of the Matrix user's reply quote. + // Need to remove that previous reply rep from this fallback body. The fallbody body should only contain the Matrix user's actual message. + // ┌──────A─────┐ A reply rep starting with >quote or -#smalltext >quote. Match until the end of the line. + // ┆ ┆┌─B─┐ There may be up to 2 reply rep lines in a row if it was created in the old format. Match all lines. + repliedToContent = repliedToContent.replace(/^((-# )?> .*\n){1,2}/, "") + } + if (repliedToContent == "") repliedToContent = "[Media]" + const {body: repliedToBody, html: repliedToHtml} = await transformContent(repliedToContent) + + // Now branch on condition 1 or 2 for a different kind of fallback + if (repliedToEventRow) { + html = `
    In reply to ${repliedToUserHtml}` + + `
    ${repliedToHtml}
    ` + + html + body = `${repliedToDisplayName}: ${repliedToBody}`.split("\n").map(line => "> " + line).join("\n") // scenario 1 part B for mentions + + "\n\n" + body + } else { // repliedToUnknownEvent + const dateDisplay = dUtils.howOldUnbridgedMessage(referenced.timestamp, message.timestamp) + html = `
    In reply to ${dateDisplay} from ${repliedToDisplayName}:` + + `
    ${repliedToHtml}
    ` + + html + body = `In reply to ${dateDisplay}:\n${repliedToDisplayName}: ${repliedToBody}`.split("\n").map(line => "> " + line).join("\n") + + "\n\n" + body + } } } From ddc7387fa0924e6da5029505b6daf84e22f9f063 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 21 Jan 2026 13:01:36 +1300 Subject: [PATCH 078/153] Do not make forwarded messages m.notice --- src/d2m/converters/message-to-event.js | 5 +- src/d2m/converters/message-to-event.test.js | 89 ++++++++++++++++++++- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 4e521767..78829bac 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -37,8 +37,8 @@ function getDiscordParseCallbacks(message, guild, useHTML, spoilers = []) { const username = message.mentions?.find(ment => ment.id === node.id)?.username || message.referenced_message?.mentions?.find(ment => ment.id === node.id)?.username || (interaction?.user.id === node.id ? interaction.user.username : null) - || (message.author.id === node.id ? message.author.username : null) - || node.id + || (message.author?.id === node.id ? message.author.username : null) + || "unknown-user" if (mxid && useHTML) { return `@${username}` } else { @@ -610,7 +610,6 @@ async function messageToEvent(message, guild, options = {}, di) { // Indent for (const event of forwardedEvents) { if (["m.text", "m.notice"].includes(event.msgtype)) { - event.msgtype = "m.notice" event.body = event.body.split("\n").map(l => "» " + l).join("\n") event.formatted_body = `
    ${event.formatted_body}
    ` } diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index a527ad85..fa51eaed 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -1090,7 +1090,7 @@ test("message2event: constructed forwarded message", async t => { formatted_body: `🔀 Forwarded from wonderland [jump to event]` + `
    What's cooking, good looking? :hipposcope:
    `, "m.mentions": {}, - msgtype: "m.notice", + msgtype: "m.text", }, { $type: "m.room.message", @@ -1149,7 +1149,7 @@ test("message2event: constructed forwarded text", async t => { formatted_body: `🔀 Forwarded from amanda-spam [jump to room]` + `
    What's cooking, good looking?
    `, "m.mentions": {}, - msgtype: "m.notice", + msgtype: "m.text", }, { $type: "m.room.message", @@ -1172,7 +1172,7 @@ test("message2event: don't scan forwarded messages for mentions", async t => { formatted_body: `🔀 Forwarded message` + `
    If some folks have spare bandwidth then helping out ArchiveTeam with archiving soon to be deleted research and government data might be worthwhile https://social.luca.run/@luca/113950834185678114
    `, "m.mentions": {}, - msgtype: "m.notice" + msgtype: "m.text" } ]) }) @@ -1429,3 +1429,86 @@ test("message2event: cross-room reply", async t => { } ]) }) + +test("message2event: forwarded message with unreferenced mention", async t => { + const events = await messageToEvent({ + type: 0, + content: "", + attachments: [], + embeds: [], + timestamp: "2026-01-20T14:14:21.281Z", + edited_timestamp: null, + flags: 16384, + components: [], + id: "1463174818823405651", + channel_id: "893634327722721290", + author: { + id: "100031256988766208", + username: "leo60228", + discriminator: "0", + avatar: "8a164f29946f23eb4f45cde71a75e5a6", + avatar_decoration_data: null, + public_flags: 768, + global_name: "leo vriska", + primary_guild: null, + collectibles: null, + display_name_styles: null + }, + bot: false, + pinned: false, + mentions: [], + mention_roles: [], + mention_everyone: false, + tts: false, + message_reference: { + type: 1, + channel_id: "937181373943382036", + message_id: "1032034158261846038", + guild_id: "936370934292549712" + }, + message_snapshots: [ + { + message: { + type: 0, + content: "<@77084495118868480>", + attachments: [ + { + id: "1463174815119704114", + filename: "2022-10-18_16-49-46.mp4", + size: 51238885, + url: "https://cdn.discordapp.com/attachments/893634327722721290/1463174815119704114/2022-10-18_16-49-46.mp4?ex=6970df3c&is=696f8dbc&hm=515d3cbcc8464bdada7f4c3d9ccc8174f671cb75391ce21a46a804fcb1e4befe&", + proxy_url: "https://media.discordapp.net/attachments/893634327722721290/1463174815119704114/2022-10-18_16-49-46.mp4?ex=6970df3c&is=696f8dbc&hm=515d3cbcc8464bdada7f4c3d9ccc8174f671cb75391ce21a46a804fcb1e4befe&", + width: 1920, + height: 1080, + content_type: "video/mp4", + content_scan_version: 3, + spoiler: false + } + ], + embeds: [], + timestamp: "2022-10-18T20:55:17.597Z", + edited_timestamp: null, + flags: 0, + components: [] + } + } + ] + }) + t.deepEqual(events, [ + { + $type: "m.room.message", + msgtype: "m.text", + body: "[🔀 Forwarded message]\n» @unknown-user:", + format: "org.matrix.custom.html", + formatted_body: `🔀 Forwarded message
    @unknown-user:
    `, + "m.mentions": {} + }, { + $type: "m.room.message", + msgtype: "m.text", + body: "» 🎞️ Uploaded file: https://bridge.example.org/download/discordcdn/893634327722721290/1463174815119704114/2022-10-18_16-49-46.mp4 (51 MB)", + format: "org.matrix.custom.html", + formatted_body: "
    🎞️ Uploaded file: 2022-10-18_16-49-46.mp4 (51 MB)
    ", + "m.mentions": {} + } + ]) +}) From 345b7d61359eb080d439ab0b04a2354c97e72421 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 21 Jan 2026 13:25:30 +1300 Subject: [PATCH 079/153] Merge attachments with message when possible --- src/d2m/converters/message-to-event.js | 36 ++++++--- src/d2m/converters/message-to-event.test.js | 81 +++++++++++++++------ test/ooye-test-data.sql | 3 +- 3 files changed, 87 insertions(+), 33 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 78829bac..449303a7 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -542,16 +542,9 @@ async function messageToEvent(message, guild, options = {}, di) { $type: "m.room.message", "m.mentions": mentions, msgtype, - body: body - } - - const isPlaintext = body === html - - if (!isPlaintext || options.alwaysReturnFormattedBody) { - Object.assign(newTextMessageEvent, { - format: "org.matrix.custom.html", - formatted_body: html - }) + body: body, + format: "org.matrix.custom.html", + formatted_body: html } events.push(newTextMessageEvent) @@ -695,7 +688,18 @@ async function messageToEvent(message, guild, options = {}, di) { // Then attachments if (message.attachments) { const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions))) - events.push(...attachmentEvents) + + // Try to merge attachment events with the previous event + // This means that if the attachments ended up as a text link, and especially if there were many of them, the events will be joined together. + let prev = events.at(-1) + for (const atch of attachmentEvents) { + if (atch.msgtype === "m.text" && prev?.body && prev?.formatted_body && ["m.text", "m.notice"].includes(prev?.msgtype)) { + prev.body = prev.body + "\n" + atch.body + prev.formatted_body = prev.formatted_body + "
    " + atch.formatted_body + } else { + events.push(atch) + } + } } // Then embeds @@ -829,6 +833,16 @@ async function messageToEvent(message, guild, options = {}, di) { }) } + // Strip formatted_body where equivalent to body + if (!options.alwaysReturnFormattedBody) { + for (const event of events) { + if (["m.text", "m.notice"].includes(event.msgtype) && event.body === event.formatted_body) { + delete event.format + delete event.formatted_body + } + } + } + return events } diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index fa51eaed..3c0c5d99 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -869,14 +869,62 @@ test("message2event: very large attachment is linked instead of being uploaded", $type: "m.room.message", "m.mentions": {}, msgtype: "m.text", - body: "hey" - }, { + body: "hey\n📄 Uploaded file: https://bridge.example.org/download/discordcdn/123/456/789.mega (100 MB)", + format: "org.matrix.custom.html", + formatted_body: 'hey
    📄 Uploaded file: hey.jpg (100 MB)' + }]) +}) + +test("message2event: multiple attachments are combined into the same event where possible", async t => { + const events = await messageToEvent({ + content: "hey", + attachments: [{ + filename: "hey.jpg", + url: "https://cdn.discordapp.com/attachments/123/456/789.mega", + content_type: "application/i-made-it-up", + size: 100e6 + }, { + filename: "SPOILER_secret.jpg", + url: "https://cdn.discordapp.com/attachments/123/456/SPOILER_secret.jpg", + content_type: "image/jpeg", + size: 38291 + }, { + filename: "my enemies.txt", + url: "https://cdn.discordapp.com/attachments/123/456/my_enemies.txt", + content_type: "text/plain", + size: 8911 + }, { + filename: "hey.jpg", + url: "https://cdn.discordapp.com/attachments/123/456/789.mega", + content_type: "application/i-made-it-up", + size: 100e6 + }] + }) + t.deepEqual(events, [{ $type: "m.room.message", "m.mentions": {}, msgtype: "m.text", - body: "📄 Uploaded file: https://bridge.example.org/download/discordcdn/123/456/789.mega (100 MB)", + body: "hey" + + "\n📄 Uploaded file: https://bridge.example.org/download/discordcdn/123/456/789.mega (100 MB)" + + "\n📸 Uploaded SPOILER file: https://bridge.example.org/download/discordcdn/123/456/SPOILER_secret.jpg (38 KB)" + + "\n📄 Uploaded file: https://bridge.example.org/download/discordcdn/123/456/789.mega (100 MB)", format: "org.matrix.custom.html", - formatted_body: '📄 Uploaded file: hey.jpg (100 MB)' + formatted_body: "hey" + + `
    📄 Uploaded file: hey.jpg (100 MB)` + + `
    📸 Uploaded SPOILER file: https://bridge.example.org/download/discordcdn/123/456/SPOILER_secret.jpg (38 KB)
    ` + + `
    📄 Uploaded file: hey.jpg (100 MB)` + }, { + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.file", + body: "my enemies.txt", + filename: "my enemies.txt", + external_url: "https://bridge.example.org/download/discordcdn/123/456/my_enemies.txt", + url: "mxc://cadence.moe/y89EOTRp2lbeOkgdsEleGOge", + info: { + mimetype: "text/plain", + size: 8911 + } }]) }) @@ -1494,21 +1542,12 @@ test("message2event: forwarded message with unreferenced mention", async t => { } ] }) - t.deepEqual(events, [ - { - $type: "m.room.message", - msgtype: "m.text", - body: "[🔀 Forwarded message]\n» @unknown-user:", - format: "org.matrix.custom.html", - formatted_body: `🔀 Forwarded message
    @unknown-user:
    `, - "m.mentions": {} - }, { - $type: "m.room.message", - msgtype: "m.text", - body: "» 🎞️ Uploaded file: https://bridge.example.org/download/discordcdn/893634327722721290/1463174815119704114/2022-10-18_16-49-46.mp4 (51 MB)", - format: "org.matrix.custom.html", - formatted_body: "
    🎞️ Uploaded file: 2022-10-18_16-49-46.mp4 (51 MB)
    ", - "m.mentions": {} - } - ]) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "[🔀 Forwarded message]\n» @unknown-user:\n» 🎞️ Uploaded file: https://bridge.example.org/download/discordcdn/893634327722721290/1463174815119704114/2022-10-18_16-49-46.mp4 (51 MB)", + format: "org.matrix.custom.html", + formatted_body: "🔀 Forwarded message
    @unknown-user:
    🎞️ Uploaded file: 2022-10-18_16-49-46.mp4 (51 MB)
    ", + "m.mentions": {} + }]) }) diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index ef3dc5f7..04e6b9b1 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -151,7 +151,8 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4', 'mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU'), ('https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg', 'mxc://cadence.moe/MRRPDggXQMYkrUjTpxQbmcxB'), ('https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml', 'mxc://cadence.moe/HnQIYQmmlIKwOQsbFsIGpzPP'), -('https://cdn.discordapp.com/attachments/112760669178241024/1296237494987133070/100km.gif', 'mxc://cadence.moe/qDAotmebTfEIfsAIVCEZptLh'); +('https://cdn.discordapp.com/attachments/112760669178241024/1296237494987133070/100km.gif', 'mxc://cadence.moe/qDAotmebTfEIfsAIVCEZptLh'), +('https://cdn.discordapp.com/attachments/123/456/my_enemies.txt', 'mxc://cadence.moe/y89EOTRp2lbeOkgdsEleGOge'); INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), From 90fcbd0ddc407434cf448ada14cc4d53d5d51b68 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 21 Jan 2026 14:33:24 +1300 Subject: [PATCH 080/153] Update Discord libraries --- package-lock.json | 64 ++++++++++--------------- package.json | 5 +- src/d2m/actions/update-pins.js | 2 +- src/d2m/converters/pins-to-list.js | 9 ++-- src/d2m/converters/pins-to-list.test.js | 5 +- src/matrix/kstate.js | 16 +++++-- test/data.js | 14 +++--- 7 files changed, 59 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index bf857a5a..432c3f18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "ansi-colors": "^4.1.3", "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.14.0", + "cloudstorm": "^0.15.2", "discord-api-types": "^0.38.36", "domino": "^2.1.6", "enquirer": "^2.4.1", @@ -35,7 +35,7 @@ "lru-cache": "^11.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", - "snowtransfer": "^0.14.2", + "snowtransfer": "^0.17.0", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "uqr": "^0.1.2", @@ -1552,30 +1552,18 @@ } }, "node_modules/cloudstorm": { - "version": "0.14.1", - "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.14.1.tgz", - "integrity": "sha512-x95WCKg818E1rE1Ru45NPD3RoIq0pg3WxwvF0GE7Eq07pAeLcjSRqM1lUmbmfjdOqZrWdSRYA1NETVZ8QhVrIA==", + "version": "0.15.2", + "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.15.2.tgz", + "integrity": "sha512-5y7E0uI39R3d7c+AWksqAQAlZlpx+qNjxjQfNIem2hh68s6QRmOFHTKu34I7pBE6JonpZf8AmoMYArY/4lLVmg==", "license": "MIT", "dependencies": { - "discord-api-types": "^0.38.21", - "snowtransfer": "^0.15.0" + "discord-api-types": "^0.38.37", + "snowtransfer": "^0.17.0" }, "engines": { "node": ">=22.0.0" } }, - "node_modules/cloudstorm/node_modules/snowtransfer": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.15.0.tgz", - "integrity": "sha512-kEDGKtFiH5nSkHsDZonEUuDx99lUasJoZ7AGrgvE8HzVG59vjvqc//C+pjWj4DuJqTj4Q+Z1L/M/MYNim8F2VA==", - "license": "MIT", - "dependencies": { - "discord-api-types": "^0.38.21" - }, - "engines": { - "node": ">=16.15.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1711,9 +1699,9 @@ } }, "node_modules/discord-api-types": { - "version": "0.38.36", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.36.tgz", - "integrity": "sha512-qrbUbjjwtyeBg5HsAlm1C859epfOyiLjPqAOzkdWlCNsZCWJrertnETF/NwM8H+waMFU58xGSc5eXUfXah+WTQ==", + "version": "0.38.37", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.37.tgz", + "integrity": "sha512-Cv47jzY1jkGkh5sv0bfHYqGgKOWO1peOrGMkDFM4UmaGMOTgOW8QSexhvixa9sVOiz8MnVOBryWYyw/CEVhj7w==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" @@ -1971,9 +1959,9 @@ } }, "node_modules/h3": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.4.tgz", - "integrity": "sha512-z5cFQWDffyOe4vQ9xIqNfCZdV4p//vy6fBnr8Q1AWnVZ0teurKMG66rLj++TKwKPUP3u7iMUvrvKaEUiQw2QWQ==", + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.5.tgz", + "integrity": "sha512-xEyq3rSl+dhGX2Lm0+eFQIAzlDN6Fs0EcC4f7BNUmzaRX/PTzeuM+Tr2lHB8FoXggsQIeXLj8EDVgs5ywxyxmg==", "license": "MIT", "dependencies": { "cookie-es": "^1.2.2", @@ -1981,9 +1969,9 @@ "defu": "^6.1.4", "destr": "^2.0.5", "iron-webcrypto": "^1.2.1", - "node-mock-http": "^1.0.2", + "node-mock-http": "^1.0.4", "radix3": "^1.1.2", - "ufo": "^1.6.1", + "ufo": "^1.6.3", "uncrypto": "^0.1.3" } }, @@ -2321,9 +2309,9 @@ } }, "node_modules/node-mock-http": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/node-mock-http/-/node-mock-http-1.0.2.tgz", - "integrity": "sha512-zWaamgDUdo9SSLw47we78+zYw/bDr5gH8pH7oRRs8V3KmBtu8GLgGIbV2p/gRPd3LWpEOpjQj7X1FOU3VFMJ8g==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/node-mock-http/-/node-mock-http-1.0.4.tgz", + "integrity": "sha512-8DY+kFsDkNXy1sJglUfuODx1/opAGJGyrTuFqEoN90oRc2Vk0ZbD4K2qmKXBBEhZQzdKHIVfEJpDU8Ak2NJEvQ==", "license": "MIT" }, "node_modules/object-assign": { @@ -2821,15 +2809,15 @@ } }, "node_modules/snowtransfer": { - "version": "0.14.2", - "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.14.2.tgz", - "integrity": "sha512-Fi8OdRmaIgeCj58oVej+tQAoY2I+Xp/6PAYV8X93jE/2E6Anc87SbTbDV6WZXCnuzTQz3gty8JOGz02qI7Qs9A==", + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.17.0.tgz", + "integrity": "sha512-H6Avpsco+HlVIkN+MbX34Q7+9g9Wci0wZQwGsvfw20VqEb7jnnk73iUcWytNMYtKZ72Ud58n6cFnQ3apTEamxw==", "license": "MIT", "dependencies": { - "discord-api-types": "^0.38.8" + "discord-api-types": "^0.38.37" }, "engines": { - "node": ">=16.15.0" + "node": ">=22.0.0" } }, "node_modules/source-map": { @@ -3244,9 +3232,9 @@ } }, "node_modules/ufo": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.1.tgz", - "integrity": "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==", + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.3.tgz", + "integrity": "sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==", "license": "MIT" }, "node_modules/uncrypto": { diff --git a/package.json b/package.json index 64e6f77c..4d0c43aa 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "ansi-colors": "^4.1.3", "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.14.0", + "cloudstorm": "^0.15.2", "discord-api-types": "^0.38.36", "domino": "^2.1.6", "enquirer": "^2.4.1", @@ -44,7 +44,7 @@ "lru-cache": "^11.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", - "snowtransfer": "^0.14.2", + "snowtransfer": "^0.17.0", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "uqr": "^0.1.2", @@ -64,7 +64,6 @@ "scripts": { "start": "node --enable-source-maps start.js", "setup": "node --enable-source-maps scripts/setup.js", - "build": "mkdir -p dist/out-of-your-element && cp -R src dist/out-of-your-element && cp -R docs dist/out-of-your-element && npx tsdown", "addbot": "node addbot.js", "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js | tap-dot", "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot", diff --git a/src/d2m/actions/update-pins.js b/src/d2m/actions/update-pins.js index 15febaad..56c9642f 100644 --- a/src/d2m/actions/update-pins.js +++ b/src/d2m/actions/update-pins.js @@ -34,7 +34,7 @@ async function updatePins(channelID, roomID, convertedTimestamp) { throw e } - const kstate = await ks.roomToKState(roomID) + const kstate = await ks.roomToKState(roomID, [["m.room.pinned_events", ""]]) const pinned = pinsToList.pinsToList(discordPins, kstate) const diff = ks.diffKState(kstate, {"m.room.pinned_events/": {pinned}}) diff --git a/src/d2m/converters/pins-to-list.js b/src/d2m/converters/pins-to-list.js index 3e890ea3..5a33c7cb 100644 --- a/src/d2m/converters/pins-to-list.js +++ b/src/d2m/converters/pins-to-list.js @@ -3,10 +3,11 @@ const {select} = require("../../passthrough") /** - * @param {import("discord-api-types/v10").RESTGetAPIChannelPinsResult} pins + * @param {import("discord-api-types/v10").RESTGetAPIChannelMessagesPinsResult} pins * @param {{"m.room.pinned_events/"?: {pinned?: string[]}}} kstate */ function pinsToList(pins, kstate) { + /** Most recent last. */ let alreadyPinned = kstate["m.room.pinned_events/"]?.pinned || [] // If any of the already pinned messages are bridged messages then remove them from the already pinned list. @@ -15,13 +16,13 @@ function pinsToList(pins, kstate) { // * Matrix-only unbridged messages that are pinned will remain pinned. alreadyPinned = alreadyPinned.filter(event_id => { const messageID = select("event_message", "message_id", {event_id}).pluck().get() - return !messageID || pins.find(m => m.id === messageID) // if it is bridged then remove it from the filter + return !messageID || pins.items.find(m => m.message.id === messageID) // if it is bridged then remove it from the filter }) /** @type {string[]} */ const result = [] - for (const message of pins) { - const eventID = select("event_message", "event_id", {message_id: message.id, part: 0}).pluck().get() + for (const pin of pins.items) { + const eventID = select("event_message", "event_id", {message_id: pin.message.id, part: 0}).pluck().get() if (eventID && !alreadyPinned.includes(eventID)) result.push(eventID) } result.reverse() diff --git a/src/d2m/converters/pins-to-list.test.js b/src/d2m/converters/pins-to-list.test.js index d0657cba..571735e7 100644 --- a/src/d2m/converters/pins-to-list.test.js +++ b/src/d2m/converters/pins-to-list.test.js @@ -1,6 +1,7 @@ const {test} = require("supertape") const data = require("../../../test/data") const {pinsToList} = require("./pins-to-list") +const mixin = require("@cloudrac3r/mixin-deep") test("pins2list: converts known IDs, ignores unknown IDs", t => { const result = pinsToList(data.pins.faked, {}) @@ -46,7 +47,9 @@ test("pins2list: already pinned unknown items are not moved", t => { }) test("pins2list: bridged messages can be unpinned", t => { - const result = pinsToList(data.pins.faked.slice(0, -2), { + const shortPins = mixin({}, data.pins.faked) + shortPins.items = shortPins.items.slice(0, -2) + const result = pinsToList(shortPins, { "m.room.pinned_events/": { pinned: [ "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA", diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index 37eed393..c901ce11 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -140,10 +140,20 @@ function diffKState(actual, target) { /** * Async because it gets all room state from the homeserver. * @param {string} roomID + * @param {[type: string, key: string][]} [limitToEvents] */ -async function roomToKState(roomID) { - const root = await api.getAllState(roomID) - return stateToKState(root) +async function roomToKState(roomID, limitToEvents) { + if (!limitToEvents) { + const root = await api.getAllState(roomID) + return stateToKState(root) + } else { + const root = [] + await Promise.all(limitToEvents.map(async ([type, key]) => { + const outer = await api.getStateEventOuter(roomID, type, key) + root.push(outer) + })) + return stateToKState(root) + } } /** diff --git a/test/data.js b/test/data.js index e80b4367..0942a87d 100644 --- a/test/data.js +++ b/test/data.js @@ -1256,12 +1256,14 @@ module.exports = { } }, pins: { - faked: [ - {id: "1126786462646550579"}, - {id: "1141501302736695316"}, - {id: "1106366167788044450"}, - {id: "1115688611186193400"} - ] + faked: { + items: [ + {message: {id: "1126786462646550579"}}, + {message: {id: "1141501302736695316"}}, + {message: {id: "1106366167788044450"}}, + {message: {id: "1115688611186193400"}} + ] + } }, message: { // Display order is text content, attachments, then stickers From 0dd8958a2fcad7c0fc0bc9c7c4ab79e178d7d5d6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 21 Jan 2026 14:53:06 +1300 Subject: [PATCH 081/153] Update dependencies --- package-lock.json | 342 ++++++++++++++++++---------------------------- package.json | 2 +- 2 files changed, 136 insertions(+), 208 deletions(-) diff --git a/package-lock.json b/package-lock.json index 432c3f18..eeccc7c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,7 +47,7 @@ "@types/node": "^22.17.1", "c8": "^10.1.2", "cross-env": "^7.0.3", - "supertape": "^11.3.0" + "supertape": "^12.0.12" }, "engines": { "node": ">=20" @@ -142,103 +142,16 @@ "license": "MIT" }, "node_modules/@cloudcmd/stub": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@cloudcmd/stub/-/stub-4.0.1.tgz", - "integrity": "sha512-7x7tVxJZOdQowHv/VKwHLo9aoNNoVRc6PdKYqyKcDHX+xrF78jSXnqEWrOplnD/gF+tCnyFafu1Is+lFfWCILw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@cloudcmd/stub/-/stub-5.0.0.tgz", + "integrity": "sha512-jLC05CmcvEKDFXWf95UZGgqyJePhP3kh6/5ZXm7BAB42hv72RIx9LsYMhqGXlPtXjShV5KioOHri6QGnWMzHwQ==", "dev": true, + "license": "MIT", "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "^27.0.6", - "strip-ansi": "^6.0.0" + "jest-diff": "^30.2.0" }, "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudcmd/stub/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/@cloudcmd/stub/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@cloudcmd/stub/node_modules/diff-sequences": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-27.5.1.tgz", - "integrity": "sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ==", - "dev": true, - "engines": { - "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" - } - }, - "node_modules/@cloudcmd/stub/node_modules/jest-diff": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-27.5.1.tgz", - "integrity": "sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^27.5.1", - "jest-get-type": "^27.5.1", - "pretty-format": "^27.5.1" - }, - "engines": { - "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" - } - }, - "node_modules/@cloudcmd/stub/node_modules/jest-get-type": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-27.5.1.tgz", - "integrity": "sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw==", - "dev": true, - "engines": { - "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" - } - }, - "node_modules/@cloudcmd/stub/node_modules/pretty-format": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", - "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", - "dev": true, - "dependencies": { - "ansi-regex": "^5.0.1", - "ansi-styles": "^5.0.0", - "react-is": "^17.0.1" - }, - "engines": { - "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" - } - }, - "node_modules/@cloudcmd/stub/node_modules/react-is": { - "version": "17.0.2", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", - "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", - "dev": true - }, - "node_modules/@cloudcmd/stub/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" + "node": ">=22" } }, "node_modules/@cloudrac3r/discord-markdown": { @@ -968,9 +881,9 @@ } }, "node_modules/@jest/get-type": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.0.1.tgz", - "integrity": "sha512-AyYdemXCptSRFirI5EPazNxyPwAL0jXt3zceFjaj8NFiKP9pOi0bfXonf6qkf82z2t3QWPeLCWWw4stPBzctLw==", + "version": "30.1.0", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", + "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", "dev": true, "license": "MIT", "engines": { @@ -1027,17 +940,17 @@ } }, "node_modules/@putout/cli-keypress": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@putout/cli-keypress/-/cli-keypress-3.0.0.tgz", - "integrity": "sha512-RwODGTbcWNaulEPvVPdxH/vnddf5dE627G3s8gyou3kexa6zQerQHvbKFX0wywNdA3HD2O/9STPv/r5mjXFUgw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@putout/cli-keypress/-/cli-keypress-4.0.0.tgz", + "integrity": "sha512-76zmDjUycBt/CHkOZADP2KMdXWud3n8c1Wb4By/LWpbpykM8G9+pC7UeWAMo9CFDp/s3OXYee2UVtACf1+oZsg==", "dev": true, "license": "MIT", "dependencies": { "ci-info": "^4.0.0", - "fullstore": "^3.0.0" + "fullstore": "^4.0.0" }, "engines": { - "node": ">=20" + "node": ">=22" } }, "node_modules/@putout/cli-validate-args": { @@ -1054,9 +967,9 @@ } }, "node_modules/@sinclair/typebox": { - "version": "0.34.38", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.38.tgz", - "integrity": "sha512-HpkxMmc2XmZKhvaKIZZThlHmx1L0I/V1hWK1NubtlFnr6ZqdiOpV72TKudZUNQjZNsyDBay72qFEhEvb+bcwcA==", + "version": "0.34.47", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.47.tgz", + "integrity": "sha512-ZGIBQ+XDvO5JQku9wmwtabcVTHJsgSWAHYtVuM9pBNNR5E88v6Jcj/llpmsjivig5X8A8HHOb4/mbEKPS5EvAw==", "dev": true, "license": "MIT" }, @@ -1070,70 +983,72 @@ } }, "node_modules/@stackoverflow/stacks-icons": { - "version": "6.7.2", - "resolved": "https://registry.npmjs.org/@stackoverflow/stacks-icons/-/stacks-icons-6.7.2.tgz", - "integrity": "sha512-zEJDPDt7eYyAOMSnJFEPKkRoKydBWsg8LfEAX3TaF0UHI7N6vrVuOW6YeDIR2/uo0NahI9rf+Avg4+BADJmRhw==", + "version": "6.9.0", + "resolved": "https://registry.npmjs.org/@stackoverflow/stacks-icons/-/stacks-icons-6.9.0.tgz", + "integrity": "sha512-SFlcnSrH0b0/SsDBhCypYANyUwJs8hyuZzpo53l4iUUm30FGRbBWNfcaWwBW/skRjpHvlFYaUXwt66ZI6zT+lg==", "license": "MIT" }, "node_modules/@supertape/engine-loader": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@supertape/engine-loader/-/engine-loader-2.0.0.tgz", - "integrity": "sha512-1G2MmfZnSxx546omLPAVNgvG/iqOQZGiXHnjJ2JXKvuf2lpPdDRnNm5eLl81lvEG473zE9neX979TzeFcr3Dxw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@supertape/engine-loader/-/engine-loader-4.0.0.tgz", + "integrity": "sha512-2HFza8zaCGIC3Inaf3TEkWn3wvCkg+JPRWuSGrX+LM+j5OUptq6XtHnPeH037iEITTIDMie7OKlvfhZcFONGcw==", "dev": true, + "license": "MIT", "dependencies": { - "try-catch": "^3.0.0" + "try-catch": "^4.0.2" }, "engines": { - "node": ">=16" + "node": ">=22" } }, "node_modules/@supertape/formatter-fail": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@supertape/formatter-fail/-/formatter-fail-4.0.0.tgz", - "integrity": "sha512-+isArOXmGkIqH14PQoq2WhJmSwO8rzpQnhurVMuBmC+kYB96R95kRdjo/KO9d9yP1KoSjum0kX94s0SwqlZ8yA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@supertape/formatter-fail/-/formatter-fail-5.0.0.tgz", + "integrity": "sha512-nwE9c07hSFwoIf2Mex9PgWSe0f7PXbCbaPqL2oK3VWewNNRdGYBJv2VkPj0oAldvV86gTgQgS1b1k84XfthUCA==", "dev": true, "license": "MIT", "dependencies": { "@supertape/formatter-tap": "^4.0.0", - "fullstore": "^3.0.0" + "fullstore": "^4.0.0" }, "engines": { - "node": ">=20" + "node": ">=22" } }, "node_modules/@supertape/formatter-json-lines": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@supertape/formatter-json-lines/-/formatter-json-lines-2.0.1.tgz", - "integrity": "sha512-9LWOCu4yOF9orf4QJseS8lP3hXkYn24qn57VqYt/3r2aRJv4vWTPfaL1ot5JRHCZs0qXrV1sqPmN6E05rRLDYA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@supertape/formatter-json-lines/-/formatter-json-lines-3.0.0.tgz", + "integrity": "sha512-xk/Tl/J4rKVUroYyNCJEqmw78+xxBfToGi49G0oRPbfWvQgzvFzvqb97jyuet9rXEe5hJ6cztE/l/oMC1n4eig==", "dev": true, + "license": "MIT", "dependencies": { - "fullstore": "^3.0.0" + "fullstore": "^4.0.0" }, "engines": { - "node": ">=16" + "node": ">=22" } }, "node_modules/@supertape/formatter-progress-bar": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@supertape/formatter-progress-bar/-/formatter-progress-bar-7.0.0.tgz", - "integrity": "sha512-JDCT86hFJkoaqE/KS8BQsRaYiy3ipMpf0j+o+vwQMcFYm0mgG35JwbotBMUQM7LFifh68bTqU4xuewy7kUS1EA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@supertape/formatter-progress-bar/-/formatter-progress-bar-8.0.0.tgz", + "integrity": "sha512-ZGhKcQgMY4aUqfaMENoUUJ48xkLshlN2icJWeAV4nyRIu1RWU36qpVqcV95SEm+XKScYn/5EZtg9rhleaM9AEg==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^5.3.0", "ci-info": "^4.0.0", "cli-progress": "^3.8.2", - "fullstore": "^3.0.0", + "fullstore": "^4.0.0", "once": "^1.4.0" }, "engines": { - "node": ">=20" + "node": ">=22" } }, "node_modules/@supertape/formatter-progress-bar/node_modules/chalk": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.5.0.tgz", - "integrity": "sha512-1tm8DTaJhPBG3bIkVeZt1iZM9GfSX2lzOeDVZH9R9ffRHpmHvxZ/QhgQH/aDTkswQVt+YHdXAdS/In/30OjCbg==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", "dev": true, "license": "MIT", "engines": { @@ -1164,27 +1079,27 @@ } }, "node_modules/@supertape/formatter-time": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@supertape/formatter-time/-/formatter-time-2.0.0.tgz", - "integrity": "sha512-5UPvVHwpg5ZJmz0nII2f5rBFqNdMxHQnBybetmhgkSDIZHb+3NTPz/VrDggZERWOGxmIf4NKebaA+BWHTBQMeA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@supertape/formatter-time/-/formatter-time-3.0.0.tgz", + "integrity": "sha512-+A0uSQPdVSYCEwHSgdnWmn9tq4C+Dg9rj0ky05uHvrHhfPktA/XKlBocU8qpFMN+HllquWuKhZXeuwd18raQBw==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^5.3.0", "ci-info": "^4.0.0", "cli-progress": "^3.8.2", - "fullstore": "^3.0.0", + "fullstore": "^4.0.0", "once": "^1.4.0", "timer-node": "^5.0.7" }, "engines": { - "node": ">=20" + "node": ">=22" } }, "node_modules/@supertape/formatter-time/node_modules/chalk": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.5.0.tgz", - "integrity": "sha512-1tm8DTaJhPBG3bIkVeZt1iZM9GfSX2lzOeDVZH9R9ffRHpmHvxZ/QhgQH/aDTkswQVt+YHdXAdS/In/30OjCbg==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", "dev": true, "license": "MIT", "engines": { @@ -1195,15 +1110,16 @@ } }, "node_modules/@supertape/operator-stub": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@supertape/operator-stub/-/operator-stub-3.1.0.tgz", - "integrity": "sha512-jzC56u1k+3DLRo854+J6v/DP/4SjRV2mAqfR6qzsyaAocC9OFe7NHYQQMmlJ4cUJwgFjUh7AVnjFfC0Z0XuH+g==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@supertape/operator-stub/-/operator-stub-4.0.0.tgz", + "integrity": "sha512-t+LAKOA92m1pidzaXYzRHMAffYqqk19QOkMEbarP57/Sav90x9Q3ndvH6kRwa3HQhU2N7SuZrc21zh7vSwIOKA==", "dev": true, + "license": "MIT", "dependencies": { - "@cloudcmd/stub": "^4.0.0" + "@cloudcmd/stub": "^5.0.0" }, "engines": { - "node": ">=16" + "node": ">=22" } }, "node_modules/@tokenizer/token": { @@ -1218,9 +1134,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "22.19.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.1.tgz", - "integrity": "sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==", + "version": "22.19.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", + "integrity": "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==", "dev": true, "license": "MIT", "dependencies": { @@ -1256,10 +1172,11 @@ } }, "node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", "dev": true, + "license": "MIT", "engines": { "node": ">=12" }, @@ -1308,9 +1225,9 @@ } }, "node_modules/backtracker": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/backtracker/-/backtracker-4.0.0.tgz", - "integrity": "sha512-XG2ldN+WDRq9niJMnoZDjLLUnhDOQGhFZc6qZQotN59xj8oOa4KXSCu6YyZQawPqi6gG3HilGFt91zT6Hbdh1w==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/backtracker/-/backtracker-5.0.0.tgz", + "integrity": "sha512-2rY1s1iMlF1FVb4jpIMxTeGE+KRppuVvPyU61q7gvap1MWVahToUI8WUqy+v3L37iip5a4mJOTRBZxNRbTv4bg==", "license": "MIT" }, "node_modules/balanced-match": { @@ -1339,9 +1256,9 @@ ] }, "node_modules/better-sqlite3": { - "version": "12.4.1", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.4.1.tgz", - "integrity": "sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ==", + "version": "12.6.2", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.6.2.tgz", + "integrity": "sha512-8VYKM3MjCa9WcaSAI3hzwhmyHVlH8tiGFwf0RlTsZPWJ1I5MkzjiudCo4KC4DxOaL/53A5B1sI/IbldNFDbsKA==", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -1349,7 +1266,7 @@ "prebuild-install": "^7.1.1" }, "engines": { - "node": "20.x || 22.x || 23.x || 24.x" + "node": "20.x || 22.x || 23.x || 24.x || 25.x" } }, "node_modules/bindings": { @@ -1488,9 +1405,9 @@ } }, "node_modules/ci-info": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.3.0.tgz", - "integrity": "sha512-l+2bNRMiQgcfILUi33labAZYIWlH1kWDp+ecNo5iisRKrbm0xcRyCww71/YU0Fkw0mAFpz9bJayXPjey6vkmaQ==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.3.1.tgz", + "integrity": "sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA==", "dev": true, "funding": [ { @@ -1872,12 +1789,13 @@ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fullstore": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/fullstore/-/fullstore-3.0.0.tgz", - "integrity": "sha512-EEIdG+HWpyygWRwSLIZy+x4u0xtghjHNfhQb0mI5825Mmjq6oFESFUY0hoZigEgd3KH8GX+ZOCK9wgmOiS7VBQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fullstore/-/fullstore-4.0.0.tgz", + "integrity": "sha512-Y9hN79Q1CFU8akjGnTZoBnTzlA/o8wmtBijJOI8dKCmdC7GLX7OekpLxmbaeRetTOi4OdFGjfsg4c5dxP3jgPw==", "dev": true, + "license": "MIT", "engines": { - "node": ">=4" + "node": ">=22" } }, "node_modules/function-bind": { @@ -1997,12 +1915,12 @@ } }, "node_modules/heatsync": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/heatsync/-/heatsync-2.8.2.tgz", - "integrity": "sha512-zO5ivWP1NYoYmngdqVxzeQGX2Q68rfLkXKbO8Dhcguj5eS2eBDVpcWPh3+KCQagM7xYP5QVzvrUryWDu4mt6Eg==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/heatsync/-/heatsync-2.8.3.tgz", + "integrity": "sha512-9pVRC3BZD1NZ0EYnU5akjoO10+s/aJc04QqUxgtBqAYUeberV8st0ctWH7selEnyU8OEAUKZhBCFxmH7MvCQQQ==", "license": "MIT", "dependencies": { - "backtracker": "^4.0.0" + "backtracker": "^5.0.0" }, "engines": { "node": ">=14.6.0" @@ -2020,9 +1938,9 @@ "dev": true }, "node_modules/htmx.org": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/htmx.org/-/htmx.org-2.0.6.tgz", - "integrity": "sha512-7ythjYneGSk3yCHgtCnQeaoF+D+o7U2LF37WU3O0JYv3gTZSicdEFiI/Ai/NJyC5ZpYJWMpUb11OC5Lr6AfAqA==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/htmx.org/-/htmx.org-2.0.8.tgz", + "integrity": "sha512-fm297iru0iWsNJlBrjvtN7V9zjaxd+69Oqjh4F/Vq9Wwi2kFisLcrLCiv5oBX0KLfOX/zG8AUo9ROMU5XUB44Q==", "license": "0BSD" }, "node_modules/ieee754": { @@ -2152,16 +2070,16 @@ } }, "node_modules/jest-diff": { - "version": "30.0.5", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.5.tgz", - "integrity": "sha512-1UIqE9PoEKaHcIKvq2vbibrCog4Y8G0zmOxgQUVEiTqwR5hJVMCoDsN1vFvI5JvwD37hjueZ1C4l2FyGnfpE0A==", + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", + "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", "dev": true, "license": "MIT", "dependencies": { "@jest/diff-sequences": "30.0.1", - "@jest/get-type": "30.0.1", + "@jest/get-type": "30.1.0", "chalk": "^4.1.2", - "pretty-format": "30.0.5" + "pretty-format": "30.2.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -2201,10 +2119,10 @@ } }, "node_modules/lru-cache": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", - "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", - "license": "ISC", + "version": "11.2.4", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.4.tgz", + "integrity": "sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==", + "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" } @@ -2450,9 +2368,9 @@ "integrity": "sha512-dLbWOa4xBn+qeWeIF60qRoB6Pk2jX5P3DIVgOQyMyvBpu931Q+8dXz8X0snJiFkQdohDDLnZQECjzsAj75hgZQ==" }, "node_modules/pretty-format": { - "version": "30.0.5", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.0.5.tgz", - "integrity": "sha512-D1tKtYvByrBkFLe2wHJl2bwMJIiT8rW+XA+TiataH79/FszLQMrpGEvzUVkzPau7OCO0Qnrhpe87PqtOAIB8Yw==", + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", + "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", "dev": true, "license": "MIT", "dependencies": { @@ -3020,43 +2938,52 @@ } }, "node_modules/supertape": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/supertape/-/supertape-11.3.1.tgz", - "integrity": "sha512-jfo8kUh6ru75tTLuwfpTEjMbkP7/Pllgd/pPkKRWTtRyePxmLIzWjvSryT2j6Af5R6SZm44KJRd0aYhb3w3EEw==", + "version": "12.0.12", + "resolved": "https://registry.npmjs.org/supertape/-/supertape-12.0.12.tgz", + "integrity": "sha512-ugmCQsB7s22fCTJKiMb6+Fd8kP7Hsvlo6/aly0qLGgOepu1PVBydhrBPMWaoY3wf+VqLtMkkvwGxUTCFde5z/g==", "dev": true, "license": "MIT", "dependencies": { - "@cloudcmd/stub": "^4.0.0", - "@putout/cli-keypress": "^3.0.0", + "@cloudcmd/stub": "^5.0.0", + "@putout/cli-keypress": "^4.0.0", "@putout/cli-validate-args": "^2.0.0", - "@supertape/engine-loader": "^2.0.0", - "@supertape/formatter-fail": "^4.0.0", - "@supertape/formatter-json-lines": "^2.0.0", - "@supertape/formatter-progress-bar": "^7.0.0", + "@supertape/engine-loader": "^4.0.0", + "@supertape/formatter-fail": "^5.0.0", + "@supertape/formatter-json-lines": "^3.0.0", + "@supertape/formatter-progress-bar": "^8.0.0", "@supertape/formatter-short": "^3.0.0", "@supertape/formatter-tap": "^4.0.0", - "@supertape/formatter-time": "^2.0.0", - "@supertape/operator-stub": "^3.0.0", + "@supertape/formatter-time": "^3.0.0", + "@supertape/operator-stub": "^4.0.0", "cli-progress": "^3.8.2", "flatted": "^3.3.1", - "fullstore": "^3.0.0", + "fullstore": "^4.0.0", "glob": "^11.0.1", "jest-diff": "^30.0.3", "json-with-bigint": "^3.4.4", "once": "^1.4.0", "resolve": "^1.17.0", "stacktracey": "^2.1.7", - "strip-ansi": "^7.0.0", - "try-to-catch": "^3.0.0", + "try-to-catch": "^4.0.0", "wraptile": "^3.0.0", "yargs-parser": "^22.0.0" }, "bin": { - "supertape": "bin/tracer.mjs", - "tape": "bin/tracer.mjs" + "supertape": "bin/tracer.js", + "tape": "bin/tracer.js" }, "engines": { - "node": ">=20" + "node": ">=22" + } + }, + "node_modules/supertape/node_modules/try-to-catch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/try-to-catch/-/try-to-catch-4.0.3.tgz", + "integrity": "sha512-mUz1zpe6nkRQW0XZ/Ojfe/Eg7e5h3s+r+h7ONfP3Oo27/Jm8mkNDAnLzZ/A3sEMApROolzuJGBiQhGmmVDAFLw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22" } }, "node_modules/supertape/node_modules/yargs-parser": { @@ -3197,12 +3124,13 @@ } }, "node_modules/try-catch": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/try-catch/-/try-catch-3.0.1.tgz", - "integrity": "sha512-91yfXw1rr/P6oLpHSyHDOHm0vloVvUoo9FVdw8YwY05QjJQG9OT0LUxe2VRAzmHG+0CUOmI3nhxDUMLxDN/NEQ==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/try-catch/-/try-catch-4.0.7.tgz", + "integrity": "sha512-gkBWUxbiN4T4PsO8KhoQYWzUPN6e0/h12H9H3YhcfPbwaN8b84fy8cFqL4rWTiPh7qHPFaEfklr6OkVxYRW0Gg==", "dev": true, + "license": "MIT", "engines": { - "node": ">=6" + "node": ">=22" } }, "node_modules/try-to-catch": { @@ -3457,9 +3385,9 @@ } }, "node_modules/zod": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", - "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.5.tgz", + "integrity": "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 4d0c43aa..79199602 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "@types/node": "^22.17.1", "c8": "^10.1.2", "cross-env": "^7.0.3", - "supertape": "^11.3.0" + "supertape": "^12.0.12" }, "scripts": { "start": "node --enable-source-maps start.js", From ed76dbd76a83e0379a5ecf7e378cbc5096f3f6bb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 21 Jan 2026 14:53:52 +1300 Subject: [PATCH 082/153] Wait for Discord connection before listening m->d --- src/matrix/appservice.js | 1 - start.js | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/matrix/appservice.js b/src/matrix/appservice.js index 67f16ee1..8f85a514 100644 --- a/src/matrix/appservice.js +++ b/src/matrix/appservice.js @@ -3,6 +3,5 @@ const {reg} = require("../matrix/read-registration") const {AppService} = require("@cloudrac3r/in-your-element") const as = new AppService(reg) -as.listen() module.exports.as = as diff --git a/start.js b/start.js index ca6212ba..39e8ea09 100755 --- a/start.js +++ b/start.js @@ -36,5 +36,9 @@ sync.require("./src/m2d/event-dispatcher") sync.require("./src/web/server") await power.applyPower() + discord.cloud.once("ready", () => { + as.listen() + }) + require("./src/stdin") })() From 0dc9293f0d23be86f674ef7e70c4a50fafbc2576 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 23 Jan 2026 13:28:06 +1300 Subject: [PATCH 083/153] Include user_id with sync per spec --- src/matrix/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/api.js b/src/matrix/api.js index a7b30f1a..84518577 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -162,7 +162,7 @@ function getStateEventOuter(roomID, type, key) { */ async function getInviteState(roomID) { /** @type {Ty.R.SSS} */ - const root = await mreq.mreq("POST", "/client/unstable/org.matrix.simplified_msc3575/sync", { + const root = await mreq.mreq("POST", path("/client/unstable/org.matrix.simplified_msc3575/sync", `@${reg.sender_localpart}:${reg.ooye.server_name}`), { room_subscriptions: { [roomID]: { timeline_limit: 0, From bf9f6b32fd3fda1bae76bcbd943dac5051851721 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 23 Jan 2026 23:37:55 +1300 Subject: [PATCH 084/153] Just join if registered in database --- src/m2d/event-dispatcher.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index e1f6922a..75dad4dc 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -333,14 +333,20 @@ sync.addTemporaryListener(as, "type:m.room.member", guard("m.room.member", */ async event => { if (event.state_key[0] !== "@") return - const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` - if (event.state_key === bot) { + if (event.state_key === utils.bot) { const upgraded = await roomUpgrade.onBotMembership(event, api, createRoom) if (upgraded) return } - if (event.content.membership === "invite" && event.state_key === bot) { + if (event.content.membership === "invite" && event.state_key === utils.bot) { + // Supposed to be here already? + const guildID = select("guild_space", "guild_id", {space_id: event.room_id}).pluck().get() + if (guildID) { + await api.joinRoom(event.room_id) + return + } + // We were invited to a room. We should join, and register the invite details for future reference in web. let attemptedApiMessage = "According to unsigned invite data." let inviteRoomState = event.unsigned?.invite_room_state @@ -369,7 +375,7 @@ async event => { db.prepare("DELETE FROM member_cache WHERE room_id = ? and mxid = ?").run(event.room_id, event.state_key) // Unregister room's use as a direct chat if the bot itself left - if (event.state_key === bot) { + if (event.state_key === utils.bot) { db.prepare("DELETE FROM direct WHERE room_id = ?").run(event.room_id) } } From c0bbdfde60338d04034d79c08a0575ccf19c7c11 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 24 Jan 2026 00:31:50 +1300 Subject: [PATCH 085/153] add to historical_channel_room when linking --- src/web/routes/link.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/web/routes/link.js b/src/web/routes/link.js index ce80fd48..8649348b 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -169,7 +169,10 @@ as.router.post("/api/link", defineEventHandler(async event => { const nick = await api.getStateEvent(parsedBody.matrix, "m.room.name", "").then(content => content.name || null).catch(() => null) const avatar = await api.getStateEvent(parsedBody.matrix, "m.room.avatar", "").then(content => content.url || null).catch(() => null) const topic = await api.getStateEvent(parsedBody.matrix, "m.room.topic", "").then(content => content.topic || null).catch(() => null) - db.prepare("INSERT INTO channel_room (channel_id, room_id, name, guild_id, nick, custom_avatar, custom_topic) VALUES (?, ?, ?, ?, ?, ?, ?)").run(channel.id, parsedBody.matrix, channel.name, guildID, nick, avatar, topic) + db.transaction(() => { + db.prepare("INSERT INTO channel_room (channel_id, room_id, name, guild_id, nick, custom_avatar, custom_topic) VALUES (?, ?, ?, ?, ?, ?, ?)").run(channel.id, parsedBody.matrix, channel.name, guildID, nick, avatar, topic) + db.prepare("INSERT INTO historical_channel_room (reference_channel_id, room_id, upgraded_timestamp) VALUES (?, ?, 0)").run(channel.id, parsedBody.matrix) + })() // Sync room data and space child await createRoom.syncRoom(parsedBody.discord) From 2496f4c3b046625f6751906e4b100940ec2471cb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 25 Jan 2026 13:50:16 +1300 Subject: [PATCH 086/153] Fix retrying own events as non-moderator --- src/d2m/actions/create-room.js | 5 ++--- src/m2d/event-dispatcher.js | 2 +- src/matrix/api.js | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index f9137184..acd47d46 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -487,9 +487,8 @@ async function unbridgeDeletedChannel(channel, guildID) { /** @type {Ty.Event.M_Power_Levels} */ const powerLevelContent = await api.getStateEvent(roomID, "m.room.power_levels", "") powerLevelContent.users ??= {} - const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}` for (const mxid of Object.keys(powerLevelContent.users)) { - if (powerLevelContent.users[mxid] >= 100 && mUtils.eventSenderIsFromDiscord(mxid) && mxid !== bot) { + if (powerLevelContent.users[mxid] >= 100 && mUtils.eventSenderIsFromDiscord(mxid) && mxid !== mUtils.bot) { delete powerLevelContent.users[mxid] await api.sendState(roomID, "m.room.power_levels", "", powerLevelContent, mxid) } @@ -513,7 +512,7 @@ async function unbridgeDeletedChannel(channel, guildID) { // (the room can be used with less clutter and the member list makes sense if it's bridged somewhere else) if (row.autocreate === 0) { // remove sim members - const members = db.prepare("SELECT mxid FROM sim_member WHERE room_id = ? AND mxid <> ?").pluck().all(roomID, bot) + const members = db.prepare("SELECT mxid FROM sim_member WHERE room_id = ? AND mxid <> ?").pluck().all(roomID, mUtils.bot) const preparedDelete = db.prepare("DELETE FROM sim_member WHERE room_id = ? AND mxid = ?") for (const mxid of members) { await api.leaveRoom(roomID, mxid) diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 75dad4dc..e86dac57 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -173,7 +173,7 @@ async function onRetryReactionAdd(reactionEvent) { if (event.sender !== `@${reg.sender_localpart}:${reg.ooye.server_name}` || !error) return // To stop people injecting misleading messages, the reaction needs to come from either the original sender or a room moderator - if (reactionEvent.sender !== event.sender) { + if (reactionEvent.sender !== error.payload.sender) { // Check if it's a room moderator const {powers: {[reactionEvent.sender]: senderPower}, powerLevels} = await utils.getEffectivePower(roomID, [reactionEvent.sender], api) if (senderPower < (powerLevels.state_default ?? 50)) return diff --git a/src/matrix/api.js b/src/matrix/api.js index 84518577..b71c0685 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -162,7 +162,7 @@ function getStateEventOuter(roomID, type, key) { */ async function getInviteState(roomID) { /** @type {Ty.R.SSS} */ - const root = await mreq.mreq("POST", path("/client/unstable/org.matrix.simplified_msc3575/sync", `@${reg.sender_localpart}:${reg.ooye.server_name}`), { + const root = await mreq.mreq("POST", path("/client/unstable/org.matrix.simplified_msc3575/sync", `@${reg.sender_localpart}:${reg.ooye.server_name}`, {timeout: "0"}), { room_subscriptions: { [roomID]: { timeline_limit: 0, From e565342ac8f0d8f1b695a4d335d41c9843773314 Mon Sep 17 00:00:00 2001 From: Ellie Algase Date: Sat, 24 Jan 2026 18:43:30 -0600 Subject: [PATCH 087/153] initial polls support (not exactly working) --- src/d2m/actions/add-or-remove-vote.js | 81 +++++++ src/d2m/actions/close-poll.js | 139 ++++++++++++ src/d2m/actions/send-message.js | 14 ++ src/d2m/converters/edit-to-changes.js | 4 +- src/d2m/converters/message-to-event.js | 64 ++++++ src/d2m/converters/message-to-event.test.js | 54 +++++ src/d2m/discord-client.js | 3 +- src/d2m/event-dispatcher.js | 10 + src/db/migrations/0031-add-polls.sql | 19 ++ src/db/orm-defs.d.ts | 12 ++ src/m2d/actions/send-event.js | 12 +- src/m2d/actions/vote.js | 24 +++ src/m2d/converters/event-to-message.js | 41 +++- src/m2d/event-dispatcher.js | 21 ++ src/types.d.ts | 37 ++++ test/data.js | 228 +++++++++++++++++++- 16 files changed, 749 insertions(+), 14 deletions(-) create mode 100644 src/d2m/actions/add-or-remove-vote.js create mode 100644 src/d2m/actions/close-poll.js create mode 100644 src/db/migrations/0031-add-polls.sql create mode 100644 src/m2d/actions/vote.js diff --git a/src/d2m/actions/add-or-remove-vote.js b/src/d2m/actions/add-or-remove-vote.js new file mode 100644 index 00000000..6c6fbb6f --- /dev/null +++ b/src/d2m/actions/add-or-remove-vote.js @@ -0,0 +1,81 @@ +// @ts-check + +const assert = require("assert").strict + +const passthrough = require("../../passthrough") +const {discord, sync, db, select, from} = passthrough +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("./register-user")} */ +const registerUser = sync.require("./register-user") +/** @type {import("./create-room")} */ +const createRoom = sync.require("../actions/create-room") + +const inFlightPollVotes = new Set() + +/** + * @param {import("discord-api-types/v10").GatewayMessagePollVoteAddDispatch["d"]} data + */ +async function addVote(data){ + const parentID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() // Currently Discord doesn't allow sending a poll with anything else, but we bridge it after all other content so reaction_part: 0 is the part that will have the poll. + if (!parentID) return // Nothing can be done if the parent message was never bridged. + + let realAnswer = select("poll_option", "matrix_option", {message_id: data.message_id, discord_option: data.answer_id.toString()}).pluck().get() // Discord answer IDs don't match those on Matrix-created polls. + assert(realAnswer) + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, vote) VALUES (?, ?, ?)").run(data.user_id, data.message_id, realAnswer) + return modifyVote(data, parentID) +} + +/** + * @param {import("discord-api-types/v10").GatewayMessagePollVoteRemoveDispatch["d"]} data + */ +async function removeVote(data){ + const parentID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() + if (!parentID) return + + let realAnswer = select("poll_option", "matrix_option", {message_id: data.message_id, discord_option: data.answer_id.toString()}).pluck().get() // Discord answer IDs don't match those on Matrix-created polls. + assert(realAnswer) + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ? AND vote = ?").run(data.user_id, data.message_id, realAnswer) + return modifyVote(data, parentID) +} + +/** + * @param {import("discord-api-types/v10").GatewayMessagePollVoteAddDispatch["d"]} data + * @param {string} parentID + */ +async function modifyVote(data, parentID) { + + if (inFlightPollVotes.has(data.user_id+data.message_id)) { // Multiple votes on a poll, and this function has already been called on at least one of them. Need to add these together so we don't ignore votes if someone is voting rapid-fire on a bunch of different polls. + return; + } + + inFlightPollVotes.add(data.user_id+data.message_id) + + await new Promise(resolve => setTimeout(resolve, 1000)) // Wait a second. + + const user = await discord.snow.user.getUser(data.user_id) // Gateway event doesn't give us the object, only the ID. + + const roomID = await createRoom.ensureRoom(data.channel_id) + const senderMxid = await registerUser.ensureSimJoined(user, roomID) + + let answersArray = select("poll_vote", "vote", {discord_or_matrix_user_id: data.user_id, message_id: data.message_id}).pluck().all() + + const eventID = await api.sendEvent(roomID, "org.matrix.msc3381.poll.response", { + "m.relates_to": { + rel_type: "m.reference", + event_id: parentID, + }, + "org.matrix.msc3381.poll.response": { + answers: answersArray + } + }, senderMxid) + + inFlightPollVotes.delete(data.user_id+data.message_id) + + return eventID + +} + +module.exports.addVote = addVote +module.exports.removeVote = removeVote +module.exports.modifyVote = modifyVote diff --git a/src/d2m/actions/close-poll.js b/src/d2m/actions/close-poll.js new file mode 100644 index 00000000..e2460c4e --- /dev/null +++ b/src/d2m/actions/close-poll.js @@ -0,0 +1,139 @@ +// @ts-check + +const assert = require("assert").strict +const DiscordTypes = require("discord-api-types/v10") +const {isDeepStrictEqual} = require("util") + +const passthrough = require("../../passthrough") +const {discord, sync, db, select, from} = passthrough +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("./register-user")} */ +const registerUser = sync.require("./register-user") +/** @type {import("./create-room")} */ +const createRoom = sync.require("../actions/create-room") +/** @type {import("./add-or-remove-vote.js")} */ +const vote = sync.require("../actions/add-or-remove-vote") +/** @type {import("../../m2d/actions/channel-webhook")} */ +const channelWebhook = sync.require("../../m2d/actions/channel-webhook") + +// This handles, in the following order: +// * verifying Matrix-side votes are accurate for a poll originating on Discord, sending missed votes to Matrix if necessary +// * sending a message to Discord if a vote in that poll has been cast on Matrix +// This does *not* handle bridging of poll closures on Discord to Matrix; that takes place in converters/message-to-event.js. + +/** + * @param {number} percent + */ +function barChart(percent){ + let bars = Math.floor(percent*10) + return "█".repeat(bars) + "▒".repeat(10-bars) +} + +async function getAllVotes(channel_id, message_id, answer_id){ + let voteUsers = [] + let after = 0; + while (!voteUsers.length || after){ + let curVotes = await discord.snow.requestHandler.request("/channels/"+channel_id+"/polls/"+message_id+"/answers/"+answer_id, {after: after, limit: 100}, "get", "json") + if (curVotes.users.length == 0 && after == 0){ // Zero votes. + break + } + if (curVotes.users[99]){ + after = curVotes.users[99].id + } + voteUsers = voteUsers.concat(curVotes.users) + } + return voteUsers +} + + +/** + * @param {typeof import("../../../test/data.js")["poll_close"]} message + * @param {DiscordTypes.APIGuild} guild +*/ +async function closePoll(message, guild){ + const pollCloseObject = message.embeds[0] + + const parentID = select("event_message", "event_id", {message_id: message.message_reference.message_id, event_type: "org.matrix.msc3381.poll.start"}).pluck().get() + if (!parentID) return // Nothing we can send Discord-side if we don't have the original poll. We will still send a results message Matrix-side. + + const pollOptions = select("poll_option", "discord_option", {message_id: message.message_reference.message_id}).pluck().all() + // If the closure came from Discord, we want to fetch all the votes there again and bridge over any that got lost to Matrix before posting the results. + // Database reads are cheap, and API calls are expensive, so we will only query Discord when the totals don't match. + + let totalVotes = pollCloseObject.fields.find(element => element.name === "total_votes").value // We could do [2], but best not to rely on the ordering staying consistent. + + let databaseVotes = select("poll_vote", ["discord_or_matrix_user_id", "vote"], {message_id: message.message_reference.message_id}, " AND discord_or_matrix_user_id NOT LIKE '@%'").all() + + if (databaseVotes.length != totalVotes) { // Matching length should be sufficient for most cases. + let voteUsers = [...new Set(databaseVotes.map(vote => vote.discord_or_matrix_user_id))] // Unique array of all users we have votes for in the database. + + // Main design challenge here: we get the data by *answer*, but we need to send it to Matrix by *user*. + + let updatedAnswers = [] // This will be our new array of answers: [{user: ID, votes: [1, 2, 3]}]. + for (let i=0;i{ + let userLocation = updatedAnswers.findIndex(item=>item.id===user.id) + if (userLocation === -1){ // We haven't seen this user yet, so we need to add them. + updatedAnswers.push({id: user.id, votes: [pollOptions[i].toString()]}) // toString as this is what we store and get from the database and send to Matrix. + } else { // This user already voted for another option on the poll. + updatedAnswers[userLocation].votes.push(pollOptions[i]) + } + }) + } + updatedAnswers.map(async user=>{ + voteUsers = voteUsers.filter(item => item != user.id) // Remove any users we have updated answers for from voteUsers. The only remaining entries in this array will be users who voted, but then removed their votes before the poll ended. + let userAnswers = select("poll_vote", "vote", {discord_or_matrix_user_id: user.id, message_id: message.message_reference.message_id}).pluck().all().sort() + let updatedUserAnswers = user.votes.sort() // Sorting both just in case. + if (isDeepStrictEqual(userAnswers,updatedUserAnswers)){ + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(user.id, message.message_reference.message_id) // Delete existing stored votes. + updatedUserAnswers.map(vote=>{ + db.prepare("INSERT INTO poll_vote (discord_or_matrix_user_id, message_id, vote) VALUES (?, ?, ?)").run(user.id, message.message_reference.message_id, vote) + }) + await vote.modifyVote({user_id: user.id, message_id: message.message_reference.message_id, channel_id: message.channel_id, answer_id: 0}, parentID) // Fake answer ID, not actually needed (but we're sorta faking the datatype to call this function). + } + }) + + voteUsers.map(async user_id=>{ // Remove these votes. + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(user_id, message.message_reference.message_id) + await vote.modifyVote({user_id: user_id, message_id: message.message_reference.message_id, channel_id: message.channel_id, answer_id: 0}, parentID) + }) + } + + let combinedVotes = 0; + + let pollResults = pollOptions.map(option => { + let votes = Number(db.prepare("SELECT COUNT(*) FROM poll_vote WHERE message_id = ? AND vote = ?").get(message.message_reference.message_id, option)["COUNT(*)"]) + combinedVotes = combinedVotes + votes + return {answer: option, votes: votes} + }) + + if (combinedVotes!=totalVotes){ // This means some votes were cast on Matrix! + let pollAnswersObject = (await discord.snow.channel.getChannelMessage(message.channel_id, message.message_reference.message_id)).poll.answers + // Now that we've corrected the vote totals, we can get the results again and post them to Discord! + let winningAnswer = 0 + let unique = true + for (let i=1;ipollResults[winningAnswer].votes){ + winningAnswer = i + unique = true + } else if (pollResults[i].votes==pollResults[winningAnswer].votes){ + unique = false + } + } + + let messageString = "📶 Results with Matrix votes\n" + for (let i=0;i{ + let matrixText = answer.poll_media.text + if (answer.poll_media.emoji) { + if (answer.poll_media.emoji.id) { + // Custom emoji. It seems like no Matrix client allows custom emoji in poll answers, so leaving this unimplemented. + } else { + matrixText = "[" + answer.poll_media.emoji.name + "] " + matrixText + } + } + let matrixAnswer = { + id: answer.answer_id.toString(), + "org.matrix.msc1767.text": matrixText + } + fallbackText = fallbackText + "\n" + answer.answer_id.toString() + ". " + matrixText + return matrixAnswer; + }) + return { + $type: "org.matrix.msc3381.poll.start", + "org.matrix.msc3381.poll.start": { + question: { + "org.matrix.msc1767.text": poll.question.text, + body: poll.question.text, + msgtype: "m.text" + }, + kind: "org.matrix.msc3381.poll.disclosed", // Discord always lets you see results, so keeping this consistent with that. + max_selections: maxSelections, + answers: answers + }, + "org.matrix.msc1767.text": fallbackText + } +} + /** * @param {DiscordTypes.APIMessage} message * @param {DiscordTypes.APIGuild} guild @@ -226,6 +266,20 @@ async function messageToEvent(message, guild, options = {}, di) { return [] } + if (message.type === DiscordTypes.MessageType.PollResult) { + const event_id = select("event_message", "event_id", {message_id: message.message_reference?.message_id}).pluck().get() + return [{ + $type: "org.matrix.msc3381.poll.end", + "m.relates_to": { + rel_type: "m.reference", + event_id + }, + "org.matrix.msc3381.poll.end": {}, + body: "This poll has ended.", + msgtype: "m.text" + }] + } + if (message.type === DiscordTypes.MessageType.ThreadStarterMessage) { // This is the message that appears at the top of a thread when the thread was based off an existing message. // It's just a message reference, no content. @@ -702,6 +756,12 @@ async function messageToEvent(message, guild, options = {}, di) { } } + // Then polls + if (message.poll) { + const pollEvent = await pollToEvent(message.poll) + events.push(pollEvent) + } + // Then embeds const urlPreviewEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 for (const embed of message.embeds || []) { @@ -713,6 +773,10 @@ async function messageToEvent(message, guild, options = {}, di) { continue // Matrix's own URL previews are fine for images. } + if (embed.type === "poll_result") { + // The code here is only for the message to be bridged to Matrix. Dealing with the Discord-side updates is in actions/poll-close.js. + } + if (embed.url?.startsWith("https://discord.com/")) { continue // If discord creates an embed preview for a discord channel link, don't copy that embed } diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index 3c0c5d99..c30cd1f6 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -1551,3 +1551,57 @@ test("message2event: forwarded message with unreferenced mention", async t => { "m.mentions": {} }]) }) + +test("message2event: single-choice poll", async t => { + const events = await messageToEvent(data.message.poll_single_choice, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "org.matrix.msc3381.poll.start", + "org.matrix.msc3381.poll.start": { + question: { + "org.matrix.msc1767.text": "only one answer allowed!", + body: "only one answer allowed!", + msgtype: "m.text" + }, + kind: "org.matrix.msc3381.poll.disclosed", // Discord always lets you see results, so keeping this consistent with that. + max_selections: 1, + answers: [{ + id: "1", + "org.matrix.msc1767.text": "[\ud83d\udc4d] answer one" + }, { + id: "2", + "org.matrix.msc1767.text": "[\ud83d\udc4e] answer two" + }, { + id: "3", + "org.matrix.msc1767.text": "answer three" + }] + }, + "org.matrix.msc1767.text": "only one answer allowed!\n1. [\ud83d\udc4d] answer one\n2. [\ud83d\udc4e] answer two\n3. answer three" + }]) +}) + +test("message2event: multiple-choice poll", async t => { + const events = await messageToEvent(data.message.poll_multiple_choice, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "org.matrix.msc3381.poll.start", + "org.matrix.msc3381.poll.start": { + question: { + "org.matrix.msc1767.text": "more than one answer allowed", + body: "more than one answer allowed", + msgtype: "m.text" + }, + kind: "org.matrix.msc3381.poll.disclosed", // Discord always lets you see results, so keeping this consistent with that. + max_selections: 3, + answers: [{ + id: "1", + "org.matrix.msc1767.text": "[😭] no" + }, { + id: "2", + "org.matrix.msc1767.text": "oh no" + }, { + id: "3", + "org.matrix.msc1767.text": "oh noooooo" + }] + }, + "org.matrix.msc1767.text": "more than one answer allowed\n1. [😭] no\n2. oh no\n3. oh noooooo" + }]) +}) diff --git a/src/d2m/discord-client.js b/src/d2m/discord-client.js index c84b466d..7b0fcf8b 100644 --- a/src/d2m/discord-client.js +++ b/src/d2m/discord-client.js @@ -23,7 +23,7 @@ class DiscordClient { /** @type {import("cloudstorm").IClientOptions["intents"]} */ const intents = [ "DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING", - "GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS", + "GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS", "GUILD_MESSAGE_POLLS", "MESSAGE_CONTENT" ] if (reg.ooye.receive_presences !== false) intents.push("GUILD_PRESENCES") @@ -31,7 +31,6 @@ class DiscordClient { this.snow = new SnowTransfer(discordToken) this.cloud = new CloudStorm(discordToken, { shards: [0], - reconnect: true, snowtransferInstance: this.snow, intents, ws: { diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 0a619ef8..599db498 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -32,6 +32,8 @@ const speedbump = sync.require("./actions/speedbump") const retrigger = sync.require("./actions/retrigger") /** @type {import("./actions/set-presence")} */ const setPresence = sync.require("./actions/set-presence") +/** @type {import("./actions/add-or-remove-vote")} */ +const vote = sync.require("./actions/add-or-remove-vote") /** @type {import("../m2d/event-dispatcher")} */ const matrixEventDispatcher = sync.require("../m2d/event-dispatcher") /** @type {import("../discord/interactions/matrix-info")} */ @@ -370,6 +372,14 @@ module.exports = { await createSpace.syncSpaceExpressions(data, false) }, + async MESSAGE_POLL_VOTE_ADD(client, data){ + await vote.addVote(data) + }, + + async MESSAGE_POLL_VOTE_REMOVE(client, data){ + await vote.removeVote(data) + }, + /** * @param {import("./discord-client")} client * @param {DiscordTypes.GatewayPresenceUpdateDispatchData} data diff --git a/src/db/migrations/0031-add-polls.sql b/src/db/migrations/0031-add-polls.sql new file mode 100644 index 00000000..ec879f50 --- /dev/null +++ b/src/db/migrations/0031-add-polls.sql @@ -0,0 +1,19 @@ +BEGIN TRANSACTION; + +CREATE TABLE "poll_option" ( + "message_id" TEXT NOT NULL, + "matrix_option" TEXT NOT NULL, + "discord_option" TEXT NOT NULL, + PRIMARY KEY("message_id","matrix_option") + FOREIGN KEY ("message_id") REFERENCES "message_channel" ("message_id") ON DELETE CASCADE +) WITHOUT ROWID; + +CREATE TABLE "poll_vote" ( + "vote" TEXT NOT NULL, + "message_id" TEXT NOT NULL, + "discord_or_matrix_user_id" TEXT NOT NULL, + PRIMARY KEY("vote","message_id","discord_or_matrix_user_id"), + FOREIGN KEY("message_id") REFERENCES "message_channel" ("message_id") ON DELETE CASCADE +) WITHOUT ROWID; + +COMMIT; diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index 38932cc7..797361ad 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -139,6 +139,18 @@ export type Models = { encoded_emoji: string original_encoding: string | null } + + poll_vote: { + vote: string + message_id: string + discord_or_matrix_user_id: string + } + + poll_option: { + message_id: string + matrix_option: string + discord_option: string + } } export type Prepared = { diff --git a/src/m2d/actions/send-event.js b/src/m2d/actions/send-event.js index ddf82f96..141d33ff 100644 --- a/src/m2d/actions/send-event.js +++ b/src/m2d/actions/send-event.js @@ -22,8 +22,8 @@ const editMessage = sync.require("../../d2m/actions/edit-message") const emojiSheet = sync.require("../actions/emoji-sheet") /** - * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[], pendingFiles?: ({name: string, mxc: string} | {name: string, mxc: string, key: string, iv: string} | {name: string, buffer: Buffer | stream.Readable})[]}} message - * @returns {Promise} + * @param {{poll?: Ty.SendingPoll} & DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[], pendingFiles?: ({name: string, mxc: string} | {name: string, mxc: string, key: string, iv: string} | {name: string, buffer: Buffer | stream.Readable})[]}} message + * @returns {Promise<{poll?: Ty.SendingPoll} & DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]}>} */ async function resolvePendingFiles(message) { if (!message.pendingFiles) return message @@ -59,7 +59,7 @@ async function resolvePendingFiles(message) { return newMessage } -/** @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker} event */ +/** @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start} event */ async function sendEvent(event) { const row = from("channel_room").where({room_id: event.room_id}).select("channel_id", "thread_parent").get() if (!row) return [] // allow the bot to exist in unbridged rooms, just don't do anything with it @@ -133,6 +133,12 @@ async function sendEvent(event) { }, guild, null) ) } + + if (message.poll){ // Need to store answer mapping in the database. + for (let i=0; i{ + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, vote) VALUES (?, ?, ?)").run(event.sender, messageID, answer) + }) +} + +module.exports.updateVote = updateVote diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index b9f80f35..d9ca0747 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -517,7 +517,7 @@ async function getL1L2ReplyLine(called = false) { } /** - * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event + * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start} event * @param {DiscordTypes.APIGuild} guild * @param {DiscordTypes.APIGuildTextChannel} channel * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, mxcDownloader: (mxc: string) => Promise}} di simple-as-nails dependency injection for the matrix API @@ -544,13 +544,15 @@ async function eventToMessage(event, guild, channel, di) { displayNameRunoff = "" } - let content = event.content.body // ultimate fallback + let content = event.content["body"] || "" // ultimate fallback /** @type {{id: string, filename: string}[]} */ const attachments = [] /** @type {({name: string, mxc: string} | {name: string, mxc: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} */ const pendingFiles = [] /** @type {DiscordTypes.APIUser[]} */ const ensureJoined = [] + /** @type {Ty.SendingPoll} */ + let poll = null // Convert content depending on what the message is // Handle images first - might need to handle their `body`/`formatted_body` as well, which will fall through to the text processor @@ -628,6 +630,24 @@ async function eventToMessage(event, guild, channel, di) { } attachments.push({id: "0", filename}) pendingFiles.push({name: filename, mxc: event.content.url}) + + } else if (event.type === "org.matrix.msc3381.poll.start") { + content = "" + const pollContent = event.content["org.matrix.msc3381.poll.start"] // just for convenience + let allowMultiselect = (pollContent.max_selections != 1) + let answers = pollContent.answers.map(answer=>{ + return {poll_media: {text: answer["org.matrix.msc1767.text"]}, matrix_option: answer["id"]} + }) + poll = { + question: { + text: event.content["org.matrix.msc3381.poll.start"].question["org.matrix.msc1767.text"] + }, + answers: answers, + duration: 768, // Maximum duration (32 days). Matrix doesn't allow automatically-expiring polls, so this is the only thing that makes sense to send. + allow_multiselect: allowMultiselect, + layout_type: 1 + } + } else { // Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events. // this event ---is an edit of--> original event ---is a reply to--> past event @@ -828,7 +848,7 @@ async function eventToMessage(event, guild, channel, di) { '' + input + '' ); const root = doc.getElementById("turndown-root"); - async function forEachNode(node) { + async function forEachNode(event, node) { for (; node; node = node.nextSibling) { // Check written mentions if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) { @@ -876,10 +896,10 @@ async function eventToMessage(event, guild, channel, di) { node.setAttribute("data-suppress", "") } } - await forEachNode(node.firstChild) + await forEachNode(event, node.firstChild) } } - await forEachNode(root) + await forEachNode(event, root) // SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet. // First we need to determine which emojis are at the end. @@ -960,7 +980,7 @@ async function eventToMessage(event, guild, channel, di) { // Split into 2000 character chunks const chunks = chunk(content, 2000) - /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]})[]} */ + /** @type {({poll?: Ty.SendingPoll} & DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]})[]} */ const messages = chunks.map(content => ({ content, allowed_mentions: { @@ -983,6 +1003,15 @@ async function eventToMessage(event, guild, channel, di) { messages[0].pendingFiles = pendingFiles } + if (poll) { + if (!messages.length) messages.push({ + content: " ", // stopgap, remove when library updates + username: displayNameShortened, + avatar_url: avatarURL + }) + messages[0].poll = poll + } + const messagesToEdit = [] const messagesToSend = [] for (let i = 0; i < messages.length; i++) { diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index e86dac57..424ad588 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -18,6 +18,8 @@ const addReaction = sync.require("./actions/add-reaction") const redact = sync.require("./actions/redact") /** @type {import("./actions/update-pins")}) */ const updatePins = sync.require("./actions/update-pins") +/** @type {import("./actions/vote")}) */ +const vote = sync.require("./actions/vote") /** @type {import("../matrix/matrix-command-handler")} */ const matrixCommandHandler = sync.require("../matrix/matrix-command-handler") /** @type {import("../matrix/utils")} */ @@ -218,6 +220,25 @@ async event => { await api.ackEvent(event) })) +sync.addTemporaryListener(as, "type:org.matrix.msc3381.poll.start", guard("org.matrix.msc3381.poll.start", +/** + * @param {Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start} event it is a org.matrix.msc3381.poll.start because that's what this listener is filtering for + */ +async event => { + if (utils.eventSenderIsFromDiscord(event.sender)) return + const messageResponses = await sendEvent.sendEvent(event) + await api.ackEvent(event) +})) + +sync.addTemporaryListener(as, "type:org.matrix.msc3381.poll.response", guard("org.matrix.msc3381.poll.response", +/** + * @param {Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Response} event it is a org.matrix.msc3381.poll.response because that's what this listener is filtering for + */ +async event => { + if (utils.eventSenderIsFromDiscord(event.sender)) return + await vote.updateVote(event) // Matrix votes can't be bridged, so all we do is store it in the database. +})) + sync.addTemporaryListener(as, "type:m.reaction", guard("m.reaction", /** * @param {Ty.Event.Outer} event it is a m.reaction because that's what this listener is filtering for diff --git a/src/types.d.ts b/src/types.d.ts index 25bed64a..8f879ec0 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,3 +1,5 @@ +import * as DiscordTypes from "discord-api-types/v10" + export type AppServiceRegistrationConfig = { id: string as_token: string @@ -81,6 +83,10 @@ export type WebhookAuthor = { id: string } +export type SendingPoll = DiscordTypes.RESTAPIPoll & { + answers: (DiscordTypes.APIBasePollAnswer & {matrix_option: string})[] +} + export type PkSystem = { id: string uuid: string @@ -269,6 +275,37 @@ export namespace Event { export type Outer_M_Sticker = Outer & {type: "m.sticker"} + export type Org_Matrix_Msc3381_Poll_Start = { + "org.matrix.msc3381.poll.start": { + question: { + "org.matrix.msc1767.text": string + body: string + msgtype: string + }, + kind: string + max_selections: number + answers: { + id: string + "org.matrix.msc1767.text": string + }[] + "org.matrix.msc1767.text": string + } + } + + export type Outer_Org_Matrix_Msc3381_Poll_Start = Outer & {type: "org.matrix.msc3381.poll.start"} + + export type Org_Matrix_Msc3381_Poll_Response = { + "org.matrix.msc3381.poll.response": { + answers: string[] + } + "m.relates_to": { + rel_type: string + event_id: string + } + } + + export type Outer_Org_Matrix_Msc3381_Poll_Response = Outer & {type: "org.matrix.msc3381.poll.response"} + export type M_Room_Member = { membership: string displayname?: string diff --git a/test/data.js b/test/data.js index 0942a87d..786737c6 100644 --- a/test/data.js +++ b/test/data.js @@ -3593,7 +3593,233 @@ module.exports = { }, attachments: [], guild_id: "286888431945252874" - } + }, + poll_single_choice: { + type: 0, + content: "", + mentions: [], + mention_roles: [], + attachments: [], + embeds: [], + timestamp: "2025-02-15T23:19:04.127000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + id: "1340462414176718889", + channel_id: "1340048919589158986", + author: { + id: "307894326028140546", + username: "ellienyaa", + avatar: "f98417a0a0b4aecc7d7667bece353b7e", + discriminator: "0", + public_flags: 128, + flags: 128, + banner: null, + accent_color: null, + global_name: "unambiguously boring username", + avatar_decoration_data: null, + banner_color: null, + clan: null, + primary_guild: null + }, + pinned: false, + mention_everyone: false, + tts: false, + position: 0, + poll: { + question: { + text: "only one answer allowed!" + }, + answers: [ + { + answer_id: 1, + poll_media: { + text: "answer one", + emoji: { + id: null, + name: "\ud83d\udc4d" + } + } + }, + { + answer_id: 2, + poll_media: { + text: "answer two", + emoji: { + id: null, + name: "\ud83d\udc4e" + } + } + }, + { + answer_id: 3, + poll_media: { + text: "answer three" + } + } + ], + expiry: "2025-02-16T23:19:04.122364+00:00", + allow_multiselect: false, + layout_type: 1, + results: { + answer_counts: [], + is_finalized: false + } + } + }, + poll_multiple_choice: { + type: 0, + content: "", + mentions: [], + mention_roles: [], + attachments: [], + embeds: [], + timestamp: "2025-02-16T00:47:12.310000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + id: "1340484594423562300", + channel_id: "1340048919589158986", + author: { + id: "307894326028140546", + username: "ellienyaa", + avatar: "f98417a0a0b4aecc7d7667bece353b7e", + discriminator: "0", + public_flags: 128, + flags: 128, + banner: null, + accent_color: null, + global_name: "unambiguously boring username", + avatar_decoration_data: null, + banner_color: null, + clan: null, + primary_guild: null + }, + pinned: false, + mention_everyone: false, + tts: false, + position: 0, + poll: { + question: { + text: "more than one answer allowed" + }, + answers: [ + { + answer_id: 1, + poll_media: { + text: "no", + emoji: { + id: null, + name: "😭" + } + } + }, + { + answer_id: 2, + poll_media: { + text: "oh no", + emoji: { + id: "891723675261366292", + name: "this" + } + } + }, + { + answer_id: 3, + poll_media: { + text: "oh noooooo", + emoji: { + id: "964520120682680350", + name: "disapprove" + } + } + } + ], + expiry: "2025-02-17T00:47:12.307985+00:00", + allow_multiselect: true, + layout_type: 1, + results: { + answer_counts: [], + is_finalized: false + } + } + }, + poll_close: { + type: 46, + content: "", + mentions: [ + { + id: "307894326028140546", + username: "ellienyaa", + avatar: "f98417a0a0b4aecc7d7667bece353b7e", + discriminator: "0", + public_flags: 128, + flags: 128, + banner: null, + accent_color: null, + global_name: "unambiguously boring username", + avatar_decoration_data: null, + banner_color: null, + clan: null, + primary_guild: null + } + ], + mention_roles: [], + attachments: [], + embeds: [ + { + type: "poll_result", + fields: [ + { + name: "poll_question_text", + value: "test poll that's being closed", + inline: false + }, + { + name: "victor_answer_votes", + value: "0", + inline: false + }, + { + name: "total_votes", + value: "0", + inline: false + } + ], + content_scan_version: 0 + } + ], + timestamp: "2025-02-20T23:07:12.178000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + id: "1342271367374049351", + channel_id: "1340048919589158986", + author: { + id: "307894326028140546", + username: "ellienyaa", + avatar: "f98417a0a0b4aecc7d7667bece353b7e", + discriminator: "0", + public_flags: 128, + flags: 128, + banner: null, + accent_color: null, + global_name: "unambiguously boring username", + avatar_decoration_data: null, + banner_color: null, + clan: null, + primary_guild: null + }, + pinned: false, + mention_everyone: false, + tts: false, + message_reference: { + type: 0, + channel_id: "1340048919589158986", + message_id: "1342271353990021206" + }, + position: 0 + } }, pk_message: { pk_reply_to_matrix: { From afca4de6b6e350dea6735287ad1d539c5264da4a Mon Sep 17 00:00:00 2001 From: Ellie Algase Date: Sun, 25 Jan 2026 00:28:42 -0600 Subject: [PATCH 088/153] Bridge polls from Matrix as pseudo-polls on Discord (with an embed). Not 100% working. Co-authored-by: Cadence Ember --- src/d2m/actions/add-or-remove-vote.js | 71 +++++++------ src/d2m/actions/close-poll.js | 139 +++++++++++++------------ src/d2m/actions/send-message.js | 22 +++- src/db/migrations/0031-add-polls.sql | 19 ---- src/db/migrations/0032-add-polls.sql | 34 ++++++ src/db/orm-defs.d.ts | 19 +++- src/discord/interactions/vote.js | 95 +++++++++++++++++ src/m2d/actions/send-event.js | 26 +++-- src/m2d/actions/vote.js | 2 +- src/m2d/converters/event-to-message.js | 39 ++++--- src/m2d/converters/poll-components.js | 103 ++++++++++++++++++ src/types.d.ts | 4 - 12 files changed, 417 insertions(+), 156 deletions(-) delete mode 100644 src/db/migrations/0031-add-polls.sql create mode 100644 src/db/migrations/0032-add-polls.sql create mode 100644 src/discord/interactions/vote.js create mode 100644 src/m2d/converters/poll-components.js diff --git a/src/d2m/actions/add-or-remove-vote.js b/src/d2m/actions/add-or-remove-vote.js index 6c6fbb6f..e9fee366 100644 --- a/src/d2m/actions/add-or-remove-vote.js +++ b/src/d2m/actions/add-or-remove-vote.js @@ -1,6 +1,9 @@ // @ts-check const assert = require("assert").strict +const DiscordTypes = require("discord-api-types/v10") +const {Semaphore} = require("@chriscdn/promise-semaphore") +const {scheduler} = require("timers/promises") const passthrough = require("../../passthrough") const {discord, sync, db, select, from} = passthrough @@ -11,71 +14,81 @@ const registerUser = sync.require("./register-user") /** @type {import("./create-room")} */ const createRoom = sync.require("../actions/create-room") -const inFlightPollVotes = new Set() +const inFlightPollSema = new Semaphore() /** * @param {import("discord-api-types/v10").GatewayMessagePollVoteAddDispatch["d"]} data */ async function addVote(data){ - const parentID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() // Currently Discord doesn't allow sending a poll with anything else, but we bridge it after all other content so reaction_part: 0 is the part that will have the poll. - if (!parentID) return // Nothing can be done if the parent message was never bridged. + const pollEventID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() // Currently Discord doesn't allow sending a poll with anything else, but we bridge it after all other content so reaction_part: 0 is the part that will have the poll. + if (!pollEventID) return // Nothing can be done if the parent message was never bridged. let realAnswer = select("poll_option", "matrix_option", {message_id: data.message_id, discord_option: data.answer_id.toString()}).pluck().get() // Discord answer IDs don't match those on Matrix-created polls. assert(realAnswer) - db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, vote) VALUES (?, ?, ?)").run(data.user_id, data.message_id, realAnswer) - return modifyVote(data, parentID) + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(data.user_id, data.message_id, realAnswer) + return debounceSendVotes(data, pollEventID) } /** * @param {import("discord-api-types/v10").GatewayMessagePollVoteRemoveDispatch["d"]} data */ async function removeVote(data){ - const parentID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() - if (!parentID) return + const pollEventID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() + if (!pollEventID) return let realAnswer = select("poll_option", "matrix_option", {message_id: data.message_id, discord_option: data.answer_id.toString()}).pluck().get() // Discord answer IDs don't match those on Matrix-created polls. assert(realAnswer) - db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ? AND vote = ?").run(data.user_id, data.message_id, realAnswer) - return modifyVote(data, parentID) + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ? AND matrix_option = ?").run(data.user_id, data.message_id, realAnswer) + return debounceSendVotes(data, pollEventID) } /** + * Multiple-choice polls send all the votes at the same time. This debounces and sends the combined votes. + * In the meantime, the combined votes are assembled in the `poll_vote` database table by the above functions. * @param {import("discord-api-types/v10").GatewayMessagePollVoteAddDispatch["d"]} data - * @param {string} parentID + * @param {string} pollEventID + * @return {Promise} event ID of Matrix vote */ -async function modifyVote(data, parentID) { +async function debounceSendVotes(data, pollEventID) { + return await inFlightPollSema.request(async () => { + await scheduler.wait(1000) // Wait for votes to be collected - if (inFlightPollVotes.has(data.user_id+data.message_id)) { // Multiple votes on a poll, and this function has already been called on at least one of them. Need to add these together so we don't ignore votes if someone is voting rapid-fire on a bunch of different polls. - return; + const user = await discord.snow.user.getUser(data.user_id) // Gateway event doesn't give us the object, only the ID. + return await sendVotes(user, data.channel_id, data.message_id, pollEventID) + }, `${data.user_id}/${data.message_id}`) +} + +/** + * @param {DiscordTypes.APIUser} user + * @param {string} channelID + * @param {string} pollMessageID + * @param {string} pollEventID + */ +async function sendVotes(user, channelID, pollMessageID, pollEventID) { + const latestRoomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() + const matchingRoomID = from("message_room").join("historical_channel_room", "historical_room_index").where({message_id: pollMessageID}).pluck("room_id").get() + if (!latestRoomID || latestRoomID !== matchingRoomID) { // room upgrade mid-poll?? + db.prepare("UPDATE poll SET is_closed = 1 WHERE message_id = ?").run(pollMessageID) + return } - inFlightPollVotes.add(data.user_id+data.message_id) + const senderMxid = await registerUser.ensureSimJoined(user, matchingRoomID) - await new Promise(resolve => setTimeout(resolve, 1000)) // Wait a second. - - const user = await discord.snow.user.getUser(data.user_id) // Gateway event doesn't give us the object, only the ID. - - const roomID = await createRoom.ensureRoom(data.channel_id) - const senderMxid = await registerUser.ensureSimJoined(user, roomID) - - let answersArray = select("poll_vote", "vote", {discord_or_matrix_user_id: data.user_id, message_id: data.message_id}).pluck().all() - - const eventID = await api.sendEvent(roomID, "org.matrix.msc3381.poll.response", { + const answersArray = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: user.id, message_id: pollMessageID}).pluck().all() + const eventID = await api.sendEvent(matchingRoomID, "org.matrix.msc3381.poll.response", { "m.relates_to": { rel_type: "m.reference", - event_id: parentID, + event_id: pollEventID, }, "org.matrix.msc3381.poll.response": { answers: answersArray } }, senderMxid) - inFlightPollVotes.delete(data.user_id+data.message_id) - return eventID - } module.exports.addVote = addVote module.exports.removeVote = removeVote -module.exports.modifyVote = modifyVote +module.exports.debounceSendVotes = debounceSendVotes +module.exports.sendVotes = sendVotes \ No newline at end of file diff --git a/src/d2m/actions/close-poll.js b/src/d2m/actions/close-poll.js index e2460c4e..a715177d 100644 --- a/src/d2m/actions/close-poll.js +++ b/src/d2m/actions/close-poll.js @@ -30,16 +30,25 @@ function barChart(percent){ return "█".repeat(bars) + "▒".repeat(10-bars) } -async function getAllVotes(channel_id, message_id, answer_id){ +/** + * @param {string} channelID + * @param {string} messageID + * @param {string} answerID + * @returns {Promise} + */ +async function getAllVotesOnAnswer(channelID, messageID, answerID){ + const limit = 100 + /** @type {DiscordTypes.RESTGetAPIPollAnswerVotersResult["users"]} */ let voteUsers = [] - let after = 0; - while (!voteUsers.length || after){ - let curVotes = await discord.snow.requestHandler.request("/channels/"+channel_id+"/polls/"+message_id+"/answers/"+answer_id, {after: after, limit: 100}, "get", "json") - if (curVotes.users.length == 0 && after == 0){ // Zero votes. + let after = undefined + while (!voteUsers.length || after) { + const curVotes = await discord.snow.channel.getPollAnswerVoters(channelID, messageID, answerID, {after: after, limit}) + if (curVotes.users.length === 0) { // Reached the end. break } - if (curVotes.users[99]){ - after = curVotes.users[99].id + if (curVotes.users.length >= limit) { // Loop again for the next page. + // @ts-ignore - stupid + after = curVotes.users.at(-1).id } voteUsers = voteUsers.concat(curVotes.users) } @@ -48,91 +57,89 @@ async function getAllVotes(channel_id, message_id, answer_id){ /** - * @param {typeof import("../../../test/data.js")["poll_close"]} message + * @param {typeof import("../../../test/data.js")["poll_close"]} closeMessage * @param {DiscordTypes.APIGuild} guild */ -async function closePoll(message, guild){ - const pollCloseObject = message.embeds[0] +async function closePoll(closeMessage, guild){ + const pollCloseObject = closeMessage.embeds[0] - const parentID = select("event_message", "event_id", {message_id: message.message_reference.message_id, event_type: "org.matrix.msc3381.poll.start"}).pluck().get() - if (!parentID) return // Nothing we can send Discord-side if we don't have the original poll. We will still send a results message Matrix-side. + const pollMessageID = closeMessage.message_reference.message_id + const pollEventID = select("event_message", "event_id", {message_id: pollMessageID, event_type: "org.matrix.msc3381.poll.start"}).pluck().get() + if (!pollEventID) return // Nothing we can send Discord-side if we don't have the original poll. We will still send a results message Matrix-side. + + const discordPollOptions = select("poll_option", "discord_option", {message_id: pollMessageID}).pluck().all() + assert(discordPollOptions.every(x => typeof x === "string")) // This poll originated on Discord so it will have Discord option IDs - const pollOptions = select("poll_option", "discord_option", {message_id: message.message_reference.message_id}).pluck().all() // If the closure came from Discord, we want to fetch all the votes there again and bridge over any that got lost to Matrix before posting the results. // Database reads are cheap, and API calls are expensive, so we will only query Discord when the totals don't match. - let totalVotes = pollCloseObject.fields.find(element => element.name === "total_votes").value // We could do [2], but best not to rely on the ordering staying consistent. + const totalVotes = pollCloseObject.fields.find(element => element.name === "total_votes").value // We could do [2], but best not to rely on the ordering staying consistent. - let databaseVotes = select("poll_vote", ["discord_or_matrix_user_id", "vote"], {message_id: message.message_reference.message_id}, " AND discord_or_matrix_user_id NOT LIKE '@%'").all() + const databaseVotes = select("poll_vote", ["discord_or_matrix_user_id", "matrix_option"], {message_id: pollMessageID}, " AND discord_or_matrix_user_id NOT LIKE '@%'").all() - if (databaseVotes.length != totalVotes) { // Matching length should be sufficient for most cases. + if (databaseVotes.length !== totalVotes) { // Matching length should be sufficient for most cases. let voteUsers = [...new Set(databaseVotes.map(vote => vote.discord_or_matrix_user_id))] // Unique array of all users we have votes for in the database. // Main design challenge here: we get the data by *answer*, but we need to send it to Matrix by *user*. - let updatedAnswers = [] // This will be our new array of answers: [{user: ID, votes: [1, 2, 3]}]. - for (let i=0;i{ - let userLocation = updatedAnswers.findIndex(item=>item.id===user.id) + /** @type {{user: DiscordTypes.APIUser, matrixOptionVotes: string[]}[]} This will be our new array of answers */ + const updatedAnswers = [] + + for (const discordPollOption of discordPollOptions) { + const optionUsers = await getAllVotesOnAnswer(closeMessage.channel_id, pollMessageID, discordPollOption) // Array of user IDs who voted for the option we're testing. + optionUsers.map(user => { + const userLocation = updatedAnswers.findIndex(answer => answer.user.id === user.id) + const matrixOption = select("poll_option", "matrix_option", {message_id: pollMessageID, discord_option: discordPollOption}).pluck().get() + assert(matrixOption) if (userLocation === -1){ // We haven't seen this user yet, so we need to add them. - updatedAnswers.push({id: user.id, votes: [pollOptions[i].toString()]}) // toString as this is what we store and get from the database and send to Matrix. + updatedAnswers.push({user, matrixOptionVotes: [matrixOption]}) // toString as this is what we store and get from the database and send to Matrix. } else { // This user already voted for another option on the poll. - updatedAnswers[userLocation].votes.push(pollOptions[i]) + updatedAnswers[userLocation].matrixOptionVotes.push(matrixOption) } }) } - updatedAnswers.map(async user=>{ - voteUsers = voteUsers.filter(item => item != user.id) // Remove any users we have updated answers for from voteUsers. The only remaining entries in this array will be users who voted, but then removed their votes before the poll ended. - let userAnswers = select("poll_vote", "vote", {discord_or_matrix_user_id: user.id, message_id: message.message_reference.message_id}).pluck().all().sort() - let updatedUserAnswers = user.votes.sort() // Sorting both just in case. - if (isDeepStrictEqual(userAnswers,updatedUserAnswers)){ - db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(user.id, message.message_reference.message_id) // Delete existing stored votes. - updatedUserAnswers.map(vote=>{ - db.prepare("INSERT INTO poll_vote (discord_or_matrix_user_id, message_id, vote) VALUES (?, ?, ?)").run(user.id, message.message_reference.message_id, vote) - }) - await vote.modifyVote({user_id: user.id, message_id: message.message_reference.message_id, channel_id: message.channel_id, answer_id: 0}, parentID) // Fake answer ID, not actually needed (but we're sorta faking the datatype to call this function). - } - }) - voteUsers.map(async user_id=>{ // Remove these votes. - db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(user_id, message.message_reference.message_id) - await vote.modifyVote({user_id: user_id, message_id: message.message_reference.message_id, channel_id: message.channel_id, answer_id: 0}, parentID) - }) + // Check for inconsistencies in what was cached in database vs final confirmed poll answers + // If different, sync the final confirmed answers to Matrix-side to make it accurate there too + + await Promise.all(updatedAnswers.map(async answer => { + voteUsers = voteUsers.filter(item => item !== answer.user.id) // Remove any users we have updated answers for from voteUsers. The only remaining entries in this array will be users who voted, but then removed their votes before the poll ended. + const cachedAnswers = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: answer.user.id, message_id: pollMessageID}).pluck().all() + if (!isDeepStrictEqual(new Set(cachedAnswers), new Set(answer.matrixOptionVotes))){ + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(answer.user.id, pollMessageID) // Delete existing stored votes. + for (const matrixOption of answer.matrixOptionVotes) { + db.prepare("INSERT INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(answer.user.id, pollMessageID, matrixOption) + } + await vote.debounceSendVotes({user_id: answer.user.id, message_id: pollMessageID, channel_id: closeMessage.channel_id, answer_id: 0}, pollEventID) // Fake answer ID, not actually needed (but we're sorta faking the datatype to call this function). + } + })) + + await Promise.all(voteUsers.map(async user_id => { // Remove these votes. + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(user_id, pollMessageID) + await vote.debounceSendVotes({user_id: user_id, message_id: pollMessageID, channel_id: closeMessage.channel_id, answer_id: 0}, pollEventID) + })) } - let combinedVotes = 0; + /** @type {{discord_option: string, option_text: string, count: number}[]} */ + const pollResults = db.prepare("SELECT discord_option, option_text, count(*) as count FROM poll_option INNER JOIN poll_vote USING (message_id, matrix_option) GROUP BY discord_option").all() + const combinedVotes = pollResults.reduce((a, c) => a + c.count, 0) - let pollResults = pollOptions.map(option => { - let votes = Number(db.prepare("SELECT COUNT(*) FROM poll_vote WHERE message_id = ? AND vote = ?").get(message.message_reference.message_id, option)["COUNT(*)"]) - combinedVotes = combinedVotes + votes - return {answer: option, votes: votes} - }) - - if (combinedVotes!=totalVotes){ // This means some votes were cast on Matrix! - let pollAnswersObject = (await discord.snow.channel.getChannelMessage(message.channel_id, message.message_reference.message_id)).poll.answers + if (combinedVotes !== totalVotes) { // This means some votes were cast on Matrix! + const message = await discord.snow.channel.getChannelMessage(closeMessage.channel_id, pollMessageID) + assert(message?.poll?.answers) // Now that we've corrected the vote totals, we can get the results again and post them to Discord! - let winningAnswer = 0 - let unique = true - for (let i=1;ipollResults[winningAnswer].votes){ - winningAnswer = i - unique = true - } else if (pollResults[i].votes==pollResults[winningAnswer].votes){ - unique = false - } - } + const topAnswers = pollResults.toSorted() + const unique = topAnswers.length > 1 && topAnswers[0].count === topAnswers[1].count - let messageString = "📶 Results with Matrix votes\n" - for (let i=0;i { + db.prepare("INSERT INTO poll (message_id, max_selections, question_text, is_closed) VALUES (?, ?, ?, 0)").run( + message.id, + event["org.matrix.msc3381.poll.start"].max_selections, + event["org.matrix.msc3381.poll.start"].question["org.matrix.msc1767.text"] + ) + for (const [index, option] of Object.entries(event["org.matrix.msc3381.poll.start"].answers)) { + db.prepare("INSERT INTO poll_option (message_id, matrix_option, discord_option, option_text, seq) VALUES (?, ?, ?, ?, ?)").run( + message.id, + option.id, + option.id, + option["org.matrix.msc1767.text"], + index + ) + } + })() } eventIDs.push(eventID) - } return eventIDs diff --git a/src/db/migrations/0031-add-polls.sql b/src/db/migrations/0031-add-polls.sql deleted file mode 100644 index ec879f50..00000000 --- a/src/db/migrations/0031-add-polls.sql +++ /dev/null @@ -1,19 +0,0 @@ -BEGIN TRANSACTION; - -CREATE TABLE "poll_option" ( - "message_id" TEXT NOT NULL, - "matrix_option" TEXT NOT NULL, - "discord_option" TEXT NOT NULL, - PRIMARY KEY("message_id","matrix_option") - FOREIGN KEY ("message_id") REFERENCES "message_channel" ("message_id") ON DELETE CASCADE -) WITHOUT ROWID; - -CREATE TABLE "poll_vote" ( - "vote" TEXT NOT NULL, - "message_id" TEXT NOT NULL, - "discord_or_matrix_user_id" TEXT NOT NULL, - PRIMARY KEY("vote","message_id","discord_or_matrix_user_id"), - FOREIGN KEY("message_id") REFERENCES "message_channel" ("message_id") ON DELETE CASCADE -) WITHOUT ROWID; - -COMMIT; diff --git a/src/db/migrations/0032-add-polls.sql b/src/db/migrations/0032-add-polls.sql new file mode 100644 index 00000000..06979370 --- /dev/null +++ b/src/db/migrations/0032-add-polls.sql @@ -0,0 +1,34 @@ +BEGIN TRANSACTION; + +DROP TABLE IF EXISTS "poll"; +DROP TABLE IF EXISTS "poll_option"; +DROP TABLE IF EXISTS "poll_vote"; + +CREATE TABLE "poll" ( + "message_id" TEXT NOT NULL, + "max_selections" INTEGER NOT NULL, + "question_text" TEXT NOT NULL, + "is_closed" INTEGER NOT NULL, + PRIMARY KEY ("message_id"), + FOREIGN KEY ("message_id") REFERENCES "message_room" ("message_id") ON DELETE CASCADE +) WITHOUT ROWID; + +CREATE TABLE "poll_option" ( + "message_id" TEXT NOT NULL, + "matrix_option" TEXT NOT NULL, + "discord_option" TEXT, + "option_text" TEXT NOT NULL, + "seq" INTEGER NOT NULL, + PRIMARY KEY ("message_id", "matrix_option"), + FOREIGN KEY ("message_id") REFERENCES "poll" ("message_id") ON DELETE CASCADE +) WITHOUT ROWID; + +CREATE TABLE "poll_vote" ( + "message_id" TEXT NOT NULL, + "matrix_option" TEXT NOT NULL, + "discord_or_matrix_user_id" TEXT NOT NULL, + PRIMARY KEY ("message_id", "matrix_option", "discord_or_matrix_user_id"), + FOREIGN KEY ("message_id", "matrix_option") REFERENCES "poll_option" ("message_id", "matrix_option") ON DELETE CASCADE +) WITHOUT ROWID; + +COMMIT; diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index 797361ad..e36ed491 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -140,16 +140,25 @@ export type Models = { original_encoding: string | null } - poll_vote: { - vote: string + poll: { // not actually in database yet message_id: string - discord_or_matrix_user_id: string + max_selections: number + question_text: string + is_closed: number } - + poll_option: { message_id: string matrix_option: string - discord_option: string + discord_option: string | null + option_text: string // not actually in database yet + seq: number // not actually in database yet + } + + poll_vote: { + message_id: string + matrix_option: string + discord_or_matrix_user_id: string } } diff --git a/src/discord/interactions/vote.js b/src/discord/interactions/vote.js new file mode 100644 index 00000000..91ba97e7 --- /dev/null +++ b/src/discord/interactions/vote.js @@ -0,0 +1,95 @@ +// @ts-check + +const DiscordTypes = require("discord-api-types/v10") +const {discord, sync, select, from, db} = require("../../passthrough") +const assert = require("assert/strict") +const {id: botID} = require("../../../addbot") +const {InteractionMethods} = require("snowtransfer") + +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") +/** @type {import("../../m2d/converters/poll-components")} */ +const pollComponents = sync.require("../../m2d/converters/poll-components") +/** @type {import("../../d2m/actions/add-or-remove-vote")} */ +const vote = sync.require("../../d2m/actions/add-or-remove-vote") + +/** + * @param {DiscordTypes.APIMessageComponentButtonInteraction} interaction + * @param {{api: typeof api}} di + * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} + */ +async function* _interact({data, message, member, user}, {api}) { + const discordUser = member?.user || user + assert(discordUser) + const userID = discordUser.id + + const matrixPollEvent = select("event_message", "event_id", {message_id: message.id}).pluck().get() + assert(matrixPollEvent) + + const matrixOption = select("poll_option", "matrix_option", {discord_option: data.custom_id, message_id: message.id}).pluck().get() + assert(matrixOption) + + const pollRow = select("poll", ["question_text", "max_selections"], {message_id: message.id}).get() + assert(pollRow) + const maxSelections = pollRow.max_selections + const alreadySelected = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: userID, message_id: message.id}).pluck().all() + + // Show modal (if no capacity) + if (maxSelections > 1 && alreadySelected.length === maxSelections) { + // TODO: show modal + return + } + + // We are going to do a server operation so need to show loading state + yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.DeferredMessageUpdate, + }} + + // Remove a vote + if (alreadySelected.includes(data.custom_id)) { + db.prepare("DELETE FROM poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, data.custom_id) + } + // Replace votes (if only one selection is allowed) + else if (maxSelections === 1 && alreadySelected.length === 1) { + db.transaction(() => { + db.prepare("DELETE FROM poll_vote WHERE message_id = ? AND discord_or_matrix_user_id = ?").run(message.id, userID) + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, data.custom_id) + })() + } + // Add a vote (if capacity) + else if (alreadySelected.length < maxSelections) { + db.transaction(() => { + db.prepare("DELETE FROM poll_vote WHERE message_id = ? AND discord_or_matrix_user_id = ?").run(message.id, userID) + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, data.custom_id) + })() + } + + // Sync changes to Matrix + await vote.sendVotes(discordUser, message.channel_id, message.id, matrixPollEvent) + + // Check the poll is not closed (it may have been closed by sendVotes if we discover we can't send) + const isClosed = select("poll", "is_closed", {message_id: message.id}).pluck().get() + + /** @type {{matrix_option: string, option_text: string, count: number}[]} */ + const pollResults = db.prepare("SELECT matrix_option, option_text, count(*) as count FROM poll_option INNER JOIN poll_vote USING (message_id, matrix_option) GROUP BY matrix_option").all() + return yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.UpdateMessage, + data: pollComponents.getPollComponents(!!isClosed, maxSelections, pollRow.question_text, pollResults) + }} +} + +/* c8 ignore start */ + +/** @param {DiscordTypes.APIMessageComponentButtonInteraction} interaction */ +async function interact(interaction) { + for await (const response of _interact(interaction, {api})) { + if (response.createInteractionResponse) { + await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, response.createInteractionResponse) + } + } +} + +module.exports.interact = interact +module.exports._interact = _interact diff --git a/src/m2d/actions/send-event.js b/src/m2d/actions/send-event.js index 141d33ff..f18385fa 100644 --- a/src/m2d/actions/send-event.js +++ b/src/m2d/actions/send-event.js @@ -22,8 +22,8 @@ const editMessage = sync.require("../../d2m/actions/edit-message") const emojiSheet = sync.require("../actions/emoji-sheet") /** - * @param {{poll?: Ty.SendingPoll} & DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[], pendingFiles?: ({name: string, mxc: string} | {name: string, mxc: string, key: string, iv: string} | {name: string, buffer: Buffer | stream.Readable})[]}} message - * @returns {Promise<{poll?: Ty.SendingPoll} & DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]}>} + * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[], pendingFiles?: ({name: string, mxc: string} | {name: string, mxc: string, key: string, iv: string} | {name: string, buffer: Buffer | stream.Readable})[]}} message + * @returns {Promise} */ async function resolvePendingFiles(message) { if (!message.pendingFiles) return message @@ -71,6 +71,7 @@ async function sendEvent(event) { } /** @type {DiscordTypes.APIGuildTextChannel} */ // @ts-ignore const channel = discord.channels.get(channelID) + // @ts-ignore const guild = discord.guilds.get(channel.guild_id) assert(guild) const historicalRoomIndex = select("historical_channel_room", "historical_room_index", {room_id: event.room_id}).pluck().get() @@ -133,12 +134,25 @@ async function sendEvent(event) { }, guild, null) ) } + } - if (message.poll){ // Need to store answer mapping in the database. - for (let i=0; i { + const messageID = messageResponses[0].id + db.prepare("INSERT INTO poll (message_id, max_selections, question_text, is_closed) VALUES (?, ?, ?, 0)").run( + messageID, + event.content["org.matrix.msc3381.poll.start"].max_selections, + event.content["org.matrix.msc3381.poll.start"].question["org.matrix.msc1767.text"] + ) + for (const [i, option] of Object.entries(event.content["org.matrix.msc3381.poll.start"].answers)) { + db.prepare("INSERT INTO poll_option (message_id, matrix_option, option_text, seq) VALUES (?, ?, ?, ?)").run( + messageID, + option.id, + option["org.matrix.msc1767.text"], + i + ) } - } + })() } for (const user of ensureJoined) { diff --git a/src/m2d/actions/vote.js b/src/m2d/actions/vote.js index e19328c6..5bb5cd43 100644 --- a/src/m2d/actions/vote.js +++ b/src/m2d/actions/vote.js @@ -17,7 +17,7 @@ async function updateVote(event) { db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(event.sender, messageID) // Clear all the existing votes, since this overwrites. Technically we could check and only overwrite the changes, but the complexity isn't worth it. event.content["org.matrix.msc3381.poll.response"].answers.map(answer=>{ - db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, vote) VALUES (?, ?, ?)").run(event.sender, messageID, answer) + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(event.sender, messageID, answer) }) } diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index d9ca0747..ab53d082 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -22,6 +22,8 @@ const dUtils = sync.require("../../discord/utils") const file = sync.require("../../matrix/file") /** @type {import("./emoji-sheet")} */ const emojiSheet = sync.require("./emoji-sheet") +/** @type {import("./poll-components")} */ +const pollComponents = sync.require("./poll-components") /** @type {import("../actions/setup-emojis")} */ const setupEmojis = sync.require("../actions/setup-emojis") /** @type {import("../../d2m/converters/user-to-mxid")} */ @@ -551,8 +553,8 @@ async function eventToMessage(event, guild, channel, di) { const pendingFiles = [] /** @type {DiscordTypes.APIUser[]} */ const ensureJoined = [] - /** @type {Ty.SendingPoll} */ - let poll = null + /** @type {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody?} */ + let pollMessage = null // Convert content depending on what the message is // Handle images first - might need to handle their `body`/`formatted_body` as well, which will fall through to the text processor @@ -632,21 +634,17 @@ async function eventToMessage(event, guild, channel, di) { pendingFiles.push({name: filename, mxc: event.content.url}) } else if (event.type === "org.matrix.msc3381.poll.start") { - content = "" const pollContent = event.content["org.matrix.msc3381.poll.start"] // just for convenience - let allowMultiselect = (pollContent.max_selections != 1) - let answers = pollContent.answers.map(answer=>{ - return {poll_media: {text: answer["org.matrix.msc1767.text"]}, matrix_option: answer["id"]} - }) - poll = { - question: { - text: event.content["org.matrix.msc3381.poll.start"].question["org.matrix.msc1767.text"] - }, - answers: answers, - duration: 768, // Maximum duration (32 days). Matrix doesn't allow automatically-expiring polls, so this is the only thing that makes sense to send. - allow_multiselect: allowMultiselect, - layout_type: 1 - } + const isClosed = false; + const maxSelections = pollContent.max_selections || 1 + const questionText = pollContent.question["org.matrix.msc1767.text"] + const pollOptions = pollContent.answers.map(answer => ({ + matrix_option: answer.id, + option_text: answer["org.matrix.msc1767.text"], + count: 0 // no votes initially + })) + content = "" + pollMessage = pollComponents.getPollComponents(isClosed, maxSelections, questionText, pollOptions) } else { // Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events. @@ -980,7 +978,7 @@ async function eventToMessage(event, guild, channel, di) { // Split into 2000 character chunks const chunks = chunk(content, 2000) - /** @type {({poll?: Ty.SendingPoll} & DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]})[]} */ + /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]})[]} */ const messages = chunks.map(content => ({ content, allowed_mentions: { @@ -1003,13 +1001,12 @@ async function eventToMessage(event, guild, channel, di) { messages[0].pendingFiles = pendingFiles } - if (poll) { - if (!messages.length) messages.push({ - content: " ", // stopgap, remove when library updates + if (pollMessage) { + messages.push({ + ...pollMessage, username: displayNameShortened, avatar_url: avatarURL }) - messages[0].poll = poll } const messagesToEdit = [] diff --git a/src/m2d/converters/poll-components.js b/src/m2d/converters/poll-components.js new file mode 100644 index 00000000..8aafa2b9 --- /dev/null +++ b/src/m2d/converters/poll-components.js @@ -0,0 +1,103 @@ +// @ts-check + +const DiscordTypes = require("discord-api-types/v10") + +/** + * @param {boolean} isClosed + * @param {{matrix_option: string, option_text: string, count: number}[]} pollOptions already sorted correctly + * @returns {DiscordTypes.APIMessageTopLevelComponent[]} +*/ +function optionsToComponents(isClosed, pollOptions) { + const topAnswers = pollOptions.toSorted((a, b) => b.count - a.count) + /** @type {DiscordTypes.APIMessageTopLevelComponent[]} */ + return pollOptions.map(option => { + const winningOrTied = option.count && topAnswers[0].count === option.count + return { + type: DiscordTypes.ComponentType.Container, + components: [{ + type: DiscordTypes.ComponentType.Section, + components: [{ + type: DiscordTypes.ComponentType.TextDisplay, + content: option.option_text + }], + accessory: { + type: DiscordTypes.ComponentType.Button, + style: winningOrTied ? DiscordTypes.ButtonStyle.Success : DiscordTypes.ButtonStyle.Secondary, + label: option.count.toString(), + custom_id: option.matrix_option, + disabled: isClosed + } + }] + } + }) +} + +/** + * @param {boolean} isClosed + * @param {number} maxSelections + * @param {string} questionText + * @param {{matrix_option: string, option_text: string, count: number}[]} pollOptions already sorted correctly + * @returns {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody} + */ +function getPollComponents(isClosed, maxSelections, questionText, pollOptions) { + /** @type {DiscordTypes.APIMessageTopLevelComponent} */ + let headingComponent + if (isClosed) { + const multiSelectString = + ( maxSelections === 1 ? "-# ~~Select one answer~~" + : maxSelections >= pollOptions.length ? "-# ~~Select one or more answers~~" + : `-# ~~Select up to ${maxSelections} answers~~`) + headingComponent = { // This one is for the poll heading. + type: DiscordTypes.ComponentType.Section, + components: [ + { + type: DiscordTypes.ComponentType.TextDisplay, + content: `## ${questionText}` + }, + { + type: DiscordTypes.ComponentType.TextDisplay, + content: multiSelectString + } + ], + accessory: { + type: DiscordTypes.ComponentType.Button, + style: DiscordTypes.ButtonStyle.Secondary, + custom_id: "vote", + label: "Voting closed!", + disabled: true + } + } + } + else { + const multiSelectString = + ( maxSelections === 1 ? "-# Select one answer" + : maxSelections >= pollOptions.length ? "-# Select one or more answers" + : `-# Select up to ${maxSelections} answers`) + headingComponent = { // This one is for the poll heading. + type: DiscordTypes.ComponentType.Section, + components: [ + { + type: DiscordTypes.ComponentType.TextDisplay, + content: `## ${questionText}` + }, + { + type: DiscordTypes.ComponentType.TextDisplay, + content: multiSelectString + } + ], + accessory: { + type: DiscordTypes.ComponentType.Button, + style: DiscordTypes.ButtonStyle.Primary, + custom_id: "vote", + label: "Vote!" + } + } + } + const optionComponents = optionsToComponents(isClosed, pollOptions) + return { + flags: DiscordTypes.MessageFlags.IsComponentsV2, + components: [headingComponent, ...optionComponents] + } +} + +module.exports.getPollComponents = getPollComponents \ No newline at end of file diff --git a/src/types.d.ts b/src/types.d.ts index 8f879ec0..f18116ef 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -83,10 +83,6 @@ export type WebhookAuthor = { id: string } -export type SendingPoll = DiscordTypes.RESTAPIPoll & { - answers: (DiscordTypes.APIBasePollAnswer & {matrix_option: string})[] -} - export type PkSystem = { id: string uuid: string From 90606d917664da93539ba821a0061eb0b1fd5078 Mon Sep 17 00:00:00 2001 From: Ellie Algase Date: Sun, 25 Jan 2026 07:27:59 -0600 Subject: [PATCH 089/153] Add full support for polls, both m2d and d2m. Mostly works, but a few edge-cases still need to be worked out. Co-authored-by: Cadence Ember --- docs/img/poll-star-avatar.png | Bin 0 -> 3654 bytes docs/img/poll_win.png | Bin 0 -> 6573 bytes src/d2m/actions/add-reaction.js | 2 +- src/d2m/actions/close-poll.js | 41 +++--- src/d2m/actions/remove-reaction.js | 2 +- src/d2m/actions/send-message.js | 22 ++- src/discord/interactions/poll.js | 150 +++++++++++++++++++++ src/discord/interactions/vote.js | 95 ------------- src/discord/register-interactions.js | 60 ++++++--- src/m2d/actions/send-event.js | 28 +++- src/m2d/actions/setup-emojis.js | 2 +- src/m2d/actions/vote.js | 32 ++++- src/m2d/converters/event-to-message.js | 34 +++-- src/m2d/converters/poll-components.js | 178 +++++++++++++++++++++---- src/m2d/event-dispatcher.js | 29 ++++ src/types.d.ts | 12 ++ src/web/server.js | 5 + 17 files changed, 501 insertions(+), 191 deletions(-) create mode 100644 docs/img/poll-star-avatar.png create mode 100644 docs/img/poll_win.png create mode 100644 src/discord/interactions/poll.js delete mode 100644 src/discord/interactions/vote.js diff --git a/docs/img/poll-star-avatar.png b/docs/img/poll-star-avatar.png new file mode 100644 index 0000000000000000000000000000000000000000..a4355557615dd35f4d099fa7908079d7b219b9f1 GIT binary patch literal 3654 zcmcgvdpML^7k_6&C)W-_GU}X6DYryoay^7`za}9HQ&Vn@VWeE*oJvGZjTtnEaZ5Oi z+cXTPP9ltEMxk6%G$xG7Wg275_x7EC&+~l$ed~GNXTQ&Ht-aP>d;ivY_RjM1a8i)f zlm!4l0qN}E0{{^47Xthw0}jEMY7{tZi*pW)2Y^JaA0MRZY0_s9+=)GY66+g_!s5zr1)&1$Lw;{@iYQh~+H=9_zvgk#EV|kq+ERCh`Nxd3KVAKg)m0lN3PEGlWS%f^Gs3u z>5jPbvc9}w8D#YCuXX*EYoS*^26NFnK;z7+qFj+dMNrDZx99Ps%1AHZysTkJ;>OZj z^gZ6+2Jyto1Q1e0)mo!6gC@u6>%+8d9E2_qF&R`&EH>oj2t~zL@dQcSiD%6kvcMSk^P^AYnnFQp@_5z;3yTSorFus zW60rCNt)XW5+fwf)w$!i*8Xy zo==#>`$QxLd05z(rb64NPQX36m^sE;GCPI>dYiwp5|l_Q{U#enZ1S!tO4sIO8SI^??#OyF^wqmfmo`)rBx=N#rY-^xbjj=g$i;ij_tA$O7dqcd;) zXs&^p8~E{rE7~SpHi06v0%VRFvZFYhQcI_mW-8J8KbGvoN0h@$R zl8`Xe#EzPNrdcvnq{XJo~6b}cr%cl2$8lQ{un<;hH-{Bt6vakEf zr3N0@1RV%S_B?I%4THsx^~eJXk*KcyHLB$D)Fs8uP6M0oaM1Ca#~xil_Y@?16P@&Ph9bSIOlaY zY`zK!yX-D^V&SqSqBpo4IJCc~q_h2na+keNEH+;$zRl*;pBO=?1NWvaB+LWW>Dk@p z!Z1ek8&~9Fgr-5A0%z=OujR;q>%lNy$%mmo46+s4oJ3CcGZ+1kyHLMBG z#dS7NFH+um?!Iu~hk~r2$Zo;Sjke%u4k_zcH4wMd|cmtvC{v&g$xH73u_6TN~@NSHw%R0R${)2~Tbm$+5bVIrLby_+i0%@M1UBgg2860`-k-)-F$ zsq%W`n;$QBm{i3BODSi#vv$2sJmwNq(UyN?LoKA8`#QJ&Zji61A(q#$V(dzyH$Pmr z9L$2+qE41wkP8c~2!8iIt?Lgx94z?N5e(IJcfb~dF0Y@Y(--2%0nbl}*e;A;=GTno zG%us=MtTP;FhVV23}NM({WhX@A8A;!87wknc8lod6FbhW~JOte{_970U@27vCwc+Q(x|PEMXNS+4G=ONWxKfuc9C z;DIC_KZlmK(Bmx}cM8~YC3?fhJ>L^+?S)_Yo$~J1^AF`za-F|7kf5od@x7^FqT*vo z^V9J-WKj0=5BYO@25-g#2^yN3>xBg=uJW7uhmfH#Ev=2^ADW%Ew`F8x>W@;RKeY5~ z9;FVguC2vnE3(~^R7S$MOfp$TMMVTpIgF6AX4ft*E~>|j{yDt}&djh+*L3snNc9FC ziX3YEo){w>OdbLk-t$YcndvI79PH`GGaOwls2@5kEUcygrrZ$t<}xkVn%dP>vg^j9 zZuwFdS}^~r@%N%@v&+KeGJ8`w*-{sDVe!voi2p~AfrH+F|6yudnMvt|SjVmB)l_W> zFK6Ed)E*}h%cM-@a?!Qeg@+|2H~Z5A1wOH{OEgOzKNt$|N3306&zsXSH^xaInO(`M zwhH6oBD@RBA}*K%2IS9eAN(C!Z5J9k1wp2%kfmNEcs}$>GW=)t%@KY0cRJ!jh^?tT z%$k}Sn=D0Ely16aU1(P-0*Rl8i*Z;rtw@xFpwrh95R&lP51Ws&hYUaT($y+1EIjD4 z-b&%kSuT$@Q|L;&t%LPdWp9)}Gu>+_#eDTl$)>}S*gx?rPZP;?;1=vZjS=jg!O-Xy zpGtPhmfDR^XkVVW8wu`!LCkUT4*n*_ka(;a&8g(lhPVFfU8MARqqeP7)Vo##+~=CA zHPV=)$+7+ko1eANr`US@&5i=Allp2UO3cihG5{5pl+<4CQ!^fgYIC`zz4Bs~@3{~8 zjR4Z^Mj3TakZw9cK}o3r$yu>R_LotI-%ny_8$n{Rc*$X_otYO%AT6!&>hcohW?h6U zED?@m2kEL#-c-y;QnkIUojmt&d0Dve5nm|$_!Osu#Q1|d;KSG60zI=J|p8*G?k_J%Jv*cuG70< z&_><#gH~U{}9?T5w!f0qG|aEB1x`bh`=u`48zl04WB*j$+_h<}hL>z`|1z!e<0=XFfzMhGd; zTZ=FTn=X_5df?P-<-Xx8kT8uWuiCQ1Sr(dqO|+@W{;`*3u{|4F?M(~b?4UejS)k^o zM?%xYvE}xAw1=R>XI|vOitswL&_SX37hiOt+R@m_4HTGQxQy40!J?x!_)1=t&XIRM zL6XDP){MJ<<>8E_NY|#PrxkG$JbmgQ8jUShNgAJsU7DYN_cLtE*l!Ti?$+pcn5qYu zmqrY_Zgoa~_kat3vx&m(pZ%5zo|KWprym#bf05ywRX{VMKi>f9={eLCY^cuAwly`; zD;U$O@Zs5}rAca8m*cLgcFQZs%RfFu4H_4}4bQ&z-pY{Ygjy5xc)TwD(o))f#CGeO z&}z=w;=3Qxb%5uHjAmVKVooR(j1j?J)5!|_qAgphm}k8{x0A!+2#}w%nCysP_6Vq4 zWZA{3)>4jRyrg6U5T(vtURQfpC{#6)Q)B_7@m!39wr=E$6i}S_a6y`#!+4%-R%D65w zgp;1J$>|}`ijbU~oPUYjHErg&m9zx129EsJHz#i}_h1fKhDm$DO^#V0?Bz}=!@?LK zXChGJoXMmcWl{q;a4y?a+?YGrcfDuY-Zc~C*XQ~9E63S+W%13VMHb}KG1*cXaC1O! zlm)&-8+I1-aN!k;bt#XcULM~JH4%vz42BfT;=h(wR#qzbP1^@@q?$~0Vd@dY7BDZ^ zm16Nc%c5}fcX}vp;el)h*ur(yvfJpQCD%s|=g^0@!lS{cqN%w(=C`2LzF^O4o9Nub z`#PuZUb?_fz;eg`JB(^oX>bC+1&I79rYz@008KW4D~Gm00K5403|v2 za140u4L+#+4ebK~VB^}w5AslQ1Z)6YsOxqpD?e{kh+BXc5E2q1bJzENpog2kmyBP4 z&x=j|UyTTWntE-a=N^_`1kqES}>@;06w#v}gTxXf`^F$(W1$yzE_UMvqfcVIc@HL*j20@#sLh`d zGw&_;Mc$1!b_TM4wKrUP)m3>$#dr6}r%HViN=a#K{WrDms4;2u6;uf?tQm{$tq6nw zuq_8{3FLO}VdHkYxV#^fb*OG2f$c?gt11uhp23)urPo@HM8n~KD_XIe%SmTGq)l#*=~ z;nyxZ_AViTNF9V(aR*J0)y6Kf&#lo0yjzb6jW!@$&5-u=B*@)FCzSj&dYjwCE;HGi zef#h2b6dpC**lOQRKm=+ly*1lSxc*2O~_o4J_;u6`tN@1l28N3$}`R0<*>)G?C5dA z;Tz}ROD>T4EvS+{!c6-u-2WBJ%ahRXsbHeqV-p&zXwF+rCGwCnKk{PJFS6d0{)q(;>_6|?EDyHl7FwULYye4Qwg7- zJKRVKcuz0|ddnLgO*{1E=RW`3p^q>$zYF<+8JtgHRzYg{=vJWE;f!@-?=R0ZeU@<$ zEfT)d2(+P@sYaVh+L@Qsfcd`DRu1%cbNdnY(>(e!bIy(1U)TS9 zW%Dtnkk=v5VD6ZHzTRg??}|pT=c+^vmsQ&7t=U$@8`1%p!Nc{%nu zx?IKbcth$35LV%8UlD)i?y%{mT|TDOFJ+! z3X54DuGD03D&=XeR;Fo|Y^fN~UaEl8E<(|>(bCR)ic0zzuMZ{0EyBc}TZK`v@WpK& zR@bg<0N^912fZW)}0{*fu zeaEBGGe94aG(%kjxF3#uHF1ejTsRU^C>Xm(coCZ0b@hf&BtE@B`gcWqXsN(6AjiM#QRbW z+D<0sD|Q|?`sE&BkCgNF>gaoe#+j=%S@cg+wQ&HYzKp|OX}rtu?`4H^M}&S2)E=n! z45LHW<5hE@fLtEOrz&+9gn%zY0Ct;_FDBAQdv&gu zh_K|_YYTV-8b8yy^Me}vbp4KPx-(N0c-uX0O)ZwkM?TXY1Kt;Yoi5LClT0sZ z(8CJR)D_l=jt%2(9(jO%ho;bBr!u_bk#wF1zgR^O|YwuseK}+YF?_0ZeiHzZZ!x|Ppu0A=xITw zpS$@@efXj-7Z(NCCdTsv>D>OT#jP!nMXo@%;-^8QlZS=7@_50+{voJ9OEwrnNosop zmYo;F4U)$N<2JVNR_?h7b&ygyF<#A$ib%o-_r`J$|>Z@OdatX+F^E0)G2}AX$ zhs+=U+j9Qn|09+nB3P=VaQ43Qnj5%jPiu&$)J#i)^@1e760_NtD-%LRu#-e8%EpIi&*Xjs!5&mgKcJwXhnNr0ZDLQ(A8n)cG; z!TY|4UfW%VJMFV|t1ROgzcsG@;!&vt!pBrk?tDC{^|yIexbZEQABg@CULZSo&^@WK zo~EUX=<^+5MW+&oc&;2R3G%d_r$z^2=>w6fXKF9Bupg_757%a+hYYYz$^px4edj!7rK{{d!e1zAUeG z)$Xhvoh@hKse%>DfZ*>!Q5`7ywW5)y**iz&FQRx!t9(`cNd8VDI8nb4rT2W(9eoHZ zn}!QCYS*v(wFK9@_%1z^_RjpTmlak(0xJ+0{0K$9mUbbC*-5;{QHa7?p-LIxDDv^N z8F<34PM$Gy6tcEB-Ncs5@QISNM1g)Qh;$aBMx&XDv~MEb7#plk()j9H^p&Wzz(acd z7L()+5inOn2wKh@5jWy=AdmPn9?OnBS1hNqnX%zlyp$?C-Mtv3WDX|hFzO4ZNRYpC6Q{{z8u_0gcaPr=)O;|>`Uw^OfLnE8Iz zQ*;<}No{w%GZbPswj%u+;X_)IM;vK*+SeSl$~prG7-hmv{NPeOO;>IFqMiw(*57`W z#o030aEnT1l}3?DIV(en7EROIIPrbe1Iai1#rE6x&pyh72@&}Y7rOt)xR2oo3rk|* z=Ikl%_^h*#2{V6k_daWr%%vqAwuyrJrwoLLlbAXP(y!v54 zqhs*5XbLT$Fktp!=)b2|HeVlAPWsgrDWTrdpQ#HO7$f3CApp-1G~X#0zEY7Fhp3Me zWUNfnSj-QW2g{aHtL*HJPD8rdz0zR+~UE9dF8n2_USn}J;IPjb8@ zOGg4N(D&sQk=DtLNzgvn3wORy&YZ8lVIrk&OVP#ZZcCcalLZ1)41EYGR`lE{ZWJL~ z$+)}{yT=jZS<~{iPI)J%|$wa`B$WwGMc_>}X zugEX#KfHrDqRW2pyZu;?$hZQ>PCcvSWIVr)`m(q6@z?0X(|OLfTvyq<3b!>D$-8`y z)_z1?M6HXH4xrH=@LY)d$8&kxJLzVTs)spD3whd~>gqeAr(d~j3>Pr1cQC;L*CG@e zH@95Ay{e_T`EBRblWkws?cCMT9rSqfU-qMy?Weuog?k39fc!!hZh}pV)XV&q^i8sp zcQI8XcGUDWHp->A=6C zqBBDxZ*V;HQLIWG69aCI+W$ZeVBr&G7PQJJ) zJkh;XW!uuZkQCwaL=3F!SG|aot#kNJIu^dPlQxk}!W2YHQ*%Ch^DyGc-4Cco-^av( zvh{Ky)ZnJh)2%d}t}gr6+|il>f9;8s?OayCv-|IIa1&0)&h$r7H1(z@r*5q^^6Q9) z#a$_FT4a~i?01-e9S)IT+b%u|%nsh+A%30WOp9qLAAeBvDr?N_o6%IE*#PjbyY%rJE7bzjR z(I78kY72DeaB&(sq$T~Cq4ELkuLKw82SfYcrX_gY{RY?ov#3ePsyLA+THZ?)Gj$fZ zDta@NY<2<66M&_pb~{9TNjVZ+qh=Pzb317N~ zdkI`ct`Bk4s*nfun$3%d8X(K{P<;gG(!Vbc`cSo~^vKJGdv%3zOTsps6#->+4M0yXDHIdC~$<!%5&0iZuszFH|nAxY!yZs3oh1HYO zm_Puq@;DYikCq3DeU60_L-%GX^ia114uRkXDM<*nrPf;F0}|oBBaIhqyKD}haQ@ap zuCJAiE1m9SJi;~2JqsxhL|w;#e60|LfVjq^=qofIZ(AVZRPpj{68>=P!8-?SQWTNL zIEvbXef+;;eT0E6yaoWUT=OBC+A+_-nM^xIRGY?Y%dLJ~&V*x?QyW~{sL@#r_U5He zAA8+YK8WQn5zYjU0MR~87C5Q`OGn?n{gUA@dYSd zgnkHF8_;srtTx-D0wG)X6|oV^kCklg40>@C4qPqvQRki>|*^7G_J(L zz8H%ysN}e0DwWEWPpRdBM^#Ij@LBHIp<=5r*`(U1xH9NkF{k;9J{J1u?v|9mgpsr4FjaejSXc7KFCW05DRd`$7DZLPlfxFhL2xbv=X}Ul5P;STZ6?jY!dwWd2;^@IPHK z|9D$m35nL2d^*f@@=3aj%q;~R3-kz?p5L?z{gql8v91qN+V+Pa(8(762*pO>?Q>cp zicmx^Hy)Cs)yZniC^tFJGWT17St_ScGMX zEO9@M&NlP81KDsFtsbS^WC8wxB%pJy?-5O5+s{QLh*HG<0*C#{DoqALk<4fg5YQ}k z@YIJk(x1hEz^C`3zULiOg)NCF_^v$^c_ysA^nDKJ9)uP=vj9*l*J@SNwl?icX(cX{s=H0qW@f()m9L9|FV&Yhyh#k4en;WK+(_A>*T`d{ zIJGligSVhgW=EU32UZ(&uW-->Uj`c%us2L?WG element.name === "total_votes").value // We could do [2], but best not to rely on the ordering staying consistent. + const totalVotes = +pollCloseObject.fields.find(element => element.name === "total_votes").value // We could do [2], but best not to rely on the ordering staying consistent. const databaseVotes = select("poll_vote", ["discord_or_matrix_user_id", "matrix_option"], {message_id: pollMessageID}, " AND discord_or_matrix_user_id NOT LIKE '@%'").all() @@ -120,26 +124,27 @@ async function closePoll(closeMessage, guild){ })) } - /** @type {{discord_option: string, option_text: string, count: number}[]} */ - const pollResults = db.prepare("SELECT discord_option, option_text, count(*) as count FROM poll_option INNER JOIN poll_vote USING (message_id, matrix_option) GROUP BY discord_option").all() + /** @type {{matrix_option: string, option_text: string, count: number}[]} */ + const pollResults = db.prepare("SELECT matrix_option, option_text, seq, count(discord_or_matrix_user_id) as count FROM poll_option LEFT JOIN poll_vote USING (message_id, matrix_option) WHERE message_id = ? GROUP BY matrix_option ORDER BY seq").all(pollMessageID) const combinedVotes = pollResults.reduce((a, c) => a + c.count, 0) + const totalVoters = db.prepare("SELECT count(DISTINCT discord_or_matrix_user_id) as count FROM poll_vote WHERE message_id = ?").pluck().get(pollMessageID) if (combinedVotes !== totalVotes) { // This means some votes were cast on Matrix! - const message = await discord.snow.channel.getChannelMessage(closeMessage.channel_id, pollMessageID) - assert(message?.poll?.answers) // Now that we've corrected the vote totals, we can get the results again and post them to Discord! - const topAnswers = pollResults.toSorted() - const unique = topAnswers.length > 1 && topAnswers[0].count === topAnswers[1].count - - let messageString = "📶 Results including Matrix votes\n" - for (const result of pollResults) { - if (result === topAnswers[0] && unique) { - messageString = messageString + `${barChart(result.count/combinedVotes)} **${result.option_text}** (**${result.count}**)\n` - } else { - messageString = messageString + `${barChart(result.count/combinedVotes)} ${result.option_text} (${result.count})\n` - } + const topAnswers = pollResults.toSorted((a, b) => b.count - a.count) + let messageString = "" + for (const option of pollResults) { + const medal = pollComponents.getMedal(topAnswers, option.count) + const countString = `${String(option.count).padStart(String(topAnswers[0].count).length)}` + const votesString = option.count === 1 ? "vote " : "votes" + const label = medal === "🥇" ? `**${option.option_text}**` : option.option_text + messageString += `\`\u200b${countString} ${votesString}\u200b\` ${barChart(option.count/totalVoters)} ${label} ${medal}\n` + } + return { + username: "Total results including Matrix votes", + avatar_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png`, + content: messageString } - await channelWebhook.sendMessageWithWebhook(closeMessage.channel_id, {content: messageString}, closeMessage.thread_id) } } diff --git a/src/d2m/actions/remove-reaction.js b/src/d2m/actions/remove-reaction.js index 0f7eec9c..af7fd6ab 100644 --- a/src/d2m/actions/remove-reaction.js +++ b/src/d2m/actions/remove-reaction.js @@ -22,7 +22,7 @@ async function removeSomeReactions(data) { if (!row) return const eventReactedTo = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") - .where({message_id: data.message_id, reaction_part: 0}).select("event_id", "room_id").get() + .where({message_id: data.message_id}).and("ORDER BY reaction_part").select("event_id", "room_id").get() if (!eventReactedTo) return // Due to server restrictions, all relations (i.e. reactions) have to be in the same room as the original event. diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index 1e227b56..b8b0cdaf 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -4,7 +4,7 @@ const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") -const { discord, sync, db, select } = passthrough +const { discord, sync, db, select, from} = passthrough /** @type {import("../converters/message-to-event")} */ const messageToEvent = sync.require("../converters/message-to-event") /** @type {import("../../matrix/api")} */ @@ -21,6 +21,8 @@ const createRoom = sync.require("../actions/create-room") const closePoll = sync.require("../actions/close-poll") /** @type {import("../../discord/utils")} */ const dUtils = sync.require("../../discord/utils") +/** @type {import("../../m2d/actions/channel-webhook")} */ +const channelWebhook = sync.require("../../m2d/actions/channel-webhook") /** * @param {DiscordTypes.GatewayMessageCreateDispatchData} message @@ -33,10 +35,6 @@ async function sendMessage(message, channel, guild, row) { const historicalRoomIndex = select("historical_channel_room", "historical_room_index", {room_id: roomID}).pluck().get() assert(historicalRoomIndex) - if (message.type === 46) { // This is a poll_result. We might need to send a message to Discord (if there were any Matrix-side votes), regardless of if this message was sent by the bridge or not. - await closePoll.closePoll(message, guild) - } - let senderMxid = null if (dUtils.isWebhookMessage(message)) { const useWebhookProfile = select("guild_space", "webhook_profile", {guild_id: guild.id}).pluck().get() ?? 0 @@ -104,6 +102,20 @@ async function sendMessage(message, channel, guild, row) { })() } + if (message.type === DiscordTypes.MessageType.PollResult) { // We might need to send a message to Discord (if there were any Matrix-side votes). + const detailedResultsMessage = await closePoll.closePoll(message, guild) + if (detailedResultsMessage) { + const threadParent = select("channel_room", "thread_parent", {channel_id: message.channel_id}).pluck().get() + const channelID = threadParent ? threadParent : message.channel_id + const threadID = threadParent ? message.channel_id : undefined + const sentResultsMessage = await channelWebhook.sendMessageWithWebhook(channelID, detailedResultsMessage, threadID) + db.transaction(() => { + db.prepare("UPDATE event_message SET reaction_part = 1 WHERE event_id = ?").run(eventID) + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, sentResultsMessage.id, 1, 0) // part = 1, reaction_part = 0 + })() + } + } + eventIDs.push(eventID) } diff --git a/src/discord/interactions/poll.js b/src/discord/interactions/poll.js new file mode 100644 index 00000000..94ecb4ca --- /dev/null +++ b/src/discord/interactions/poll.js @@ -0,0 +1,150 @@ +// @ts-check + +const DiscordTypes = require("discord-api-types/v10") +const {discord, sync, select, from, db} = require("../../passthrough") +const assert = require("assert/strict") +const {id: botID} = require("../../../addbot") +const {InteractionMethods} = require("snowtransfer") + +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") +/** @type {import("../../m2d/converters/poll-components")} */ +const pollComponents = sync.require("../../m2d/converters/poll-components") +/** @type {import("../../d2m/actions/add-or-remove-vote")} */ +const vote = sync.require("../../d2m/actions/add-or-remove-vote") + +/** + * @param {DiscordTypes.APIMessageComponentButtonInteraction} interaction + * @param {{api: typeof api}} di + * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} + */ +async function* _interact({data, message, member, user}, {api}) { + if (!member?.user) return + const userID = member.user.id + + const pollRow = select("poll", ["question_text", "max_selections"], {message_id: message.id}).get() + if (!pollRow) return + + // Definitely supposed to be a poll button click. We can use assertions now. + + const matrixPollEvent = select("event_message", "event_id", {message_id: message.id}).pluck().get() + assert(matrixPollEvent) + + const maxSelections = pollRow.max_selections + const alreadySelected = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: userID, message_id: message.id}).pluck().all() + + // Show modal (if no capacity or if requested) + if (data.custom_id === "POLL_VOTE" || (maxSelections > 1 && alreadySelected.length === maxSelections)) { + const options = select("poll_option", ["matrix_option", "option_text", "seq"], {message_id: message.id}, "ORDER BY seq").all().map(option => ({ + value: option.matrix_option, + label: option.option_text, + default: alreadySelected.includes(option.matrix_option) + })) + const checkboxGroupExtras = maxSelections === 1 && options.length > 1 ? {} : { + type: 22, // DiscordTypes.ComponentType.CheckboxGroup + min_values: 0, + max_values: maxSelections + } + return yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.Modal, + data: { + custom_id: "POLL_MODAL", + title: "Poll", + components: [{ + type: DiscordTypes.ComponentType.TextDisplay, + content: `-# ${pollComponents.getMultiSelectString(pollRow.max_selections, options.length)}` + }, { + type: DiscordTypes.ComponentType.Label, + label: pollRow.question_text, + component: /* { + type: 21, // DiscordTypes.ComponentType.RadioGroup + custom_id: "POLL_MODAL_SELECTION", + options, + required: false, + ...checkboxGroupExtras + } */ + { + type: DiscordTypes.ComponentType.StringSelect, + custom_id: "POLL_MODAL_SELECTION", + options, + required: false, + min_values: 0, + max_values: maxSelections, + } + }] + } + }} + } + + if (data.custom_id === "POLL_MODAL") { + // Clicked options via modal + /** @type {DiscordTypes.APIMessageStringSelectInteractionData} */ // @ts-ignore - close enough to the real thing + const component = data.components[1].component + assert.equal(component.custom_id, "POLL_MODAL_SELECTION") + + // Replace votes with selection + db.transaction(() => { + db.prepare("DELETE FROM poll_vote WHERE message_id = ? AND discord_or_matrix_user_id = ?").run(message.id, userID) + for (const option of component.values) { + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, option) + } + })() + + // Update counts on message + yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.UpdateMessage, + data: pollComponents.getPollComponentsFromDatabase(message.id) + }} + + // Sync changes to Matrix + await vote.sendVotes(member.user, message.channel_id, message.id, matrixPollEvent) + } else { + // Clicked buttons on message + const optionPrefix = "POLL_OPTION#" // we use a prefix to prevent someone from sending a Matrix poll that intentionally collides with other elements of the embed + const matrixOption = select("poll_option", "matrix_option", {matrix_option: data.custom_id.substring(optionPrefix.length), message_id: message.id}).pluck().get() + assert(matrixOption) + + // Remove a vote + if (alreadySelected.includes(matrixOption)) { + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ? AND matrix_option = ?").run(userID, message.id, matrixOption) + } + // Replace votes (if only one selection is allowed) + else if (maxSelections === 1 && alreadySelected.length === 1) { + db.transaction(() => { + db.prepare("DELETE FROM poll_vote WHERE message_id = ? AND discord_or_matrix_user_id = ?").run(message.id, userID) + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, matrixOption) + })() + } + // Add a vote (if capacity) + else if (alreadySelected.length < maxSelections) { + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, matrixOption) + } + + // Update counts on message + yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.UpdateMessage, + data: pollComponents.getPollComponentsFromDatabase(message.id) + }} + + // Sync changes to Matrix + await vote.sendVotes(member.user, message.channel_id, message.id, matrixPollEvent) + } +} + +/* c8 ignore start */ + +/** @param {DiscordTypes.APIMessageComponentButtonInteraction} interaction */ +async function interact(interaction) { + for await (const response of _interact(interaction, {api})) { + if (response.createInteractionResponse) { + await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, response.createInteractionResponse) + } else if (response.editOriginalInteractionResponse) { + await discord.snow.interaction.editOriginalInteractionResponse(botID, interaction.token, response.editOriginalInteractionResponse) + } + } +} + +module.exports.interact = interact +module.exports._interact = _interact diff --git a/src/discord/interactions/vote.js b/src/discord/interactions/vote.js deleted file mode 100644 index 91ba97e7..00000000 --- a/src/discord/interactions/vote.js +++ /dev/null @@ -1,95 +0,0 @@ -// @ts-check - -const DiscordTypes = require("discord-api-types/v10") -const {discord, sync, select, from, db} = require("../../passthrough") -const assert = require("assert/strict") -const {id: botID} = require("../../../addbot") -const {InteractionMethods} = require("snowtransfer") - -/** @type {import("../../matrix/api")} */ -const api = sync.require("../../matrix/api") -/** @type {import("../../matrix/utils")} */ -const utils = sync.require("../../matrix/utils") -/** @type {import("../../m2d/converters/poll-components")} */ -const pollComponents = sync.require("../../m2d/converters/poll-components") -/** @type {import("../../d2m/actions/add-or-remove-vote")} */ -const vote = sync.require("../../d2m/actions/add-or-remove-vote") - -/** - * @param {DiscordTypes.APIMessageComponentButtonInteraction} interaction - * @param {{api: typeof api}} di - * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} - */ -async function* _interact({data, message, member, user}, {api}) { - const discordUser = member?.user || user - assert(discordUser) - const userID = discordUser.id - - const matrixPollEvent = select("event_message", "event_id", {message_id: message.id}).pluck().get() - assert(matrixPollEvent) - - const matrixOption = select("poll_option", "matrix_option", {discord_option: data.custom_id, message_id: message.id}).pluck().get() - assert(matrixOption) - - const pollRow = select("poll", ["question_text", "max_selections"], {message_id: message.id}).get() - assert(pollRow) - const maxSelections = pollRow.max_selections - const alreadySelected = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: userID, message_id: message.id}).pluck().all() - - // Show modal (if no capacity) - if (maxSelections > 1 && alreadySelected.length === maxSelections) { - // TODO: show modal - return - } - - // We are going to do a server operation so need to show loading state - yield {createInteractionResponse: { - type: DiscordTypes.InteractionResponseType.DeferredMessageUpdate, - }} - - // Remove a vote - if (alreadySelected.includes(data.custom_id)) { - db.prepare("DELETE FROM poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, data.custom_id) - } - // Replace votes (if only one selection is allowed) - else if (maxSelections === 1 && alreadySelected.length === 1) { - db.transaction(() => { - db.prepare("DELETE FROM poll_vote WHERE message_id = ? AND discord_or_matrix_user_id = ?").run(message.id, userID) - db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, data.custom_id) - })() - } - // Add a vote (if capacity) - else if (alreadySelected.length < maxSelections) { - db.transaction(() => { - db.prepare("DELETE FROM poll_vote WHERE message_id = ? AND discord_or_matrix_user_id = ?").run(message.id, userID) - db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, data.custom_id) - })() - } - - // Sync changes to Matrix - await vote.sendVotes(discordUser, message.channel_id, message.id, matrixPollEvent) - - // Check the poll is not closed (it may have been closed by sendVotes if we discover we can't send) - const isClosed = select("poll", "is_closed", {message_id: message.id}).pluck().get() - - /** @type {{matrix_option: string, option_text: string, count: number}[]} */ - const pollResults = db.prepare("SELECT matrix_option, option_text, count(*) as count FROM poll_option INNER JOIN poll_vote USING (message_id, matrix_option) GROUP BY matrix_option").all() - return yield {createInteractionResponse: { - type: DiscordTypes.InteractionResponseType.UpdateMessage, - data: pollComponents.getPollComponents(!!isClosed, maxSelections, pollRow.question_text, pollResults) - }} -} - -/* c8 ignore start */ - -/** @param {DiscordTypes.APIMessageComponentButtonInteraction} interaction */ -async function interact(interaction) { - for await (const response of _interact(interaction, {api})) { - if (response.createInteractionResponse) { - await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, response.createInteractionResponse) - } - } -} - -module.exports.interact = interact -module.exports._interact = _interact diff --git a/src/discord/register-interactions.js b/src/discord/register-interactions.js index 0c5a0b06..63b04b0c 100644 --- a/src/discord/register-interactions.js +++ b/src/discord/register-interactions.js @@ -9,6 +9,7 @@ const invite = sync.require("./interactions/invite.js") const permissions = sync.require("./interactions/permissions.js") const reactions = sync.require("./interactions/reactions.js") const privacy = sync.require("./interactions/privacy.js") +const poll = sync.require("./interactions/poll.js") // User must have EVERY permission in default_member_permissions to be able to use the command @@ -68,25 +69,36 @@ discord.snow.interaction.bulkOverwriteApplicationCommands(id, [{ console.error(e) }) +/** @param {DiscordTypes.APIInteraction} interaction */ async function dispatchInteraction(interaction) { - const interactionId = interaction.data.custom_id || interaction.data.name + const interactionId = interaction.data?.["custom_id"] || interaction.data?.["name"] try { - if (interactionId === "Matrix info") { - await matrixInfo.interact(interaction) - } else if (interactionId === "invite") { - await invite.interact(interaction) - } else if (interactionId === "invite_channel") { - await invite.interactButton(interaction) - } else if (interactionId === "Permissions") { - await permissions.interact(interaction) - } else if (interactionId === "permissions_edit") { - await permissions.interactEdit(interaction) - } else if (interactionId === "Reactions") { - await reactions.interact(interaction) - } else if (interactionId === "privacy") { - await privacy.interact(interaction) + if (interaction.type === DiscordTypes.InteractionType.MessageComponent || interaction.type === DiscordTypes.InteractionType.ModalSubmit) { + // All we get is custom_id, don't know which context the button was clicked in. + // So we namespace these ourselves in the custom_id. Currently the only existing namespace is POLL_. + if (interaction.data.custom_id.startsWith("POLL_")) { + await poll.interact(interaction) + } else { + throw new Error(`Unknown message component ${interaction.data.custom_id}`) + } } else { - throw new Error(`Unknown interaction ${interactionId}`) + if (interactionId === "Matrix info") { + await matrixInfo.interact(interaction) + } else if (interactionId === "invite") { + await invite.interact(interaction) + } else if (interactionId === "invite_channel") { + await invite.interactButton(interaction) + } else if (interactionId === "Permissions") { + await permissions.interact(interaction) + } else if (interactionId === "permissions_edit") { + await permissions.interactEdit(interaction) + } else if (interactionId === "Reactions") { + await reactions.interact(interaction) + } else if (interactionId === "privacy") { + await privacy.interact(interaction) + } else { + throw new Error(`Unknown interaction ${interactionId}`) + } } } catch (e) { let stackLines = null @@ -97,12 +109,16 @@ async function dispatchInteraction(interaction) { stackLines = stackLines.slice(0, cloudstormLine - 2) } } - await discord.snow.interaction.createFollowupMessage(id, interaction.token, { - content: `Interaction failed: **${interactionId}**` - + `\nError trace:\n\`\`\`\n${stackLines.join("\n")}\`\`\`` - + `Interaction data:\n\`\`\`\n${JSON.stringify(interaction.data, null, 2)}\`\`\``, - flags: DiscordTypes.MessageFlags.Ephemeral - }) + try { + await discord.snow.interaction.createFollowupMessage(id, interaction.token, { + content: `Interaction failed: **${interactionId}**` + + `\nError trace:\n\`\`\`\n${stackLines.join("\n")}\`\`\`` + + `Interaction data:\n\`\`\`\n${JSON.stringify(interaction.data, null, 2)}\`\`\``, + flags: DiscordTypes.MessageFlags.Ephemeral + }) + } catch (_) { + throw e + } } } diff --git a/src/m2d/actions/send-event.js b/src/m2d/actions/send-event.js index f18385fa..00557a11 100644 --- a/src/m2d/actions/send-event.js +++ b/src/m2d/actions/send-event.js @@ -14,6 +14,8 @@ const channelWebhook = sync.require("./channel-webhook") const eventToMessage = sync.require("../converters/event-to-message") /** @type {import("../../matrix/api")}) */ const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/utils")}) */ +const utils = sync.require("../../matrix/utils") /** @type {import("../../d2m/actions/register-user")} */ const registerUser = sync.require("../../d2m/actions/register-user") /** @type {import("../../d2m/actions/edit-message")} */ @@ -59,7 +61,7 @@ async function resolvePendingFiles(message) { return newMessage } -/** @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start} event */ +/** @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_End} event */ async function sendEvent(event) { const row = from("channel_room").where({room_id: event.room_id}).select("channel_id", "thread_parent").get() if (!row) return [] // allow the bot to exist in unbridged rooms, just don't do anything with it @@ -79,7 +81,19 @@ async function sendEvent(event) { // no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it - let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, channel, {api, snow: discord.snow, mxcDownloader: emojiSheet.getAndConvertEmoji}) + const di = {api, snow: discord.snow, mxcDownloader: emojiSheet.getAndConvertEmoji} + + if (event.type === "org.matrix.msc3381.poll.end") { + // Validity already checked by dispatcher. Poll is definitely closed. Update it and DI necessary data. + const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id, event_type: "org.matrix.msc3381.poll.start", source: 0}).pluck().get() + assert(messageID) + db.prepare("UPDATE poll SET is_closed = 1 WHERE message_id = ?").run(messageID) + di.pollEnd = { + messageID + } + } + + let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, channel, di) messagesToEdit = await Promise.all(messagesToEdit.map(async e => { e.message = await resolvePendingFiles(e.message) @@ -105,8 +119,16 @@ async function sendEvent(event) { await channelWebhook.deleteMessageWithWebhook(channelID, id, threadID) } + // Poll ends do not follow the normal laws of parts. + // Normally when editing and adding extra parts, the new parts should always have part = 1 and reaction_part = 1 (because the existing part, which is being edited, already took 0). + // However for polls, the edit is actually for a different message. The message being sent is truly a new message, and should have parts = 0. + // So in that case, just override these variables to have the right values. + if (di.pollEnd) { + eventPart = 0 + } + for (const message of messagesToSend) { - const reactionPart = messagesToEdit.length === 0 && message === messagesToSend[messagesToSend.length - 1] ? 0 : 1 + const reactionPart = (messagesToEdit.length === 0 || di.pollEnd) && message === messagesToSend[messagesToSend.length - 1] ? 0 : 1 const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID) db.transaction(() => { db.prepare("INSERT INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(messageResponse.id, historicalRoomIndex) diff --git a/src/m2d/actions/setup-emojis.js b/src/m2d/actions/setup-emojis.js index 1be1d2d2..46641359 100644 --- a/src/m2d/actions/setup-emojis.js +++ b/src/m2d/actions/setup-emojis.js @@ -9,7 +9,7 @@ async function setupEmojis() { const {id} = require("../../../addbot") const {discord, db} = passthrough const emojis = await discord.snow.assets.getAppEmojis(id) - for (const name of ["L1", "L2"]) { + for (const name of ["L1", "L2", "poll_win"]) { const existing = emojis.items.find(e => e.name === name) if (existing) { db.prepare("REPLACE INTO auto_emoji (name, emoji_id) VALUES (?, ?)").run(existing.name, existing.id) diff --git a/src/m2d/actions/vote.js b/src/m2d/actions/vote.js index 5bb5cd43..926b9572 100644 --- a/src/m2d/actions/vote.js +++ b/src/m2d/actions/vote.js @@ -8,17 +8,35 @@ const crypto = require("crypto") const passthrough = require("../../passthrough") const {sync, discord, db, select} = passthrough +const {reg} = require("../../matrix/read-registration") +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") +/** @type {import("../converters/poll-components")} */ +const pollComponents = sync.require("../converters/poll-components") +/** @type {import("./channel-webhook")} */ +const webhook = sync.require("./channel-webhook") + /** @param {Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Response} event */ async function updateVote(event) { - - const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id, event_type: "org.matrix.msc3381.poll.start"}).pluck().get() + const messageRow = select("event_message", ["message_id", "source"], {event_id: event.content["m.relates_to"].event_id, event_type: "org.matrix.msc3381.poll.start"}).get() + const messageID = messageRow?.message_id if (!messageID) return // Nothing can be done if the parent message was never bridged. - db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(event.sender, messageID) // Clear all the existing votes, since this overwrites. Technically we could check and only overwrite the changes, but the complexity isn't worth it. + db.transaction(() => { + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(event.sender, messageID) // Clear all the existing votes, since this overwrites. + for (const answer of event.content["org.matrix.msc3381.poll.response"].answers) { + db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(event.sender, messageID, answer) + } + })() - event.content["org.matrix.msc3381.poll.response"].answers.map(answer=>{ - db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(event.sender, messageID, answer) - }) + // If poll was started on Matrix, the Discord version is using components, so we can update that to the current status + if (messageRow.source === 0) { + const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get() + assert(channelID) + await webhook.editMessageWithWebhook(channelID, messageID, pollComponents.getPollComponentsFromDatabase(messageID)) + } } -module.exports.updateVote = updateVote +module.exports.updateVote = updateVote \ No newline at end of file diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index ab53d082..b03de95a 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -519,10 +519,10 @@ async function getL1L2ReplyLine(called = false) { } /** - * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start} event + * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_End} event * @param {DiscordTypes.APIGuild} guild * @param {DiscordTypes.APIGuildTextChannel} channel - * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, mxcDownloader: (mxc: string) => Promise}} di simple-as-nails dependency injection for the matrix API + * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, mxcDownloader: (mxc: string) => Promise, pollEnd?: {messageID: string}}} di simple-as-nails dependency injection for the matrix API */ async function eventToMessage(event, guild, channel, di) { let displayName = event.sender @@ -553,8 +553,8 @@ async function eventToMessage(event, guild, channel, di) { const pendingFiles = [] /** @type {DiscordTypes.APIUser[]} */ const ensureJoined = [] - /** @type {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody?} */ - let pollMessage = null + /** @type {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody[]} */ + const pollMessages = [] // Convert content depending on what the message is // Handle images first - might need to handle their `body`/`formatted_body` as well, which will fall through to the text processor @@ -644,7 +644,17 @@ async function eventToMessage(event, guild, channel, di) { count: 0 // no votes initially })) content = "" - pollMessage = pollComponents.getPollComponents(isClosed, maxSelections, questionText, pollOptions) + pollMessages.push(pollComponents.getPollComponents(isClosed, maxSelections, questionText, pollOptions)) + + } else if (event.type === "org.matrix.msc3381.poll.end") { + assert(di.pollEnd) + content = "" + messageIDsToEdit.push(di.pollEnd.messageID) + pollMessages.push(pollComponents.getPollComponentsFromDatabase(di.pollEnd.messageID)) + pollMessages.push({ + ...await pollComponents.getPollEndMessageFromDatabase(channel.id, di.pollEnd.messageID), + avatar_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png` + }) } else { // Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events. @@ -1001,12 +1011,14 @@ async function eventToMessage(event, guild, channel, di) { messages[0].pendingFiles = pendingFiles } - if (pollMessage) { - messages.push({ - ...pollMessage, - username: displayNameShortened, - avatar_url: avatarURL - }) + if (pollMessages.length) { + for (const pollMessage of pollMessages) { + messages.push({ + username: displayNameShortened, + avatar_url: avatarURL, + ...pollMessage, + }) + } } const messagesToEdit = [] diff --git a/src/m2d/converters/poll-components.js b/src/m2d/converters/poll-components.js index 8aafa2b9..a8233e08 100644 --- a/src/m2d/converters/poll-components.js +++ b/src/m2d/converters/poll-components.js @@ -1,6 +1,28 @@ // @ts-check +const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") +const {sync, db, discord, select, from} = require("../../passthrough") + +/** @type {import("../actions/setup-emojis")} */ +const setupEmojis = sync.require("../actions/setup-emojis") + +/** + * @param {{count: number}[]} topAnswers + * @param {number} count + * @returns {string} + */ +function getMedal(topAnswers, count) { + const winningOrTied = count && topAnswers[0].count === count + const secondOrTied = !winningOrTied && count && topAnswers[1]?.count === count && topAnswers.slice(-1)[0].count !== count + const thirdOrTied = !winningOrTied && !secondOrTied && count && topAnswers[2]?.count === count && topAnswers.slice(-1)[0].count !== count + const medal = + ( winningOrTied ? "🥇" + : secondOrTied ? "🥈" + : thirdOrTied ? "🥉" + : "") + return medal +} /** * @param {boolean} isClosed @@ -11,20 +33,20 @@ function optionsToComponents(isClosed, pollOptions) { const topAnswers = pollOptions.toSorted((a, b) => b.count - a.count) /** @type {DiscordTypes.APIMessageTopLevelComponent[]} */ return pollOptions.map(option => { - const winningOrTied = option.count && topAnswers[0].count === option.count + const medal = getMedal(topAnswers, option.count) return { type: DiscordTypes.ComponentType.Container, components: [{ type: DiscordTypes.ComponentType.Section, components: [{ type: DiscordTypes.ComponentType.TextDisplay, - content: option.option_text + content: medal && isClosed ? `${medal} ${option.option_text}` : option.option_text }], accessory: { type: DiscordTypes.ComponentType.Button, - style: winningOrTied ? DiscordTypes.ButtonStyle.Success : DiscordTypes.ButtonStyle.Secondary, + style: medal === "🥇" && isClosed ? DiscordTypes.ButtonStyle.Success : DiscordTypes.ButtonStyle.Secondary, label: option.count.toString(), - custom_id: option.matrix_option, + custom_id: `POLL_OPTION#${option.matrix_option}`, disabled: isClosed } }] @@ -32,6 +54,34 @@ function optionsToComponents(isClosed, pollOptions) { }) } +/** + * @param {number} maxSelections + * @param {number} optionCount + */ +function getMultiSelectString(maxSelections, optionCount) { + if (maxSelections === 1) { + return "Select one answer" + } else if (maxSelections >= optionCount) { + return "Select one or more answers" + } else { + return `Select up to ${maxSelections} answers` + } +} + +/** + * @param {number} maxSelections + * @param {number} optionCount + */ +function getMultiSelectClosedString(maxSelections, optionCount) { + if (maxSelections === 1) { + return "Single choice" + } else if (maxSelections >= optionCount) { + return "Multiple choice" + } else { + return `Multiple choice (up to ${maxSelections})` + } +} + /** * @param {boolean} isClosed * @param {number} maxSelections @@ -40,39 +90,31 @@ function optionsToComponents(isClosed, pollOptions) { * @returns {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody} */ function getPollComponents(isClosed, maxSelections, questionText, pollOptions) { + /** @type {DiscordTypes.APIMessageTopLevelComponent[]} array because it can move around */ + const multiSelectInfoComponent = [{ + type: DiscordTypes.ComponentType.TextDisplay, + content: isClosed ? `-# ${getMultiSelectClosedString(maxSelections, pollOptions.length)}` : `-# ${getMultiSelectString(maxSelections, pollOptions.length)}` + }] /** @type {DiscordTypes.APIMessageTopLevelComponent} */ let headingComponent if (isClosed) { - const multiSelectString = - ( maxSelections === 1 ? "-# ~~Select one answer~~" - : maxSelections >= pollOptions.length ? "-# ~~Select one or more answers~~" - : `-# ~~Select up to ${maxSelections} answers~~`) headingComponent = { // This one is for the poll heading. type: DiscordTypes.ComponentType.Section, components: [ { type: DiscordTypes.ComponentType.TextDisplay, content: `## ${questionText}` - }, - { - type: DiscordTypes.ComponentType.TextDisplay, - content: multiSelectString } ], accessory: { type: DiscordTypes.ComponentType.Button, style: DiscordTypes.ButtonStyle.Secondary, - custom_id: "vote", - label: "Voting closed!", + custom_id: "POLL_VOTE", + label: "Voting closed", disabled: true } } - } - else { - const multiSelectString = - ( maxSelections === 1 ? "-# Select one answer" - : maxSelections >= pollOptions.length ? "-# Select one or more answers" - : `-# Select up to ${maxSelections} answers`) + } else { headingComponent = { // This one is for the poll heading. type: DiscordTypes.ComponentType.Section, components: [ @@ -80,15 +122,13 @@ function getPollComponents(isClosed, maxSelections, questionText, pollOptions) { type: DiscordTypes.ComponentType.TextDisplay, content: `## ${questionText}` }, - { - type: DiscordTypes.ComponentType.TextDisplay, - content: multiSelectString - } + // @ts-ignore + multiSelectInfoComponent.pop() ], accessory: { type: DiscordTypes.ComponentType.Button, style: DiscordTypes.ButtonStyle.Primary, - custom_id: "vote", + custom_id: "POLL_VOTE", label: "Vote!" } } @@ -96,8 +136,92 @@ function getPollComponents(isClosed, maxSelections, questionText, pollOptions) { const optionComponents = optionsToComponents(isClosed, pollOptions) return { flags: DiscordTypes.MessageFlags.IsComponentsV2, - components: [headingComponent, ...optionComponents] + components: [headingComponent, ...optionComponents, ...multiSelectInfoComponent] } } -module.exports.getPollComponents = getPollComponents \ No newline at end of file +/** @param {string} messageID */ +function getPollComponentsFromDatabase(messageID) { + const pollRow = select("poll", ["max_selections", "is_closed", "question_text"], {message_id: messageID}).get() + assert(pollRow) + /** @type {{matrix_option: string, option_text: string, count: number}[]} */ + const pollResults = db.prepare("SELECT matrix_option, option_text, seq, count(discord_or_matrix_user_id) as count FROM poll_option LEFT JOIN poll_vote USING (message_id, matrix_option) WHERE message_id = ? GROUP BY matrix_option ORDER BY seq").all(messageID) + return getPollComponents(!!pollRow.is_closed, pollRow.max_selections, pollRow.question_text, pollResults) +} + +/** + * @param {string} channelID + * @param {string} messageID + * @param {string} questionText + * @param {{matrix_option: string, option_text: string, count: number}[]} pollOptions already sorted correctly + * @returns {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody} + */ +function getPollEndMessage(channelID, messageID, questionText, pollOptions) { + const topAnswers = pollOptions.toSorted((a, b) => b.count - a.count) + const totalVotes = pollOptions.reduce((a, c) => a + c.count, 0) + const tied = topAnswers[0].count === topAnswers[1].count + const titleString = `-# The poll **${questionText}** has closed.` + let winnerString = "" + let resultsString = "" + if (totalVotes == 0) { + winnerString = "There was no winner" + } else if (tied) { + winnerString = "It's a draw!" + resultsString = `${Math.round((topAnswers[0].count/totalVotes)*100)}%` + } else { + const pollWin = select("auto_emoji", ["name", "emoji_id"], {name: "poll_win"}).get() + winnerString = `${topAnswers[0].option_text} <:${pollWin?.name}:${pollWin?.emoji_id}>` + resultsString = `Winning answer • ${Math.round((topAnswers[0].count/totalVotes)*100)}%` + } + // @ts-ignore + const guildID = discord.channels.get(channelID).guild_id + let mainContent = `**${winnerString}**` + if (resultsString) { + mainContent += `\n-# ${resultsString}` + } + return { + flags: DiscordTypes.MessageFlags.IsComponentsV2, + components: [{ + type: DiscordTypes.ComponentType.TextDisplay, + content: titleString + }, { + type: DiscordTypes.ComponentType.Container, + components: [{ + type: DiscordTypes.ComponentType.Section, + components: [{ + type: DiscordTypes.ComponentType.TextDisplay, + content: `**${winnerString}**\n-# ${resultsString}` + }], + accessory: { + type: DiscordTypes.ComponentType.Button, + style: DiscordTypes.ButtonStyle.Link, + url: `https://discord.com/channels/${guildID}/${channelID}/${messageID}`, + label: "View Poll" + } + }] + }] + } +} + +/** + * @param {string} channelID + * @param {string} messageID + */ +async function getPollEndMessageFromDatabase(channelID, messageID) { + const pollWin = select("auto_emoji", ["name", "emoji_id"], {name: "poll_win"}).get() + if (!pollWin) { + await setupEmojis.setupEmojis() + } + + const pollRow = select("poll", ["max_selections", "question_text"], {message_id: messageID}).get() + assert(pollRow) + /** @type {{matrix_option: string, option_text: string, count: number}[]} */ + const pollResults = db.prepare("SELECT matrix_option, option_text, seq, count(discord_or_matrix_user_id) as count FROM poll_option LEFT JOIN poll_vote USING (message_id, matrix_option) WHERE message_id = ? GROUP BY matrix_option ORDER BY seq").all(messageID) + return getPollEndMessage(channelID, messageID, pollRow.question_text, pollResults) +} + +module.exports.getMultiSelectString = getMultiSelectString +module.exports.getPollComponents = getPollComponents +module.exports.getPollComponentsFromDatabase = getPollComponentsFromDatabase +module.exports.getPollEndMessageFromDatabase = getPollEndMessageFromDatabase +module.exports.getMedal = getMedal diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 424ad588..eb1fba02 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -237,6 +237,35 @@ sync.addTemporaryListener(as, "type:org.matrix.msc3381.poll.response", guard("or async event => { if (utils.eventSenderIsFromDiscord(event.sender)) return await vote.updateVote(event) // Matrix votes can't be bridged, so all we do is store it in the database. + await api.ackEvent(event) +})) + +sync.addTemporaryListener(as, "type:org.matrix.msc3381.poll.end", guard("org.matrix.msc3381.poll.end", +/** + * @param {Ty.Event.Outer_Org_Matrix_Msc3381_Poll_End} event it is a org.matrix.msc3381.poll.end because that's what this listener is filtering for + */ +async event => { + if (utils.eventSenderIsFromDiscord(event.sender)) return + const pollEventID = event.content["m.relates_to"]?.event_id + if (!pollEventID) return // Validity check + const messageID = select("event_message", "message_id", {event_id: pollEventID, event_type: "org.matrix.msc3381.poll.start", source: 0}).pluck().get() + if (!messageID) return // Nothing can be done if the parent message was never bridged. Also, Discord-native polls cannot be ended by others, so this only works for polls started on Matrix. + try { + var pollEvent = await api.getEvent(event.room_id, pollEventID) // Poll start event must exist for this to be valid + } catch (e) { + return + } + + // According to the rules, the poll end is only allowed if it was sent by the poll starter, or by someone with redact powers. + if (pollEvent.sender !== event.sender) { + const {powerLevels, powers: {[event.sender]: enderPower}} = await utils.getEffectivePower(event.room_id, [event.sender], api) + if (enderPower < (powerLevels.redact ?? 50)) { + return // Not allowed + } + } + + const messageResponses = await sendEvent.sendEvent(event) + await api.ackEvent(event) })) sync.addTemporaryListener(as, "type:m.reaction", guard("m.reaction", diff --git a/src/types.d.ts b/src/types.d.ts index f18116ef..951d93c9 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -302,6 +302,18 @@ export namespace Event { export type Outer_Org_Matrix_Msc3381_Poll_Response = Outer & {type: "org.matrix.msc3381.poll.response"} + export type Org_Matrix_Msc3381_Poll_End = { + "org.matrix.msc3381.poll.end": {}, + "org.matrix.msc1767.text": string, + body: string, + "m.relates_to": { + rel_type: string + event_id: string + } + } + + export type Outer_Org_Matrix_Msc3381_Poll_End = Outer & {type: "org.matrix.msc3381.poll.end"} + export type M_Room_Member = { membership: string displayname?: string diff --git a/src/web/server.js b/src/web/server.js index 3cb3060c..9d9f5a30 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -69,3 +69,8 @@ as.router.get("/icon.png", defineEventHandler(event => { handleCacheHeaders(event, {maxAge: 86400}) return fs.promises.readFile(join(__dirname, "../../docs/img/icon.png")) })) + +as.router.get("/discord/poll-star-avatar.png", defineEventHandler(event => { + handleCacheHeaders(event, {maxAge: 86400}) + return fs.promises.readFile(join(__dirname, "../../docs/img/poll-star-avatar.png")) +})) From f3ae7ba7920ef17ed1929f2ab7f6016c3a09ecc7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 26 Jan 2026 02:35:58 +1300 Subject: [PATCH 090/153] Rename poll files a bit better --- src/d2m/actions/{close-poll.js => poll-end.js} | 8 ++++---- .../actions/{add-or-remove-vote.js => poll-vote.js} | 0 src/d2m/actions/send-message.js | 6 +++--- src/d2m/event-dispatcher.js | 4 ++-- src/discord/interactions/poll.js | 10 +++++----- 5 files changed, 14 insertions(+), 14 deletions(-) rename src/d2m/actions/{close-poll.js => poll-end.js} (97%) rename src/d2m/actions/{add-or-remove-vote.js => poll-vote.js} (100%) diff --git a/src/d2m/actions/close-poll.js b/src/d2m/actions/poll-end.js similarity index 97% rename from src/d2m/actions/close-poll.js rename to src/d2m/actions/poll-end.js index d97b7b42..55a4aec4 100644 --- a/src/d2m/actions/close-poll.js +++ b/src/d2m/actions/poll-end.js @@ -13,8 +13,8 @@ const api = sync.require("../../matrix/api") const registerUser = sync.require("./register-user") /** @type {import("./create-room")} */ const createRoom = sync.require("../actions/create-room") -/** @type {import("./add-or-remove-vote.js")} */ -const vote = sync.require("../actions/add-or-remove-vote") +/** @type {import("./poll-vote")} */ +const vote = sync.require("../actions/poll-vote") /** @type {import("../../m2d/converters/poll-components")} */ const pollComponents = sync.require("../../m2d/converters/poll-components") /** @type {import("../../m2d/actions/channel-webhook")} */ @@ -64,7 +64,7 @@ async function getAllVotesOnAnswer(channelID, messageID, answerID){ * @param {typeof import("../../../test/data.js")["poll_close"]} closeMessage * @param {DiscordTypes.APIGuild} guild */ -async function closePoll(closeMessage, guild){ +async function endPoll(closeMessage, guild){ const pollCloseObject = closeMessage.embeds[0] const pollMessageID = closeMessage.message_reference.message_id @@ -148,4 +148,4 @@ async function closePoll(closeMessage, guild){ } } -module.exports.closePoll = closePoll +module.exports.endPoll = endPoll diff --git a/src/d2m/actions/add-or-remove-vote.js b/src/d2m/actions/poll-vote.js similarity index 100% rename from src/d2m/actions/add-or-remove-vote.js rename to src/d2m/actions/poll-vote.js diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index b8b0cdaf..3fb8d20b 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -17,8 +17,8 @@ const registerPkUser = sync.require("./register-pk-user") const registerWebhookUser = sync.require("./register-webhook-user") /** @type {import("../actions/create-room")} */ const createRoom = sync.require("../actions/create-room") -/** @type {import("../actions/close-poll")} */ -const closePoll = sync.require("../actions/close-poll") +/** @type {import("../actions/poll-end")} */ +const pollEnd = sync.require("../actions/poll-end") /** @type {import("../../discord/utils")} */ const dUtils = sync.require("../../discord/utils") /** @type {import("../../m2d/actions/channel-webhook")} */ @@ -103,7 +103,7 @@ async function sendMessage(message, channel, guild, row) { } if (message.type === DiscordTypes.MessageType.PollResult) { // We might need to send a message to Discord (if there were any Matrix-side votes). - const detailedResultsMessage = await closePoll.closePoll(message, guild) + const detailedResultsMessage = await pollEnd.endPoll(message, guild) if (detailedResultsMessage) { const threadParent = select("channel_room", "thread_parent", {channel_id: message.channel_id}).pluck().get() const channelID = threadParent ? threadParent : message.channel_id diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 599db498..e8c20a6e 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -32,8 +32,8 @@ const speedbump = sync.require("./actions/speedbump") const retrigger = sync.require("./actions/retrigger") /** @type {import("./actions/set-presence")} */ const setPresence = sync.require("./actions/set-presence") -/** @type {import("./actions/add-or-remove-vote")} */ -const vote = sync.require("./actions/add-or-remove-vote") +/** @type {import("./actions/poll-vote")} */ +const vote = sync.require("./actions/poll-vote") /** @type {import("../m2d/event-dispatcher")} */ const matrixEventDispatcher = sync.require("../m2d/event-dispatcher") /** @type {import("../discord/interactions/matrix-info")} */ diff --git a/src/discord/interactions/poll.js b/src/discord/interactions/poll.js index 94ecb4ca..0a4689d5 100644 --- a/src/discord/interactions/poll.js +++ b/src/discord/interactions/poll.js @@ -12,8 +12,8 @@ const api = sync.require("../../matrix/api") const utils = sync.require("../../matrix/utils") /** @type {import("../../m2d/converters/poll-components")} */ const pollComponents = sync.require("../../m2d/converters/poll-components") -/** @type {import("../../d2m/actions/add-or-remove-vote")} */ -const vote = sync.require("../../d2m/actions/add-or-remove-vote") +/** @type {import("../../d2m/actions/poll-vote")} */ +const vote = sync.require("../../d2m/actions/poll-vote") /** * @param {DiscordTypes.APIMessageComponentButtonInteraction} interaction @@ -34,7 +34,7 @@ async function* _interact({data, message, member, user}, {api}) { const maxSelections = pollRow.max_selections const alreadySelected = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: userID, message_id: message.id}).pluck().all() - + // Show modal (if no capacity or if requested) if (data.custom_id === "POLL_VOTE" || (maxSelections > 1 && alreadySelected.length === maxSelections)) { const options = select("poll_option", ["matrix_option", "option_text", "seq"], {message_id: message.id}, "ORDER BY seq").all().map(option => ({ @@ -91,7 +91,7 @@ async function* _interact({data, message, member, user}, {api}) { db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, option) } })() - + // Update counts on message yield {createInteractionResponse: { type: DiscordTypes.InteractionResponseType.UpdateMessage, @@ -105,7 +105,7 @@ async function* _interact({data, message, member, user}, {api}) { const optionPrefix = "POLL_OPTION#" // we use a prefix to prevent someone from sending a Matrix poll that intentionally collides with other elements of the embed const matrixOption = select("poll_option", "matrix_option", {matrix_option: data.custom_id.substring(optionPrefix.length), message_id: message.id}).pluck().get() assert(matrixOption) - + // Remove a vote if (alreadySelected.includes(matrixOption)) { db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ? AND matrix_option = ?").run(userID, message.id, matrixOption) From 0c781f9b72be4d61b2f2c7c6f3e0228c484adf6b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 26 Jan 2026 20:51:30 +1300 Subject: [PATCH 091/153] Fixes to vote counting --- package-lock.json | 8 +++--- package.json | 2 +- src/d2m/actions/poll-end.js | 48 +++++++++++++-------------------- src/d2m/actions/poll-vote.js | 23 +++++++++------- src/d2m/actions/send-message.js | 29 +++++++++++--------- src/d2m/event-dispatcher.js | 10 +++++-- 6 files changed, 63 insertions(+), 57 deletions(-) diff --git a/package-lock.json b/package-lock.json index eeccc7c0..dd0cbbff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,7 @@ "lru-cache": "^11.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", - "snowtransfer": "^0.17.0", + "snowtransfer": "^0.17.1", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "uqr": "^0.1.2", @@ -2727,9 +2727,9 @@ } }, "node_modules/snowtransfer": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.17.0.tgz", - "integrity": "sha512-H6Avpsco+HlVIkN+MbX34Q7+9g9Wci0wZQwGsvfw20VqEb7jnnk73iUcWytNMYtKZ72Ud58n6cFnQ3apTEamxw==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.17.1.tgz", + "integrity": "sha512-WSXj055EJhzzfD7B3oHVyRTxkqFCaxcVhwKY6B3NkBSHRyM6wHxZLq6VbFYhopUg+lMtd7S1ZO8JM+Ut+js2iA==", "license": "MIT", "dependencies": { "discord-api-types": "^0.38.37" diff --git a/package.json b/package.json index 79199602..1cad1787 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "lru-cache": "^11.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", - "snowtransfer": "^0.17.0", + "snowtransfer": "^0.17.1", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "uqr": "^0.1.2", diff --git a/src/d2m/actions/poll-end.js b/src/d2m/actions/poll-end.js index 55a4aec4..936dedf2 100644 --- a/src/d2m/actions/poll-end.js +++ b/src/d2m/actions/poll-end.js @@ -7,18 +7,10 @@ const {isDeepStrictEqual} = require("util") const passthrough = require("../../passthrough") const {discord, sync, db, select, from} = passthrough const {reg} = require("../../matrix/read-registration") -/** @type {import("../../matrix/api")} */ -const api = sync.require("../../matrix/api") -/** @type {import("./register-user")} */ -const registerUser = sync.require("./register-user") -/** @type {import("./create-room")} */ -const createRoom = sync.require("../actions/create-room") /** @type {import("./poll-vote")} */ const vote = sync.require("../actions/poll-vote") /** @type {import("../../m2d/converters/poll-components")} */ const pollComponents = sync.require("../../m2d/converters/poll-components") -/** @type {import("../../m2d/actions/channel-webhook")} */ -const channelWebhook = sync.require("../../m2d/actions/channel-webhook") // This handles, in the following order: // * verifying Matrix-side votes are accurate for a poll originating on Discord, sending missed votes to Matrix if necessary @@ -28,7 +20,7 @@ const channelWebhook = sync.require("../../m2d/actions/channel-webhook") /** * @param {number} percent */ -function barChart(percent){ +function barChart(percent) { const width = 12 const bars = Math.floor(percent*width) return "█".repeat(bars) + "▒".repeat(width-bars) @@ -40,31 +32,27 @@ function barChart(percent){ * @param {string} answerID * @returns {Promise} */ -async function getAllVotesOnAnswer(channelID, messageID, answerID){ +async function getAllVotesOnAnswer(channelID, messageID, answerID) { const limit = 100 /** @type {DiscordTypes.RESTGetAPIPollAnswerVotersResult["users"]} */ let voteUsers = [] let after = undefined - while (!voteUsers.length || after) { + while (true) { const curVotes = await discord.snow.channel.getPollAnswerVoters(channelID, messageID, answerID, {after: after, limit}) - if (curVotes.users.length === 0) { // Reached the end. - break - } + voteUsers = voteUsers.concat(curVotes.users) if (curVotes.users.length >= limit) { // Loop again for the next page. // @ts-ignore - stupid after = curVotes.users.at(-1).id + } else { // Reached the end. + return voteUsers } - voteUsers = voteUsers.concat(curVotes.users) } - return voteUsers } - /** * @param {typeof import("../../../test/data.js")["poll_close"]} closeMessage - * @param {DiscordTypes.APIGuild} guild */ -async function endPoll(closeMessage, guild){ +async function endPoll(closeMessage) { const pollCloseObject = closeMessage.embeds[0] const pollMessageID = closeMessage.message_reference.message_id @@ -91,16 +79,16 @@ async function endPoll(closeMessage, guild){ for (const discordPollOption of discordPollOptions) { const optionUsers = await getAllVotesOnAnswer(closeMessage.channel_id, pollMessageID, discordPollOption) // Array of user IDs who voted for the option we're testing. - optionUsers.map(user => { + for (const user of optionUsers) { const userLocation = updatedAnswers.findIndex(answer => answer.user.id === user.id) const matrixOption = select("poll_option", "matrix_option", {message_id: pollMessageID, discord_option: discordPollOption}).pluck().get() assert(matrixOption) - if (userLocation === -1){ // We haven't seen this user yet, so we need to add them. + if (userLocation === -1) { // We haven't seen this user yet, so we need to add them. updatedAnswers.push({user, matrixOptionVotes: [matrixOption]}) // toString as this is what we store and get from the database and send to Matrix. } else { // This user already voted for another option on the poll. updatedAnswers[userLocation].matrixOptionVotes.push(matrixOption) } - }) + } } // Check for inconsistencies in what was cached in database vs final confirmed poll answers @@ -109,18 +97,20 @@ async function endPoll(closeMessage, guild){ await Promise.all(updatedAnswers.map(async answer => { voteUsers = voteUsers.filter(item => item !== answer.user.id) // Remove any users we have updated answers for from voteUsers. The only remaining entries in this array will be users who voted, but then removed their votes before the poll ended. const cachedAnswers = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: answer.user.id, message_id: pollMessageID}).pluck().all() - if (!isDeepStrictEqual(new Set(cachedAnswers), new Set(answer.matrixOptionVotes))){ - db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(answer.user.id, pollMessageID) // Delete existing stored votes. - for (const matrixOption of answer.matrixOptionVotes) { - db.prepare("INSERT INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(answer.user.id, pollMessageID, matrixOption) - } - await vote.debounceSendVotes({user_id: answer.user.id, message_id: pollMessageID, channel_id: closeMessage.channel_id, answer_id: 0}, pollEventID) // Fake answer ID, not actually needed (but we're sorta faking the datatype to call this function). + if (!isDeepStrictEqual(new Set(cachedAnswers), new Set(answer.matrixOptionVotes))) { + db.transaction(() => { + db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(answer.user.id, pollMessageID) // Delete existing stored votes. + for (const matrixOption of answer.matrixOptionVotes) { + db.prepare("INSERT INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(answer.user.id, pollMessageID, matrixOption) + } + })() + await vote.sendVotes(answer.user, closeMessage.channel_id, pollMessageID, pollEventID) } })) await Promise.all(voteUsers.map(async user_id => { // Remove these votes. db.prepare("DELETE FROM poll_vote WHERE discord_or_matrix_user_id = ? AND message_id = ?").run(user_id, pollMessageID) - await vote.debounceSendVotes({user_id: user_id, message_id: pollMessageID, channel_id: closeMessage.channel_id, answer_id: 0}, pollEventID) + await vote.sendVotes(user_id, closeMessage.channel_id, pollMessageID, pollEventID) })) } diff --git a/src/d2m/actions/poll-vote.js b/src/d2m/actions/poll-vote.js index e9fee366..85a223dd 100644 --- a/src/d2m/actions/poll-vote.js +++ b/src/d2m/actions/poll-vote.js @@ -11,15 +11,13 @@ const {discord, sync, db, select, from} = passthrough const api = sync.require("../../matrix/api") /** @type {import("./register-user")} */ const registerUser = sync.require("./register-user") -/** @type {import("./create-room")} */ -const createRoom = sync.require("../actions/create-room") const inFlightPollSema = new Semaphore() /** * @param {import("discord-api-types/v10").GatewayMessagePollVoteAddDispatch["d"]} data */ -async function addVote(data){ +async function addVote(data) { const pollEventID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() // Currently Discord doesn't allow sending a poll with anything else, but we bridge it after all other content so reaction_part: 0 is the part that will have the poll. if (!pollEventID) return // Nothing can be done if the parent message was never bridged. @@ -32,7 +30,7 @@ async function addVote(data){ /** * @param {import("discord-api-types/v10").GatewayMessagePollVoteRemoveDispatch["d"]} data */ -async function removeVote(data){ +async function removeVote(data) { const pollEventID = from("event_message").join("poll_option", "message_id").pluck("event_id").where({message_id: data.message_id, event_type: "org.matrix.msc3381.poll.start"}).get() if (!pollEventID) return @@ -59,12 +57,12 @@ async function debounceSendVotes(data, pollEventID) { } /** - * @param {DiscordTypes.APIUser} user + * @param {DiscordTypes.APIUser | string} userOrID * @param {string} channelID * @param {string} pollMessageID * @param {string} pollEventID */ -async function sendVotes(user, channelID, pollMessageID, pollEventID) { +async function sendVotes(userOrID, channelID, pollMessageID, pollEventID) { const latestRoomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() const matchingRoomID = from("message_room").join("historical_channel_room", "historical_room_index").where({message_id: pollMessageID}).pluck("room_id").get() if (!latestRoomID || latestRoomID !== matchingRoomID) { // room upgrade mid-poll?? @@ -72,9 +70,16 @@ async function sendVotes(user, channelID, pollMessageID, pollEventID) { return } - const senderMxid = await registerUser.ensureSimJoined(user, matchingRoomID) + if (typeof userOrID === "string") { // just a string when double-checking a vote removal - good thing the unvoter is already here from having voted + var userID = userOrID + var senderMxid = from("sim").join("sim_member", "mxid").where({user_id: userOrID, room_id: matchingRoomID}).pluck("mxid").get() + if (!senderMxid) return + } else { // sent in full when double-checking adding a vote, so we can properly ensure joined + var userID = userOrID.id + var senderMxid = await registerUser.ensureSimJoined(userOrID, matchingRoomID) + } - const answersArray = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: user.id, message_id: pollMessageID}).pluck().all() + const answersArray = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: userID, message_id: pollMessageID}).pluck().all() const eventID = await api.sendEvent(matchingRoomID, "org.matrix.msc3381.poll.response", { "m.relates_to": { rel_type: "m.reference", @@ -91,4 +96,4 @@ async function sendVotes(user, channelID, pollMessageID, pollEventID) { module.exports.addVote = addVote module.exports.removeVote = removeVote module.exports.debounceSendVotes = debounceSendVotes -module.exports.sendVotes = sendVotes \ No newline at end of file +module.exports.sendVotes = sendVotes diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index 3fb8d20b..3005ca8c 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -55,6 +55,16 @@ async function sendMessage(message, channel, guild, row) { } } + if (message.type === DiscordTypes.MessageType.PollResult) { // ensure all Discord-side votes were pushed to Matrix before a poll is closed + const detailedResultsMessage = await pollEnd.endPoll(message) + if (detailedResultsMessage) { + const threadParent = select("channel_room", "thread_parent", {channel_id: message.channel_id}).pluck().get() + const channelID = threadParent ? threadParent : message.channel_id + const threadID = threadParent ? message.channel_id : undefined + var sentResultsMessage = await channelWebhook.sendMessageWithWebhook(channelID, detailedResultsMessage, threadID) + } + } + const events = await messageToEvent.messageToEvent(message, guild, {}, {api, snow: discord.snow}) const eventIDs = [] if (events.length) { @@ -102,18 +112,13 @@ async function sendMessage(message, channel, guild, row) { })() } - if (message.type === DiscordTypes.MessageType.PollResult) { // We might need to send a message to Discord (if there were any Matrix-side votes). - const detailedResultsMessage = await pollEnd.endPoll(message, guild) - if (detailedResultsMessage) { - const threadParent = select("channel_room", "thread_parent", {channel_id: message.channel_id}).pluck().get() - const channelID = threadParent ? threadParent : message.channel_id - const threadID = threadParent ? message.channel_id : undefined - const sentResultsMessage = await channelWebhook.sendMessageWithWebhook(channelID, detailedResultsMessage, threadID) - db.transaction(() => { - db.prepare("UPDATE event_message SET reaction_part = 1 WHERE event_id = ?").run(eventID) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, sentResultsMessage.id, 1, 0) // part = 1, reaction_part = 0 - })() - } + // part/reaction_part consistency for polls + if (sentResultsMessage) { + db.transaction(() => { + db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(sentResultsMessage.id, historicalRoomIndex) + db.prepare("UPDATE event_message SET reaction_part = 1 WHERE event_id = ?").run(eventID) + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, sentResultsMessage.id, 1, 0) // part = 1, reaction_part = 0 + })() } eventIDs.push(eventID) diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index e8c20a6e..7c2e1184 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -372,11 +372,17 @@ module.exports = { await createSpace.syncSpaceExpressions(data, false) }, - async MESSAGE_POLL_VOTE_ADD(client, data){ + /** + * @param {import("./discord-client")} client + * @param {DiscordTypes.GatewayMessagePollVoteDispatchData} data + */ + async MESSAGE_POLL_VOTE_ADD(client, data) { + if (retrigger.eventNotFoundThenRetrigger(data.message_id, module.exports.MESSAGE_POLL_VOTE_ADD, client, data)) return await vote.addVote(data) }, - async MESSAGE_POLL_VOTE_REMOVE(client, data){ + async MESSAGE_POLL_VOTE_REMOVE(client, data) { + if (retrigger.eventNotFoundThenRetrigger(data.message_id, module.exports.MESSAGE_POLL_VOTE_REMOVE, client, data)) return await vote.removeVote(data) }, From d7f5f8bac41e023a4c4f036c05c8a00e81f347de Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 30 Jan 2026 13:43:01 +1300 Subject: [PATCH 092/153] Fix getting invite state This SSS API call should work on Synapse, Tuwunel, and Continuwuity. A fallback via hierarchy is provided for Conduit. --- src/m2d/event-dispatcher.js | 34 ++++----------- src/matrix/api.js | 84 ++++++++++++++++++++++++++++++++----- src/matrix/mreq.js | 1 + src/types.d.ts | 1 + 4 files changed, 84 insertions(+), 36 deletions(-) diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index eb1fba02..48bff119 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -357,15 +357,7 @@ async event => { await api.ackEvent(event) })) -function getFromInviteRoomState(inviteRoomState, nskey, key) { - if (!Array.isArray(inviteRoomState)) return null - for (const event of inviteRoomState) { - if (event.type === nskey && event.state_key === "") { - return event.content[key] - } - } - return null -} + sync.addTemporaryListener(as, "type:m.space.child", guard("m.space.child", /** @@ -398,24 +390,16 @@ async event => { } // We were invited to a room. We should join, and register the invite details for future reference in web. - let attemptedApiMessage = "According to unsigned invite data." - let inviteRoomState = event.unsigned?.invite_room_state - if (!Array.isArray(inviteRoomState) || inviteRoomState.length === 0) { - try { - inviteRoomState = await api.getInviteState(event.room_id) - attemptedApiMessage = "According to SSS API." - } catch (e) { - attemptedApiMessage = "According to unsigned invite data. SSS API unavailable: " + e.toString() - } + try { + var inviteRoomState = await api.getInviteState(event.room_id, event) + } catch (e) { + console.error(e) + return await api.leaveRoomWithReason(event.room_id, `I wasn't able to find out what this room is. Please report this as a bug. Check console for more details. (${e.toString()})`) } - const name = getFromInviteRoomState(inviteRoomState, "m.room.name", "name") - const topic = getFromInviteRoomState(inviteRoomState, "m.room.topic", "topic") - const avatar = getFromInviteRoomState(inviteRoomState, "m.room.avatar", "url") - const creationType = getFromInviteRoomState(inviteRoomState, "m.room.create", "type") - if (!name) return await api.leaveRoomWithReason(event.room_id, `Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite! (${attemptedApiMessage})`) + if (!inviteRoomState?.name) return await api.leaveRoomWithReason(event.room_id, `Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite.`) await api.joinRoom(event.room_id) - db.prepare("INSERT OR IGNORE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)").run(event.sender, event.room_id, creationType, name, topic, avatar) - if (avatar) utils.getPublicUrlForMxc(avatar) // make sure it's available in the media_proxy allowed URLs + db.prepare("INSERT OR IGNORE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)").run(event.sender, event.room_id, inviteRoomState.type, inviteRoomState.name, inviteRoomState.topic, inviteRoomState.avatar) + if (inviteRoomState.avatar) utils.getPublicUrlForMxc(inviteRoomState.avatar) // make sure it's available in the media_proxy allowed URLs } if (utils.eventSenderIsFromDiscord(event.state_key)) return diff --git a/src/matrix/api.js b/src/matrix/api.js index b71c0685..7e503c27 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -158,20 +158,82 @@ function getStateEventOuter(roomID, type, key) { /** * @param {string} roomID - * @returns {Promise} + * @param {{unsigned?: {invite_room_state?: Ty.Event.InviteStrippedState[]}}} [event] + * @returns {Promise<{name: string?, topic: string?, avatar: string?, type: string?}>} */ -async function getInviteState(roomID) { - /** @type {Ty.R.SSS} */ - const root = await mreq.mreq("POST", path("/client/unstable/org.matrix.simplified_msc3575/sync", `@${reg.sender_localpart}:${reg.ooye.server_name}`, {timeout: "0"}), { - room_subscriptions: { - [roomID]: { - timeline_limit: 0, - required_state: [] +async function getInviteState(roomID, event) { + function getFromInviteRoomState(strippedState, nskey, key) { + if (!Array.isArray(strippedState)) return null + for (const event of strippedState) { + if (event.type === nskey && event.state_key === "") { + return event.content[key] } } - }) - const roomResponse = root.rooms[roomID] - return "stripped_state" in roomResponse ? roomResponse.stripped_state : roomResponse.invite_state + return null + } + + // Try extracting from event (if passed) + if (Array.isArray(event?.unsigned?.invite_room_state) && event.unsigned.invite_room_state.length) { + return { + name: getFromInviteRoomState(event.unsigned.invite_room_state, "m.room.name", "name"), + topic: getFromInviteRoomState(event.unsigned.invite_room_state, "m.room.topic", "topic"), + avatar: getFromInviteRoomState(event.unsigned.invite_room_state, "m.room.avatar", "url"), + type: getFromInviteRoomState(event.unsigned.invite_room_state, "m.room.create", "type") + } + } + + // Try calling sliding sync API and extracting from stripped state + try { + /** @type {Ty.R.SSS} */ + var root = await mreq.mreq("POST", path("/client/unstable/org.matrix.simplified_msc3575/sync", `@${reg.sender_localpart}:${reg.ooye.server_name}`, {timeout: "0"}), { + lists: { + a: { + ranges: [[0, 999]], + timeline_limit: 0, + required_state: [], + filters: { + is_invite: true + } + } + } + }) + + // Extract from sliding sync response if valid (seems to be okay on Synapse, Tuwunel and Continuwuity at time of writing) + if ("lists" in root) { + if (!root.rooms?.[roomID]) { + const e = new Error("Room data unavailable via SSS") + e["data_sss"] = root + throw e + } + + const roomResponse = root.rooms[roomID] + const strippedState = "stripped_state" in roomResponse ? roomResponse.stripped_state : roomResponse.invite_state + + return { + name: getFromInviteRoomState(strippedState, "m.room.name", "name"), + topic: getFromInviteRoomState(strippedState, "m.room.topic", "topic"), + avatar: getFromInviteRoomState(strippedState, "m.room.avatar", "url"), + type: getFromInviteRoomState(strippedState, "m.room.create", "type") + } + } + } catch (e) {} + + // Invalid sliding sync response, try alternative (required for Conduit at time of writing) + const hierarchy = await getHierarchy(roomID, {limit: 1}) + if (hierarchy?.rooms?.[0]?.room_id === roomID) { + const room = hierarchy?.rooms?.[0] + return { + name: room.name ?? null, + topic: room.topic ?? null, + avatar: room.avatar_url ?? null, + type: room.room_type + } + } + + const e = new Error("Room data unavailable via SSS/hierarchy") + e["data_sss"] = root + e["data_hierarchy"] = hierarchy + throw e } /** diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index 9085add8..bb59506f 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -77,6 +77,7 @@ async function mreq(method, url, bodyIn, extra = {}) { /** @type {any} */ var root = JSON.parse(text) } catch (e) { + delete opts.headers?.["Authorization"] throw new MatrixServerError(text, {baseUrl, url, ...opts}) } diff --git a/src/types.d.ts b/src/types.d.ts index 951d93c9..6ee2eb13 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -433,6 +433,7 @@ export namespace R { guest_can_join: boolean join_rule?: string name?: string + topic?: string num_joined_members: number room_id: string room_type?: string From 6200d0b9862102d51f3c51ee735983aabe366c33 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 30 Jan 2026 13:44:13 +1300 Subject: [PATCH 093/153] Fix selective kstate failing on missing events --- src/matrix/kstate.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/matrix/kstate.js b/src/matrix/kstate.js index c901ce11..3648f2d7 100644 --- a/src/matrix/kstate.js +++ b/src/matrix/kstate.js @@ -149,8 +149,10 @@ async function roomToKState(roomID, limitToEvents) { } else { const root = [] await Promise.all(limitToEvents.map(async ([type, key]) => { - const outer = await api.getStateEventOuter(roomID, type, key) - root.push(outer) + try { + const outer = await api.getStateEventOuter(roomID, type, key) + root.push(outer) + } catch (e) {} })) return stateToKState(root) } From 02d62c091442aa5eae39870922ec0753448a4866 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 30 Jan 2026 13:58:29 +1300 Subject: [PATCH 094/153] Only show video embeds when they have extra info --- src/d2m/converters/message-to-event.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 7329f4ae..ffce2f03 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -773,8 +773,12 @@ async function messageToEvent(message, guild, options = {}, di) { continue // Matrix's own URL previews are fine for images. } + if (embed.type === "video" && !embed.title && !embed.description && message.content.includes(embed.video?.url)) { + continue // Doesn't add extra information and the direct video URL is already there. + } + if (embed.type === "poll_result") { - // The code here is only for the message to be bridged to Matrix. Dealing with the Discord-side updates is in actions/poll-close.js. + // The code here is only for the message to be bridged to Matrix. Dealing with the Discord-side updates is in d2m/actions/poll-end.js. } if (embed.url?.startsWith("https://discord.com/")) { From fca4c755223af29a07949808a0671cae7458fd59 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 30 Jan 2026 19:21:10 +1300 Subject: [PATCH 095/153] Fix speedbump+retrigger interactions Send and then edit over speedbump should now just post the edit. Hopefully this doesn't have any negative consequences. --- src/d2m/actions/edit-message.js | 2 +- src/d2m/actions/speedbump.js | 33 ++++++++++++++++--- ...est.js => message-to-event.test.embeds.js} | 0 ...pk.test.js => message-to-event.test.pk.js} | 0 src/d2m/event-dispatcher.js | 8 +++-- 5 files changed, 35 insertions(+), 8 deletions(-) rename src/d2m/converters/{message-to-event.embeds.test.js => message-to-event.test.embeds.js} (100%) rename src/d2m/converters/{message-to-event.pk.test.js => message-to-event.test.pk.js} (100%) diff --git a/src/d2m/actions/edit-message.js b/src/d2m/actions/edit-message.js index 5970b59c..57b8f410 100644 --- a/src/d2m/actions/edit-message.js +++ b/src/d2m/actions/edit-message.js @@ -21,7 +21,7 @@ const mreq = sync.require("../../matrix/mreq") async function editMessage(message, guild, row) { const historicalRoomOfMessage = from("message_room").join("historical_channel_room", "historical_room_index").where({message_id: message.id}).select("room_id").get() const currentRoom = from("channel_room").join("historical_channel_room", "room_id").where({channel_id: message.channel_id}).select("room_id", "historical_room_index").get() - assert(currentRoom) + if (!currentRoom) return if (historicalRoomOfMessage && historicalRoomOfMessage.room_id !== currentRoom.room_id) return // tombstoned rooms should not have new events (including edits) sent to them diff --git a/src/d2m/actions/speedbump.js b/src/d2m/actions/speedbump.js index 7c3109b1..1a6ef63c 100644 --- a/src/d2m/actions/speedbump.js +++ b/src/d2m/actions/speedbump.js @@ -4,6 +4,14 @@ const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") const {discord, select, db} = passthrough +const DEBUG_SPEEDBUMP = false + +function debugSpeedbump(message) { + if (DEBUG_SPEEDBUMP) { + console.log(message) + } +} + const SPEEDBUMP_SPEED = 4000 // 4 seconds delay const SPEEDBUMP_UPDATE_FREQUENCY = 2 * 60 * 60 // 2 hours @@ -27,8 +35,8 @@ async function updateCache(channelID, lastChecked) { db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_webhook_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(foundApplication, foundWebhook, now, channelID) } -/** @type {Set} set of messageID */ -const bumping = new Set() +/** @type {Map} messageID -> number of gateway events currently bumping */ +const bumping = new Map() /** * Slow down a message. After it passes the speedbump, return whether it's okay or if it's been deleted. @@ -36,9 +44,26 @@ const bumping = new Set() * @returns whether it was deleted */ async function doSpeedbump(messageID) { - bumping.add(messageID) + let value = (bumping.get(messageID) ?? 0) + 1 + bumping.set(messageID, value) + debugSpeedbump(`[speedbump] WAIT ${messageID}++ = ${value}`) + await new Promise(resolve => setTimeout(resolve, SPEEDBUMP_SPEED)) - return !bumping.delete(messageID) + + if (!bumping.has(messageID)) { + debugSpeedbump(`[speedbump] DELETED ${messageID}`) + return true + } + value = bumping.get(messageID) - 1 + if (value === 0) { + debugSpeedbump(`[speedbump] OK ${messageID}-- = ${value}`) + bumping.delete(messageID) + return false + } else { + debugSpeedbump(`[speedbump] MULTI ${messageID}-- = ${value}`) + bumping.set(messageID, value) + return true + } } /** diff --git a/src/d2m/converters/message-to-event.embeds.test.js b/src/d2m/converters/message-to-event.test.embeds.js similarity index 100% rename from src/d2m/converters/message-to-event.embeds.test.js rename to src/d2m/converters/message-to-event.test.embeds.js diff --git a/src/d2m/converters/message-to-event.pk.test.js b/src/d2m/converters/message-to-event.test.pk.js similarity index 100% rename from src/d2m/converters/message-to-event.pk.test.js rename to src/d2m/converters/message-to-event.test.pk.js diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 7c2e1184..c25d1c6f 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -274,7 +274,7 @@ module.exports = { // Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes. // If the message content is a string then it includes all interesting fields and is meaningful. // Otherwise, if there are embeds, then the system generated URL preview embeds. - if (!(typeof data.content === "string" || "embeds" in data)) return + if (!(typeof data.content === "string" || "embeds" in data || "components" in data)) return if (dUtils.isEphemeralMessage(data)) return // Ephemeral messages are for the eyes of the receiver only! @@ -282,8 +282,10 @@ module.exports = { const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id) if (affected) return - // Check that the sending-to room exists, and deal with Eventual Consistency(TM) - if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_UPDATE, client, data)) return + if (!row) { + // Check that the sending-to room exists, and deal with Eventual Consistency(TM) + if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_UPDATE, client, data)) return + } /** @type {DiscordTypes.GatewayMessageCreateDispatchData} */ // @ts-ignore From e3e38b9f24e9dac58e782ba904c56d5100ae1263 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 30 Jan 2026 19:22:13 +1300 Subject: [PATCH 096/153] Components v2 support --- src/d2m/converters/message-to-event.js | 111 ++++++++++- .../message-to-event.test.components.js | 79 ++++++++ src/matrix/utils.js | 3 +- test/data.js | 188 ++++++++++++++++++ test/test.js | 5 +- 5 files changed, 376 insertions(+), 10 deletions(-) create mode 100644 src/d2m/converters/message-to-event.test.components.js diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index ffce2f03..8a8e50f4 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -107,9 +107,10 @@ const embedTitleParser = markdown.markdownEngine.parserFor({ /** * @param {{room?: boolean, user_ids?: string[]}} mentions - * @param {DiscordTypes.APIAttachment} attachment + * @param {Omit} attachment + * @param {boolean} [alwaysLink] */ -async function attachmentToEvent(mentions, attachment) { +async function attachmentToEvent(mentions, attachment, alwaysLink) { const external_url = dUtils.getPublicUrlForCdn(attachment.url) const emoji = attachment.content_type?.startsWith("image/jp") ? "📸" @@ -130,7 +131,7 @@ async function attachmentToEvent(mentions, attachment) { } } // for large files, always link them instead of uploading so I don't use up all the space in the content repo - else if (attachment.size > reg.ooye.max_file_size) { + else if (alwaysLink || attachment.size > reg.ooye.max_file_size) { return { $type: "m.room.message", "m.mentions": mentions, @@ -228,6 +229,7 @@ async function pollToEvent(poll) { return matrixAnswer; }) return { + /** @type {"org.matrix.msc3381.poll.start"} */ $type: "org.matrix.msc3381.poll.start", "org.matrix.msc3381.poll.start": { question: { @@ -538,7 +540,7 @@ async function messageToEvent(message, guild, options = {}, di) { // 1. The replied-to event is in a different room to where the reply will be sent (i.e. a room upgrade occurred between) // 2. The replied-to message has no corresponding Matrix event (repliedToUnknownEvent is true) // This branch is optional - do NOT change anything apart from the reply fallback, since it may not be run - if ((repliedToEventRow || repliedToUnknownEvent) && options.includeReplyFallback !== false) { + if ((repliedToEventRow || repliedToUnknownEvent) && options.includeReplyFallback !== false && events.length === 0) { const latestRoomID = repliedToEventRow ? select("channel_room", "room_id", {channel_id: repliedToEventRow.channel_id}).pluck().get() : null if (latestRoomID !== repliedToEventRow?.room_id) repliedToEventInDifferentRoom = true @@ -741,7 +743,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Then attachments if (message.attachments) { - const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions))) + const attachmentEvents = await Promise.all(message.attachments.map(attachment => attachmentToEvent(mentions, attachment))) // Try to merge attachment events with the previous event // This means that if the attachments ended up as a text link, and especially if there were many of them, the events will be joined together. @@ -756,6 +758,101 @@ async function messageToEvent(message, guild, options = {}, di) { } } + // Then components + if (message.components?.length) { + const stack = [new mxUtils.MatrixStringBuilder()] + /** @param {DiscordTypes.APIMessageComponent} component */ + async function processComponent(component) { + // Standalone components + if (component.type === DiscordTypes.ComponentType.TextDisplay) { + const {body, html} = await transformContent(component.content) + stack[0].addParagraph(body, html) + } + else if (component.type === DiscordTypes.ComponentType.Separator) { + stack[0].addParagraph("----", "
    ") + } + else if (component.type === DiscordTypes.ComponentType.File) { + const ev = await attachmentToEvent({}, {...component.file, filename: component.name, size: component.size}, true) + stack[0].addLine(ev.body, ev.formatted_body) + } + else if (component.type === DiscordTypes.ComponentType.MediaGallery) { + const description = component.items.length === 1 ? component.items[0].description || "Image:" : "Image gallery:" + const images = component.items.map(item => { + const publicURL = dUtils.getPublicUrlForCdn(item.media.url) + return { + url: publicURL, + estimatedName: item.media.url.match(/\/([^/?]+)(\?|$)/)?.[1] || publicURL + } + }) + stack[0].addLine(`🖼️ ${description} ${images.map(i => i.url).join(", ")}`, tag`🖼️ ${description} $${images.map(i => tag`${i.estimatedName}`).join(", ")}`) + } + // string select, text input, user select, role select, mentionable select, channel select + + // Components that can have things nested + else if (component.type === DiscordTypes.ComponentType.Container) { + // May contain action row, text display, section, media gallery, separator, file + stack.unshift(new mxUtils.MatrixStringBuilder()) + for (const innerComponent of component.components) { + await processComponent(innerComponent) + } + let {body, formatted_body} = stack.shift().get() + body = body.split("\n").map(l => "| " + l).join("\n") + formatted_body = `
    ${formatted_body}
    ` + if (stack[0].body) stack[0].body += "\n\n" + stack[0].add(body, formatted_body) + } + else if (component.type === DiscordTypes.ComponentType.Section) { + // May contain text display, possibly more in the future + // Accessory may be button or thumbnail + stack.unshift(new mxUtils.MatrixStringBuilder()) + for (const innerComponent of component.components) { + await processComponent(innerComponent) + } + if (component.accessory) { + stack.unshift(new mxUtils.MatrixStringBuilder()) + await processComponent(component.accessory) + const {body, formatted_body} = stack.shift().get() + stack[0].addLine(body, formatted_body) + } + const {body, formatted_body} = stack.shift().get() + stack[0].addParagraph(body, formatted_body) + } + else if (component.type === DiscordTypes.ComponentType.ActionRow) { + const linkButtons = component.components.filter(c => c.type === DiscordTypes.ComponentType.Button && c.style === DiscordTypes.ButtonStyle.Link) + if (linkButtons.length) { + stack[0].addLine("") + for (const linkButton of linkButtons) { + await processComponent(linkButton) + } + } + } + // Components that can only be inside things + else if (component.type === DiscordTypes.ComponentType.Thumbnail) { + // May only be a section accessory + stack[0].add(`🖼️ ${component.media.url}`, tag`🖼️ ${component.media.url}`) + } + else if (component.type === DiscordTypes.ComponentType.Button) { + // May only be a section accessory or in an action row (up to 5) + if (component.style === DiscordTypes.ButtonStyle.Link) { + if (component.label) { + stack[0].add(`[${component.label} ${component.url}] `, tag`${component.label} `) + } else { + stack[0].add(component.url) + } + } + } + + // Not handling file upload or label because they are modal-only components + } + + for (const component of message.components) { + await processComponent(component) + } + + const {body, formatted_body} = stack[0].get() + await addTextEvent(body, formatted_body, "m.text") + } + // Then polls if (message.poll) { const pollEvent = await pollToEvent(message.poll) @@ -773,7 +870,7 @@ async function messageToEvent(message, guild, options = {}, di) { continue // Matrix's own URL previews are fine for images. } - if (embed.type === "video" && !embed.title && !embed.description && message.content.includes(embed.video?.url)) { + if (embed.type === "video" && !embed.title && message.content.includes(embed.video?.url)) { continue // Doesn't add extra information and the direct video URL is already there. } @@ -904,7 +1001,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Strip formatted_body where equivalent to body if (!options.alwaysReturnFormattedBody) { for (const event of events) { - if (["m.text", "m.notice"].includes(event.msgtype) && event.body === event.formatted_body) { + if (event.$type === "m.room.message" && "msgtype" in event && ["m.text", "m.notice"].includes(event.msgtype) && event.body === event.formatted_body) { delete event.format delete event.formatted_body } diff --git a/src/d2m/converters/message-to-event.test.components.js b/src/d2m/converters/message-to-event.test.components.js new file mode 100644 index 00000000..7d875a6b --- /dev/null +++ b/src/d2m/converters/message-to-event.test.components.js @@ -0,0 +1,79 @@ +const {test} = require("supertape") +const {messageToEvent} = require("./message-to-event") +const data = require("../../../test/data") + +test("message2event components: pk question mark output", async t => { + const events = await messageToEvent(data.message_with_components.pk_question_mark_response, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + body: + "| ### Lillith (INX)" + + "\n| " + + "\n| **Display name:** Lillith (she/her)" + + "\n| **Pronouns:** She/Her" + + "\n| **Message count:** 3091" + + "\n| 🖼️ https://files.inx.moe/p/cdn/lillith.webp" + + "\n| " + + "\n| ----" + + "\n| " + + "\n| **Proxy tags:**" + + "\n| ``l;text``" + + "\n| ``l:text``" + + "\n| ``l.text``" + + "\n| ``textl.``" + + "\n| ``textl;``" + + "\n| ``textl:``" + + "\n" + + "\n-# System ID: `xffgnx` ∙ Member ID: `pphhoh`" + + "\n-# Created: 2025-12-31 03:16:45 UTC" + + "\n[View on dashboard https://dash.pluralkit.me/profile/m/pphhoh] " + + "\n" + + "\n----" + + "\n" + + "\n| **System:** INX (`xffgnx`)" + + "\n| **Member:** Lillith (`pphhoh`)" + + "\n| **Sent by:** infinidoge1337 (@unknown-user:)" + + "\n| " + + "\n| **Account Roles (7)**" + + "\n| §b, !, ‼, Ears Port Ping, Ears Update Ping, Yttr Ping, unsup Ping" + + "\n| 🖼️ https://files.inx.moe/p/cdn/lillith.webp" + + "\n| " + + "\n| ----" + + "\n| " + + "\n| Same hat" + + "\n| 🖼️ Image: https://bridge.example.org/download/discordcdn/934955898965729280/1466556006527012987/image.png" + + "\n" + + "\n-# Original Message ID: 1466556003645657118 · ", + format: "org.matrix.custom.html", + formatted_body: "
    " + + "

    Lillith (INX)

    " + + "

    Display name: Lillith (she/her)" + + "
    Pronouns: She/Her" + + "
    Message count: 3091

    " + + `🖼️ https://files.inx.moe/p/cdn/lillith.webp` + + "
    " + + "

    Proxy tags:" + + "
    l;text" + + "
    l:text" + + "
    l.text" + + "
    textl." + + "
    textl;" + + "
    textl:

    " + + "

    System ID: xffgnx ∙ Member ID: pphhoh
    " + + "Created: 2025-12-31 03:16:45 UTC

    " + + `View on dashboard ` + + "
    " + + "

    System: INX (xffgnx)" + + "
    Member: Lillith (pphhoh)" + + "
    Sent by: infinidoge1337 (@unknown-user:)" + + "

    Account Roles (7)" + + "
    §b, !, ‼, Ears Port Ping, Ears Update Ping, Yttr Ping, unsup Ping

    " + + `🖼️ https://files.inx.moe/p/cdn/lillith.webp` + + "
    " + + "

    Same hat

    " + + `🖼️ Image: image.png
    ` + + "

    Original Message ID: 1466556003645657118 · <t:1769724599:f>

    ", + "m.mentions": {}, + msgtype: "m.text", + }]) +}) diff --git a/src/matrix/utils.js b/src/matrix/utils.js index f299d95f..b1315102 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -106,7 +106,8 @@ class MatrixStringBuilder { if (formattedBody == undefined) formattedBody = body if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n\n" this.body += body - formattedBody = `

    ${formattedBody}

    ` + const match = formattedBody.match(/^<([a-zA-Z]+[a-zA-Z0-9]*)/) + if (!match || !BLOCK_ELEMENTS.includes(match[1].toUpperCase())) formattedBody = `

    ${formattedBody}

    ` this.formattedBody += formattedBody } return this diff --git a/test/data.js b/test/data.js index 786737c6..09749e66 100644 --- a/test/data.js +++ b/test/data.js @@ -4975,6 +4975,194 @@ module.exports = { tts: false } }, + message_with_components: { + pk_question_mark_response: { + type: 0, + content: '', + mentions: [], + mention_roles: [], + attachments: [], + embeds: [], + timestamp: '2026-01-30T01:20:07.488000+00:00', + edited_timestamp: null, + flags: 32768, + author: { + id: '772659086046658620', + username: 'cadence.worm', + avatar: '466df0c98b1af1e1388f595b4c1ad1b9', + discriminator: '0', + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: 'cadence', + avatar_decoration_data: null, + collectibles: null, + display_name_styles: null, + banner_color: null, + clan: { + identity_guild_id: '532245108070809601', + identity_enabled: true, + tag: 'doll', + badge: 'dba08126b4e810a0e096cc7cd5bc37f0' + }, + primary_guild: { + identity_guild_id: '532245108070809601', + identity_enabled: true, + tag: 'doll', + badge: 'dba08126b4e810a0e096cc7cd5bc37f0' + } + }, + components: [ + { + type: 17, + id: 1, + accent_color: 1042150, + components: [ + { + type: 9, + id: 2, + components: [ + { type: 10, id: 3, content: '### Lillith (INX)' }, + { + type: 10, + id: 4, + content: '**Display name:** Lillith (she/her)\n' + + '**Pronouns:** She/Her\n' + + '**Message count:** 3091' + } + ], + accessory: { + type: 11, + id: 5, + media: { + id: '1466603856149610687', + url: 'https://files.inx.moe/p/cdn/lillith.webp', + proxy_url: 'https://images-ext-1.discordapp.net/external/Kn5b32mM4o8AAQbq0k39KOzp9-fy6D1tWKvK_XI27LI/https/files.inx.moe/p/cdn/lillith.webp', + width: 256, + height: 256, + placeholder: 'KVoKJwSnt7lZl5ecj1mal5eGWjAHZXIA', + placeholder_version: 1, + content_scan_metadata: { version: 4, flags: 0 }, + content_type: 'image/webp', + loading_state: 2, + flags: 0 + }, + description: null, + spoiler: false + } + }, + { type: 14, id: 6, spacing: 1, divider: true }, + { + type: 10, + id: 7, + content: '**Proxy tags:**\n' + + '``l;text``\n' + + '``l:text``\n' + + '``l.text``\n' + + '``textl.``\n' + + '``textl;``\n' + + '``textl:``' + } + ], + spoiler: false + }, + { + type: 9, + id: 8, + components: [ + { + type: 10, + id: 9, + content: '-# System ID: `xffgnx` ∙ Member ID: `pphhoh`\n' + + '-# Created: 2025-12-31 03:16:45 UTC' + } + ], + accessory: { + type: 2, + id: 10, + style: 5, + label: 'View on dashboard', + url: 'https://dash.pluralkit.me/profile/m/pphhoh' + } + }, + { type: 14, id: 11, spacing: 1, divider: true }, + { + type: 17, + id: 12, + accent_color: null, + components: [ + { + type: 9, + id: 13, + components: [ + { + type: 10, + id: 14, + content: '**System:** INX (`xffgnx`)\n' + + '**Member:** Lillith (`pphhoh`)\n' + + '**Sent by:** infinidoge1337 (<@197126718400626689>)\n' + + '\n' + + '**Account Roles (7)**\n' + + '§b, !, ‼, Ears Port Ping, Ears Update Ping, Yttr Ping, unsup Ping' + } + ], + accessory: { + type: 11, + id: 15, + media: { + id: '1466603856149610689', + url: 'https://files.inx.moe/p/cdn/lillith.webp', + proxy_url: 'https://images-ext-1.discordapp.net/external/Kn5b32mM4o8AAQbq0k39KOzp9-fy6D1tWKvK_XI27LI/https/files.inx.moe/p/cdn/lillith.webp', + width: 256, + height: 256, + placeholder: 'KVoKJwSnt7lZl5ecj1mal5eGWjAHZXIA', + placeholder_version: 1, + content_scan_metadata: { version: 4, flags: 0 }, + content_type: 'image/webp', + loading_state: 2, + flags: 0 + }, + description: null, + spoiler: false + } + }, + { type: 14, id: 16, spacing: 2, divider: true }, + { type: 10, id: 17, content: 'Same hat' }, + { + type: 12, + id: 18, + items: [ + { + media: { + id: '1466603856149610690', + url: 'https://cdn.discordapp.com/attachments/934955898965729280/1466556006527012987/image.png?ex=697d2c37&is=697bdab7&hm=09c5028be61ce01ebbdda5c79c42e4dc10d053ce0c4b12c9d84135a0708e9db6&', + proxy_url: 'https://media.discordapp.net/attachments/934955898965729280/1466556006527012987/image.png?ex=697d2c37&is=697bdab7&hm=09c5028be61ce01ebbdda5c79c42e4dc10d053ce0c4b12c9d84135a0708e9db6&', + width: 285, + height: 126, + placeholder: '0PcBA4BqSIl9t/dnn9f0rm0=', + placeholder_version: 1, + content_scan_metadata: { version: 4, flags: 0 }, + content_type: 'image/png', + loading_state: 2, + flags: 0 + }, + description: null, + spoiler: false + } + ] + } + ], + spoiler: false + }, + { + type: 10, + id: 19, + content: '-# Original Message ID: 1466556003645657118 · ' + } + ] + } + }, message_update: { edit_by_webhook: { application_id: "684280192553844747", diff --git a/test/test.js b/test/test.js index 5ae9f67e..81c079a4 100644 --- a/test/test.js +++ b/test/test.js @@ -160,8 +160,9 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/d2m/converters/emoji-to-key.test") require("../src/d2m/converters/lottie.test") require("../src/d2m/converters/message-to-event.test") - require("../src/d2m/converters/message-to-event.embeds.test") - require("../src/d2m/converters/message-to-event.pk.test") + require("../src/d2m/converters/message-to-event.test.components") + require("../src/d2m/converters/message-to-event.test.embeds") + require("../src/d2m/converters/message-to-event.test.pk") require("../src/d2m/converters/pins-to-list.test") require("../src/d2m/converters/remove-reaction.test") require("../src/d2m/converters/thread-to-announcement.test") From 44208b6fd5917a34c2d3b9b1bc397782e4abff03 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 30 Jan 2026 19:22:32 +1300 Subject: [PATCH 097/153] Add /ping command --- src/d2m/converters/edit-to-changes.js | 2 +- src/discord/interactions/ping.js | 195 ++++++++++++++++++++++++++ src/discord/register-interactions.js | 17 +++ src/matrix/api.js | 14 ++ 4 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 src/discord/interactions/ping.js diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index 869bb3c5..48b7dd3e 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -227,8 +227,8 @@ async function editToChanges(message, guild, api) { */ function makeReplacementEventContent(oldID, newFallbackContent, newInnerContent) { const content = { - ...newFallbackContent, "m.mentions": {}, + ...newFallbackContent, "m.new_content": { ...newInnerContent }, diff --git a/src/discord/interactions/ping.js b/src/discord/interactions/ping.js new file mode 100644 index 00000000..57b48b1e --- /dev/null +++ b/src/discord/interactions/ping.js @@ -0,0 +1,195 @@ +// @ts-check + +const assert = require("assert").strict +const Ty = require("../../types") +const DiscordTypes = require("discord-api-types/v10") +const {discord, sync, select, from} = require("../../passthrough") +const {id: botID} = require("../../../addbot") +const {InteractionMethods} = require("snowtransfer") + +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") +/** @type {import("../../web/routes/guild")} */ +const webGuild = sync.require("../../web/routes/guild") + +/** + * @param {DiscordTypes.APIApplicationCommandAutocompleteGuildInteraction} interaction + * @param {{api: typeof api}} di + * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} + */ +async function* _interactAutocomplete({data, channel}, {api}) { + function exit() { + return {createInteractionResponse: { + /** @type {DiscordTypes.InteractionResponseType.ApplicationCommandAutocompleteResult} */ + type: DiscordTypes.InteractionResponseType.ApplicationCommandAutocompleteResult, + data: { + choices: [] + } + }} + } + + // Check it was used in a bridged channel + const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get() + if (!roomID) return yield exit() + + // Check we are in fact autocompleting the first option, the user + if (!data.options?.[0] || data.options[0].type !== DiscordTypes.ApplicationCommandOptionType.String || !data.options[0].focused) { + return yield exit() + } + + /** @type {{displayname: string | null, mxid: string}[][]} */ + const providedMatches = [] + + const input = data.options[0].value + if (input === "") { + const events = await api.getEvents(roomID, "b", {limit: 40}) + const recents = new Set(events.chunk.map(e => e.sender)) + const matches = select("member_cache", ["mxid", "displayname"], {room_id: roomID}, "AND displayname IS NOT NULL LIMIT 25").all() + matches.sort((a, b) => +recents.has(b.mxid) - +recents.has(a.mxid)) + providedMatches.push(matches) + } else if (input.startsWith("@")) { // only autocomplete mxids + const query = input.replaceAll(/[%_$]/g, char => `$${char}`) + "%" + const matches = select("member_cache", ["mxid", "displayname"], {room_id: roomID}, "AND mxid LIKE ? ESCAPE '$' LIMIT 25").all(query) + providedMatches.push(matches) + } else { + const query = "%" + input.replaceAll(/[%_$]/g, char => `$${char}`) + "%" + const displaynameMatches = select("member_cache", ["mxid", "displayname"], {room_id: roomID}, "AND displayname IS NOT NULL AND displayname LIKE ? ESCAPE '$' LIMIT 25").all(query) + // prioritise matches closer to the start + displaynameMatches.sort((a, b) => { + let ai = a.displayname.toLowerCase().indexOf(input.toLowerCase()) + if (ai === -1) ai = 999 + let bi = b.displayname.toLowerCase().indexOf(input.toLowerCase()) + if (bi === -1) bi = 999 + return ai - bi + }) + providedMatches.push(displaynameMatches) + let mxidMatches = select("member_cache", ["mxid", "displayname"], {room_id: roomID}, "AND displayname IS NOT NULL AND mxid LIKE ? ESCAPE '$' LIMIT 25").all(query) + mxidMatches = mxidMatches.filter(match => { + // don't include matches in domain part of mxid + if (!match.mxid.match(/^[^:]*/)?.includes(query)) return false + if (displaynameMatches.some(m => m.mxid === match.mxid)) return false + return true + }) + providedMatches.push(mxidMatches) + } + + // merge together + let matches = providedMatches.flat() + + // don't include bot + matches = matches.filter(m => m.mxid !== utils.bot) + + // remove duplicates and count up to 25 + const limitedMatches = [] + const seen = new Set() + for (const match of matches) { + if (limitedMatches.length >= 25) break + if (seen.has(match.mxid)) continue + limitedMatches.push(match) + seen.add(match.mxid) + } + + yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.ApplicationCommandAutocompleteResult, + data: { + choices: limitedMatches.map(row => ({name: (row.displayname || row.mxid).slice(0, 100), value: row.mxid.slice(0, 100)})) + } + }} +} + +/** + * @param {DiscordTypes.APIChatInputApplicationCommandGuildInteraction & {channel: DiscordTypes.APIGuildTextChannel}} interaction + * @param {{api: typeof api}} di + * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} + */ +async function* _interactCommand({data, channel, guild_id}, {api}) { + const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get() + if (!roomID) { + return yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, + data: { + flags: DiscordTypes.MessageFlags.Ephemeral, + content: "This channel isn't bridged to Matrix." + } + }} + } + + assert(data.options?.[0]?.type === DiscordTypes.ApplicationCommandOptionType.String) + const mxid = data.options[0].value + if (!mxid.match(/^@[^:]*:./)) { + return yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, + data: { + flags: DiscordTypes.MessageFlags.Ephemeral, + content: "⚠️ To use `/ping`, you must select an option from autocomplete, or type a full Matrix ID.\n> Tip: This command is not necessary. You can also ping Matrix users just by typing @their name in your message. It won't look like anything, but it does go through." + } + }} + } + + yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.DeferredChannelMessageWithSource + }} + + try { + /** @type {Ty.Event.M_Room_Member} */ + var member = await api.getStateEvent(roomID, "m.room.member", mxid) + } catch (e) {} + + if (!member || member.membership !== "join") { + const inChannels = discord.guildChannelMap.get(guild_id) + .map(cid => discord.channels.get(cid)) + .sort((a, b) => webGuild._getPosition(a, discord.channels) - webGuild._getPosition(b, discord.channels)) + .filter(channel => from("channel_room").join("member_cache", "room_id").select("mxid").where({channel_id: channel.id, mxid}).get()) + if (inChannels.length) { + return yield {editOriginalInteractionResponse: { + content: `That person isn't in this channel. They have only joined the following channels:\n${inChannels.map(c => `<#${c.id}>`).join(" • ")}\nYou can ask them to join this channel with \`/invite\`.`, + }} + } else { + return yield {editOriginalInteractionResponse: { + content: "That person isn't in this channel. You can invite them with `/invite`." + }} + } + } + + yield {editOriginalInteractionResponse: { + content: "@" + (member.displayname || mxid) + }} + + yield {createFollowupMessage: { + flags: DiscordTypes.MessageFlags.Ephemeral | DiscordTypes.MessageFlags.IsComponentsV2, + components: [{ + type: DiscordTypes.ComponentType.Container, + components: [{ + type: DiscordTypes.ComponentType.TextDisplay, + content: "Tip: This command is not necessary. You can also ping Matrix users just by typing @their name in your message. It won't look like anything, but it does go through." + }] + }] + }} +} + +/* c8 ignore start */ + +/** @param {(DiscordTypes.APIChatInputApplicationCommandGuildInteraction & {channel: DiscordTypes.APIGuildTextChannel}) | DiscordTypes.APIApplicationCommandAutocompleteGuildInteraction} interaction */ +async function interact(interaction) { + if (interaction.type === DiscordTypes.InteractionType.ApplicationCommandAutocomplete) { + for await (const response of _interactAutocomplete(interaction, {api})) { + if (response.createInteractionResponse) { + await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, response.createInteractionResponse) + } + } + } else { + for await (const response of _interactCommand(interaction, {api})) { + if (response.createInteractionResponse) { + await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, response.createInteractionResponse) + } else if (response.editOriginalInteractionResponse) { + await discord.snow.interaction.editOriginalInteractionResponse(botID, interaction.token, response.editOriginalInteractionResponse) + } else if (response.createFollowupMessage) { + await discord.snow.interaction.createFollowupMessage(botID, interaction.token, response.createFollowupMessage) + } + } + } +} + +module.exports.interact = interact diff --git a/src/discord/register-interactions.js b/src/discord/register-interactions.js index 63b04b0c..b37f28e9 100644 --- a/src/discord/register-interactions.js +++ b/src/discord/register-interactions.js @@ -10,6 +10,7 @@ const permissions = sync.require("./interactions/permissions.js") const reactions = sync.require("./interactions/reactions.js") const privacy = sync.require("./interactions/privacy.js") const poll = sync.require("./interactions/poll.js") +const ping = sync.require("./interactions/ping.js") // User must have EVERY permission in default_member_permissions to be able to use the command @@ -38,6 +39,20 @@ discord.snow.interaction.bulkOverwriteApplicationCommands(id, [{ description: "The Matrix user to invite, e.g. @username:example.org", name: "user" } + ], +}, { + name: "ping", + contexts: [DiscordTypes.InteractionContextType.Guild], + type: DiscordTypes.ApplicationCommandType.ChatInput, + description: "Ping a Matrix user.", + options: [ + { + type: DiscordTypes.ApplicationCommandOptionType.String, + description: "Display name or ID of the Matrix user", + name: "user", + autocomplete: true, + required: true + } ] }, { name: "privacy", @@ -94,6 +109,8 @@ async function dispatchInteraction(interaction) { await permissions.interactEdit(interaction) } else if (interactionId === "Reactions") { await reactions.interact(interaction) + } else if (interactionId === "ping") { + await ping.interact(interaction) } else if (interactionId === "privacy") { await privacy.interact(interaction) } else { diff --git a/src/matrix/api.js b/src/matrix/api.js index 7e503c27..1cd05d34 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -128,6 +128,19 @@ async function getEventForTimestamp(roomID, ts) { return root } +/** + * @param {string} roomID + * @param {"b" | "f"} dir + * @param {{from?: string, limit?: any}} [pagination] + * @param {any} [filter] + */ +async function getEvents(roomID, dir, pagination = {}, filter) { + filter = filter && JSON.stringify(filter) + /** @type {Ty.Pagination>} */ + const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/messages`, null, {...pagination, dir, filter})) + return root +} + /** * @param {string} roomID * @returns {Promise[]>} @@ -583,6 +596,7 @@ module.exports.leaveRoom = leaveRoom module.exports.leaveRoomWithReason = leaveRoomWithReason module.exports.getEvent = getEvent module.exports.getEventForTimestamp = getEventForTimestamp +module.exports.getEvents = getEvents module.exports.getAllState = getAllState module.exports.getStateEvent = getStateEvent module.exports.getStateEventOuter = getStateEventOuter From af9e2d89a5362c52402add2491d01c6ba8eb3041 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 30 Jan 2026 20:01:08 +1300 Subject: [PATCH 098/153] Wrangle generated embeds; fix edit m.mentions --- package.json | 2 +- src/d2m/converters/edit-to-changes.js | 38 ++++++++++++++-------- src/d2m/converters/edit-to-changes.test.js | 10 +++++- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 1cad1787..d0a154c6 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "author": "Cadence, PapiOphidian", "license": "AGPL-3.0-or-later", "engines": { - "node": ">=20" + "node": ">=22" }, "dependencies": { "@chriscdn/promise-semaphore": "^3.0.1", diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index 48b7dd3e..86a40287 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -46,7 +46,7 @@ async function editToChanges(message, guild, api) { // Now, this code path is only used by generated embeds for messages that were originally sent from Matrix. const originallyFromMatrix = oldEventRows.find(r => r.part === 0)?.source === 0 - const isGeneratedEmbed = !("content" in message) || originallyFromMatrix + const mightBeGeneratedEmbed = !("content" in message) || originallyFromMatrix // Figure out who to send as @@ -79,7 +79,7 @@ async function editToChanges(message, guild, api) { */ /** * 1. Events that are matched, and should be edited by sending another m.replace event - * @type {{old: typeof oldEventRows[0], newFallbackContent: typeof newFallbackContent[0], newInnerContent: typeof newInnerContent[0]}[]} + * @type {{old: typeof oldEventRows[0], oldMentions?: any, newFallbackContent: typeof newFallbackContent[0], newInnerContent: typeof newInnerContent[0]}[]} */ let eventsToReplace = [] /** @@ -133,20 +133,20 @@ async function editToChanges(message, guild, api) { // If this is a generated embed update, only allow the embeds to be updated, since the system only sends data about events. Ignore changes to other things. // This also prevents Matrix events that were re-subtyped during conversion (e.g. large image -> text link) from being mistakenly included. - if (isGeneratedEmbed) { + if (mightBeGeneratedEmbed) { unchangedEvents = unchangedEvents.concat( dUtils.filterTo(eventsToRedact, e => e.old.event_subtype === "m.notice" && e.old.source === 1), // Move everything except embeds from eventsToRedact to unchangedEvents. dUtils.filterTo(eventsToReplace, e => e.old.event_subtype === "m.notice" && e.old.source === 1) // Move everything except embeds from eventsToReplace to unchangedEvents. ) eventsToSend = eventsToSend.filter(e => e.msgtype === "m.notice") // Don't send new events that aren't the embed. + } - // Don't post new generated embeds for messages if it's been a while since the message was sent. Detached embeds look weird. - const messageTooOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 30 * 1000 // older than 30 seconds ago - // Don't post new generated embeds for messages if the setting was disabled. - const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 - if (messageTooOld || !embedsEnabled) { - eventsToSend = [] - } + // Don't post new generated embeds for messages if it's been a while since the message was sent. Detached embeds look weird. + const messageTooOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 30 * 1000 // older than 30 seconds ago + // Don't post new generated embeds for messages if the setting was disabled. + const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 + if ((messageTooOld || !embedsEnabled) && !message.author.bot) { + eventsToSend = eventsToSend.filter(e => e.msgtype !== "m.notice") // Only send events that aren't embeds. } // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! @@ -161,6 +161,7 @@ async function editToChanges(message, guild, api) { const event = eventsToReplace[i] if (!eventIsText(event)) continue // not text, can't analyse const oldEvent = await api.getEvent(roomID, eventsToReplace[i].old.event_id) + eventsToReplace[i].oldMentions = oldEvent.content["m.mentions"] const oldEventBodyWithoutQuotedReply = oldEvent.content.body?.replace(/^(>.*\n)*\n*/sm, "") if (oldEventBodyWithoutQuotedReply !== event.newInnerContent.body) continue // event changed, must replace it // Move it from eventsToRedact to unchangedEvents. @@ -210,7 +211,7 @@ async function editToChanges(message, guild, api) { // Removing unnecessary properties before returning return { roomID, - eventsToReplace: eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)})), + eventsToReplace: eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.oldMentions, e.newFallbackContent, e.newInnerContent)})), eventsToRedact: eventsToRedact.map(e => e.old.event_id), eventsToSend, senderMxid, @@ -221,14 +222,25 @@ async function editToChanges(message, guild, api) { /** * @template T * @param {string} oldID + * @param {any} oldMentions * @param {T} newFallbackContent * @param {T} newInnerContent * @returns {import("../../types").Event.ReplacementContent} content */ -function makeReplacementEventContent(oldID, newFallbackContent, newInnerContent) { +function makeReplacementEventContent(oldID, oldMentions, newFallbackContent, newInnerContent) { + const mentions = {} + const newMentionUsers = new Set(newFallbackContent["m.mentions"]?.user_ids || []) + const oldMentionUsers = new Set(oldMentions?.user_ids || []) + const mentionDiff = newMentionUsers.difference(oldMentionUsers) + if (mentionDiff.size) { + mentions.user_ids = [...mentionDiff.values()] + } + if (newFallbackContent["m.mentions"]?.room && !oldMentions?.room) { + mentions.room = true + } const content = { - "m.mentions": {}, ...newFallbackContent, + "m.mentions": mentions, "m.new_content": { ...newInnerContent }, diff --git a/src/d2m/converters/edit-to-changes.test.js b/src/d2m/converters/edit-to-changes.test.js index d6877020..0fd85b85 100644 --- a/src/d2m/converters/edit-to-changes.test.js +++ b/src/d2m/converters/edit-to-changes.test.js @@ -42,7 +42,14 @@ test("edit2changes: bot response", async t => { const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.bot_response, data.guild.general, { getEvent(roomID, eventID) { t.equal(eventID, "$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY") - return {content: {body: "dummy"}} + return { + content: { + "m.mentions": { + user_ids: ["@cadence:cadence.moe"], + }, + body: "dummy" + } + } }, async getJoinedMembers(roomID) { t.equal(roomID, "!hYnGGlPHlbujVVfktC:cadence.moe") @@ -365,6 +372,7 @@ test("edit2changes: generated embed", async t => { test("edit2changes: generated embed on a reply", async t => { let called = 0 + data.message_update.embed_generated_on_reply.timestamp = new Date().toISOString() const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.embed_generated_on_reply, data.guild.general, { getEvent(roomID, eventID) { called++ From b16d731ddbdef62807e28f3f904f1e683ea2cf10 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 3 Feb 2026 01:02:57 +1300 Subject: [PATCH 099/153] Better emoji pack names --- src/d2m/actions/create-space.js | 6 ++++-- src/d2m/actions/expression.js | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/d2m/actions/create-space.js b/src/d2m/actions/create-space.js index 89e0f084..1417b2d0 100644 --- a/src/d2m/actions/create-space.js +++ b/src/d2m/actions/create-space.js @@ -229,14 +229,16 @@ async function syncSpaceExpressions(data, checkBeforeSync) { */ async function update(spaceID, key, eventKey, fn) { if (!(key in data) || !data[key].length) return - const content = await fn(data[key]) + const guild = discord.guilds.get(data.guild_id) + assert(guild) + const content = await fn(data[key], guild) if (checkBeforeSync) { let existing try { existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey) } catch (e) { // State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake. - existing = fn([]) + existing = fn([], guild) } if (isDeepStrictEqual(existing, content)) return } diff --git a/src/d2m/actions/expression.js b/src/d2m/actions/expression.js index 1c95dda1..c7ab27a0 100644 --- a/src/d2m/actions/expression.js +++ b/src/d2m/actions/expression.js @@ -9,11 +9,12 @@ const file = sync.require("../../matrix/file") /** * @param {DiscordTypes.APIEmoji[]} emojis + * @param {DiscordTypes.APIGuild} guild */ -async function emojisToState(emojis) { +async function emojisToState(emojis, guild) { const result = { pack: { - display_name: "Discord Emojis", + display_name: `${guild.name} (Discord Emojis)`, usage: ["emoticon"] // we'll see... }, images: { @@ -42,11 +43,12 @@ async function emojisToState(emojis) { /** * @param {DiscordTypes.APISticker[]} stickers + * @param {DiscordTypes.APIGuild} guild */ -async function stickersToState(stickers) { +async function stickersToState(stickers, guild) { const result = { pack: { - display_name: "Discord Stickers", + display_name: `${guild.name} (Discord Stickers)`, usage: ["sticker"] // we'll see... }, images: { From 45285a4835774a26c1d7aba2d86af33ba57a683d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 3 Feb 2026 01:22:38 +1300 Subject: [PATCH 100/153] Only send components if they rendered to something --- src/d2m/converters/message-to-event.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 8a8e50f4..f48146ff 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -850,7 +850,9 @@ async function messageToEvent(message, guild, options = {}, di) { } const {body, formatted_body} = stack[0].get() - await addTextEvent(body, formatted_body, "m.text") + if (body.trim().length) { + await addTextEvent(body, formatted_body, "m.text") + } } // Then polls From f287806bcd106e20fd4b1483648b20316c483e42 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 3 Feb 2026 01:23:12 +1300 Subject: [PATCH 101/153] Remove smalltext from non-bots I don't like it. --- src/d2m/converters/message-to-event.js | 6 ++++++ src/d2m/converters/message-to-event.test.js | 15 +++++++++++++++ test/data.js | 21 +++++---------------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index f48146ff..3c953c4c 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -474,6 +474,12 @@ async function messageToEvent(message, guild, options = {}, di) { content = transformAttachmentLinks(content) content = await transformContentMessageLinks(content) + // Remove smalltext from non-bots (I don't like it). Webhooks included due to PluralKit. + const isHumanOrDataMissing = !message.author?.bot + if (isHumanOrDataMissing || dUtils.isWebhookMessage(message)) { + content = content.replaceAll(/^-# +([^\n].*?)/gm, "...$1") + } + // Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter. // So we scan the message ahead of time for all its emojis and ensure they are in the DB. const emojiMatches = [...content.matchAll(/<(a?):([^:>]{1,64}):([0-9]+)>/g)] diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index c30cd1f6..ee64c2de 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -1605,3 +1605,18 @@ test("message2event: multiple-choice poll", async t => { "org.matrix.msc1767.text": "more than one answer allowed\n1. [😭] no\n2. oh no\n3. oh noooooo" }]) }) + +test("message2event: smalltext from regular user", async t => { + const events = await messageToEvent({ + content: "-# hmm", + author: { + bot: false + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + "m.mentions": {}, + body: "...hmm" + }]) +}) diff --git a/test/data.js b/test/data.js index 09749e66..22056767 100644 --- a/test/data.js +++ b/test/data.js @@ -4987,31 +4987,20 @@ module.exports = { edited_timestamp: null, flags: 32768, author: { - id: '772659086046658620', - username: 'cadence.worm', + id: '466378653216014359', + username: 'PluralKit', avatar: '466df0c98b1af1e1388f595b4c1ad1b9', discriminator: '0', public_flags: 0, flags: 0, + bot: true, banner: null, accent_color: null, - global_name: 'cadence', + global_name: 'PluralKit', avatar_decoration_data: null, collectibles: null, display_name_styles: null, - banner_color: null, - clan: { - identity_guild_id: '532245108070809601', - identity_enabled: true, - tag: 'doll', - badge: 'dba08126b4e810a0e096cc7cd5bc37f0' - }, - primary_guild: { - identity_guild_id: '532245108070809601', - identity_enabled: true, - tag: 'doll', - badge: 'dba08126b4e810a0e096cc7cd5bc37f0' - } + banner_color: null }, components: [ { From 5aa112f962586852893c6dfa91de2f5008435e25 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 3 Feb 2026 12:35:16 +1300 Subject: [PATCH 102/153] Better detect reply rep in reply fallback --- src/d2m/converters/message-to-event.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 3c953c4c..92f44c3f 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -572,7 +572,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Content let repliedToContent = referenced.content - if (repliedToContent?.match(/^(-# )?> (-# )?<:L1:/)) { + if (repliedToContent?.match(/^(-# )?> (-# )?quote or -#smalltext >quote. Match until the end of the line. From 15aa6ed5027224c5b810290d9ec983fcfe8eaf24 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 3 Feb 2026 22:25:35 +1300 Subject: [PATCH 103/153] Highlight scanned mentions on Matrix side --- src/d2m/converters/edit-to-changes.test.js | 8 +- src/d2m/converters/find-mentions.js | 157 ++++++++++++++++++ src/d2m/converters/find-mentions.test.js | 118 ++++++++++++++ src/d2m/converters/message-to-event.js | 35 ++-- src/d2m/converters/message-to-event.test.js | 167 +++++++++++++++++++- src/m2d/converters/event-to-message.test.js | 41 +++++ test/test.js | 1 + 7 files changed, 503 insertions(+), 24 deletions(-) create mode 100644 src/d2m/converters/find-mentions.js create mode 100644 src/d2m/converters/find-mentions.test.js diff --git a/src/d2m/converters/edit-to-changes.test.js b/src/d2m/converters/edit-to-changes.test.js index 0fd85b85..cb1fb5a1 100644 --- a/src/d2m/converters/edit-to-changes.test.js +++ b/src/d2m/converters/edit-to-changes.test.js @@ -78,18 +78,18 @@ test("edit2changes: bot response", async t => { newContent: { $type: "m.room.message", msgtype: "m.text", - body: "* :ae_botrac4r: @cadence asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.", + body: "* :ae_botrac4r: [@cadence](https://matrix.to/#/@cadence:cadence.moe) asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.", format: "org.matrix.custom.html", - formatted_body: '* :ae_botrac4r: @cadence asked ­, I respond: Stop drinking paint. (No)

    Hit :bn_re: to reroll.', + formatted_body: '* :ae_botrac4r: @cadence asked ­, I respond: Stop drinking paint. (No)

    Hit :bn_re: to reroll.', "m.mentions": { // Client-Server API spec 11.37.7: Copy Discord's behaviour by not re-notifying anyone that an *edit occurred* }, // *** Replaced With: *** "m.new_content": { msgtype: "m.text", - body: ":ae_botrac4r: @cadence asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.", + body: ":ae_botrac4r: [@cadence](https://matrix.to/#/@cadence:cadence.moe) asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.", format: "org.matrix.custom.html", - formatted_body: ':ae_botrac4r: @cadence asked ­, I respond: Stop drinking paint. (No)

    Hit :bn_re: to reroll.', + formatted_body: ':ae_botrac4r: @cadence asked ­, I respond: Stop drinking paint. (No)

    Hit :bn_re: to reroll.', "m.mentions": { // Client-Server API spec 11.37.7: This should contain the mentions for the final version of the event "user_ids": ["@cadence:cadence.moe"] diff --git a/src/d2m/converters/find-mentions.js b/src/d2m/converters/find-mentions.js new file mode 100644 index 00000000..0f529920 --- /dev/null +++ b/src/d2m/converters/find-mentions.js @@ -0,0 +1,157 @@ +// @ts-check + +const assert = require("assert") + +const {reg} = require("../../matrix/read-registration") +const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) + +/** + * @typedef {{text: string, index: number, end: number}} Token + */ + +/** @typedef {{mxids: {localpart: string, mxid: string, displayname?: string}[], names: {displaynameTokens: Token[], mxid: string}[]}} ProcessedJoined */ + +const lengthBonusLengthCap = 50 +const lengthBonusValue = 0.5 +/** + * Score by how many characters in a row at the start of input are in localpart. 2x if it matches at the start. +1 tiebreaker bonus if it matches all. + * 0 = no match + * @param {string} localpart + * @param {string} input + * @param {string} [displayname] only for the super tiebreaker + * @returns {{score: number, matchedInputTokens: Token[]}} + */ +function scoreLocalpart(localpart, input, displayname) { + let score = 0 + let atStart = false + let matchingLocations = [] + do { + atStart = matchingLocations[0] === 0 + let chars = input[score] + if (score === 0) { + // add all possible places + let i = 0 + while ((i = localpart.indexOf(chars, i)) !== -1) { + matchingLocations.push(i) + i++ + } + } else { + // trim down remaining places + matchingLocations = matchingLocations.filter(i => localpart[i+score] === input[score]) + } + if (matchingLocations.length) { + score++ + if (score === localpart.length) break + } + } while (matchingLocations.length) + /** @type {Token} */ + const fakeToken = {text: input.slice(0, score), index: 0, end: score} + const displaynameLength = displayname?.length ?? 0 + if (score === localpart.length) score = score * 2 + 1 + Math.max(((lengthBonusLengthCap-displaynameLength)/lengthBonusLengthCap)*lengthBonusValue, 0) + else if (atStart) score = score * 2 + return {score, matchedInputTokens: [fakeToken]} +} + +const decayDistance = 10 +const decayValue = 0.33 +/** + * Score by how many tokens in sequence (not necessarily back to back) at the start of input are in display name tokens. Score each token on its length. 2x if it matches at the start. +1 tiebreaker bonus if it matches all + * @param {Token[]} displaynameTokens + * @param {Token[]} inputTokens + * @returns {{score: number, matchedInputTokens: Token[]}} + */ +function scoreName(displaynameTokens, inputTokens) { + let matchedInputTokens = [] + let score = 0 + let searchFrom = 0 + for (let nextInputTokenIndex = 0; nextInputTokenIndex < inputTokens.length; nextInputTokenIndex++) { + // take next + const nextToken = inputTokens[nextInputTokenIndex] + // see if it's there + let foundAt = displaynameTokens.findIndex((tk, idx) => idx >= searchFrom && tk.text === nextToken.text) + if (foundAt !== -1) { + // update scoring + matchedInputTokens.push(nextToken) + score += nextToken.text.length * Math.max(((decayDistance-foundAt)*(1+decayValue))/(decayDistance*(1+decayValue)), decayValue) // decay score 100%->33% the further into the displayname it's found + // prepare for next loop + searchFrom = foundAt + 1 + } else { + break + } + } + const firstTextualInputToken = inputTokens.find(t => t.text.match(/^\w/)) + if (matchedInputTokens[0] === inputTokens[0] || matchedInputTokens[0] === firstTextualInputToken) score *= 2 + if (matchedInputTokens.length === displaynameTokens.length) score += 1 + return {score, matchedInputTokens} +} + +/** + * @param {string} name + * @returns {Token[]} + */ +function tokenise(name) { + let index = 0 + let result = [] + for (const part of name.split(/(_|\s|\b)/g)) { + if (part.trim()) { + result.push({text: part.toLowerCase(), index, end: index + part.length}) + } + index += part.length + } + return result +} + +/** + * @param {{mxid: string, displayname?: string}[]} joined + * @returns {ProcessedJoined} + */ +function processJoined(joined) { + joined = joined.filter(j => !userRegex.some(rx => j.mxid.match(rx))) + return { + mxids: joined.map(j => { + const localpart = j.mxid.match(/@([^:]*)/) + assert(localpart) + return { + localpart: localpart[1].toLowerCase(), + mxid: j.mxid, + displayname: j.displayname + } + }), + names: joined.filter(j => j.displayname).map(j => { + return { + displaynameTokens: tokenise(j.displayname), + mxid: j.mxid + } + }) + } +} + +/** + * @param {ProcessedJoined} pjr + * @param {string} maximumWrittenSection lowercase please + * @param {string} content + */ +function findMention(pjr, maximumWrittenSection, baseOffset, prefix, content) { + if (!pjr.mxids.length && !pjr.names.length) return + const maximumWrittenSectionTokens = tokenise(maximumWrittenSection) + /** @type {{mxid: string, scored: {score: number, matchedInputTokens: Token[]}}[]} */ + let allItems = pjr.mxids.map(mxid => ({...mxid, scored: scoreLocalpart(mxid.localpart, maximumWrittenSection, mxid.displayname)})) + allItems = allItems.concat(pjr.names.map(name => ({...name, scored: scoreName(name.displaynameTokens, maximumWrittenSectionTokens)}))) + const best = allItems.sort((a, b) => b.scored.score - a.scored.score)[0] + if (best.scored.score > 4) { // requires in smallest case perfect match of 2 characters, or in largest case a partial middle match of 5+ characters in a row + // Highlight the relevant part of the message + const start = baseOffset + best.scored.matchedInputTokens[0].index + const end = baseOffset + prefix.length + best.scored.matchedInputTokens.at(-1).end + const newContent = content.slice(0, start) + "[" + content.slice(start, end) + "](https://matrix.to/#/" + best.mxid + ")" + content.slice(end) + return { + mxid: best.mxid, + newContent + } + } +} + +module.exports.scoreLocalpart = scoreLocalpart +module.exports.scoreName = scoreName +module.exports.tokenise = tokenise +module.exports.processJoined = processJoined +module.exports.findMention = findMention diff --git a/src/d2m/converters/find-mentions.test.js b/src/d2m/converters/find-mentions.test.js new file mode 100644 index 00000000..fc950e33 --- /dev/null +++ b/src/d2m/converters/find-mentions.test.js @@ -0,0 +1,118 @@ +// @ts-check + +const {test} = require("supertape") +const {scoreLocalpart, scoreName, tokenise} = require("./find-mentions") + +test("score localpart: score against cadence", t => { + const localparts = [ + "cadence", + "cadence_test", + "roblkyogre", + "cat", + "arcade_cabinet" + ] + t.deepEqual(localparts.map(l => scoreLocalpart(l, "cadence").score), [ + 15.5, + 14, + 0, + 4, + 4 + ]) +}) + +test("score mxid: tiebreak multiple perfect matches on name length", t => { + const users = [ + {displayname: "Emma [it/its] ⚡️", localpart: "emma"}, + {displayname: "Emma [it/its]", localpart: "emma"} + ] + const results = users.map(u => scoreLocalpart(u.localpart, "emma", u.displayname).score) + t.ok(results[0] < results[1], `comparison: ${results.join(" < ")}`) +}) + +test("score name: score against cadence", t => { + const names = [ + "bgt lover", + "Ash 🦑 (xey/it)", + "Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆", + "underscore_idiot #sunshine", + "INX | Evil Lillith (she/her)", + "INX | Lillith (she/her)", + "🌟luna🌟", + "#1 Ritsuko Kinnie" + ] + t.deepEqual(names.map(n => scoreName(tokenise(n), tokenise("cadence")).score), [ + 0, + 0, + 14, + 0, + 0, + 0, + 0, + 0 + ]) +}) + +test("score name: nothing scored after a token doesn't match", t => { + const names = [ + "bgt lover", + "Ash 🦑 (xey/it)", + "Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆", + "underscore_idiot #sunshine", + "INX | Evil Lillith (she/her)", + "INX | Lillith (she/her)", + "🌟luna🌟", + "#1 Ritsuko Kinnie" + ] + t.deepEqual(names.map(n => scoreName(tokenise(n), tokenise("I hope so")).score), [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ]) +}) + +test("score name: prefers earlier match", t => { + const names = [ + "INX | Lillith (she/her)", + "INX | Evil Lillith (she/her)" + ] + const results = names.map(n => scoreName(tokenise(n), tokenise("lillith")).score) + t.ok(results[0] > results[1], `comparison: ${results.join(" > ")}`) +}) + +test("score name: matches lots of tokens", t => { + t.deepEqual( + Math.round(scoreName(tokenise("Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆"), tokenise("cadence maid of creation eye of clarity empress of hope")).score), + 50 + ) +}) + +test("score name: prefers variation when you specify it", t => { + const names = [ + "Cadence (test account)", + "Cadence" + ] + const results = names.map(n => scoreName(tokenise(n), tokenise("cadence test")).score) + t.ok(results[0] > results[1], `comparison: ${results.join(" > ")}`) +}) + +test("score name: prefers original when not specified", t => { + const names = [ + "Cadence (test account)", + "Cadence" + ] + const results = names.map(n => scoreName(tokenise(n), tokenise("cadence")).score) + t.ok(results[0] < results[1], `comparison: ${results.join(" < ")}`) +}) + +test("score name: finds match location", t => { + const message = "evil lillith is an inspiration" + const result = scoreName(tokenise("INX | Evil Lillith (she/her)"), tokenise(message)) + const startLocation = result.matchedInputTokens[0].index + const endLocation = result.matchedInputTokens.at(-1).end + t.equal(message.slice(startLocation, endLocation), "evil lillith") +}) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 92f44c3f..c049f181 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -18,10 +18,10 @@ const lottie = sync.require("../actions/lottie") const mxUtils = sync.require("../../matrix/utils") /** @type {import("../../discord/utils")} */ const dUtils = sync.require("../../discord/utils") +/** @type {import("./find-mentions")} */ +const findMentions = sync.require("./find-mentions") const {reg} = require("../../matrix/read-registration") -const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) - /** * @param {DiscordTypes.APIMessage} message * @param {DiscordTypes.APIGuild} guild @@ -684,23 +684,28 @@ async function messageToEvent(message, guild, options = {}, di) { // Then text content if (message.content) { // Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention. - const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)] - if (options.scanTextForMentions !== false && matches.length && matches.some(m => m[1].match(/[a-z]/i) && m[1] !== "everyone" && m[1] !== "here")) { - const writtenMentionsText = matches.map(m => m[1].toLowerCase()) - const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get() - assert(roomID) - const {joined} = await di.api.getJoinedMembers(roomID) - for (const [mxid, member] of Object.entries(joined)) { - if (!userRegex.some(rx => mxid.match(rx))) { - const localpart = mxid.match(/@([^:]*)/) - assert(localpart) - const displayName = member.display_name || localpart[1] - if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid) + let content = message.content + if (options.scanTextForMentions !== false) { + const matches = [...content.matchAll(/(@ ?)([a-z0-9_.][^@\n]+)/gi)] + for (let i = matches.length; i--;) { + const m = matches[i] + const prefix = m[1] + const maximumWrittenSection = m[2].toLowerCase() + if (maximumWrittenSection.match(/^!?&?[0-9]+>/) || maximumWrittenSection.match(/^everyone\b/) || maximumWrittenSection.match(/^here\b/)) continue + + var roomID = roomID ?? select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get() + assert(roomID) + var pjr = pjr ?? findMentions.processJoined(Object.entries((await di.api.getJoinedMembers(roomID)).joined).map(([mxid, ev]) => ({mxid, displayname: ev.display_name}))) + + const found = findMentions.findMention(pjr, maximumWrittenSection, m.index, prefix, content) + if (found) { + addMention(found.mxid) + content = found.newContent } } } - const {body, html} = await transformContent(message.content) + const {body, html} = await transformContent(content) await addTextEvent(body, html, msgtype) } diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index ee64c2de..7a7d86f3 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -789,11 +789,13 @@ test("message2event: simple written @mention for matrix user", async t => { ] }, msgtype: "m.text", - body: "@ash do you need anything from the store btw as I'm heading there after gym" + body: "[@ash](https://matrix.to/#/@she_who_brings_destruction:cadence.moe) do you need anything from the store btw as I'm heading there after gym", + format: "org.matrix.custom.html", + formatted_body: `@ash do you need anything from the store btw as I'm heading there after gym` }]) }) -test("message2event: advanced written @mentions for matrix users", async t => { +test("message2event: many written @mentions for matrix users", async t => { let called = 0 const events = await messageToEvent(data.message.advanced_written_at_mention_for_matrix, data.guild.general, {}, { api: { @@ -831,16 +833,171 @@ test("message2event: advanced written @mentions for matrix users", async t => { $type: "m.room.message", "m.mentions": { user_ids: [ - "@cadence:cadence.moe", - "@huckleton:cadence.moe" + "@huckleton:cadence.moe", + "@cadence:cadence.moe" ] }, msgtype: "m.text", - body: "@Cadence, tell me about @Phil, the creator of the Chin Trick, who has become ever more powerful under the mentorship of @botrac4r and @huck" + body: "[@Cadence](https://matrix.to/#/@cadence:cadence.moe), tell me about @Phil, the creator of the Chin Trick, who has become ever more powerful under the mentorship of @botrac4r and [@huck](https://matrix.to/#/@huckleton:cadence.moe)", + format: "org.matrix.custom.html", + formatted_body: `@Cadence, tell me about @Phil, the creator of the Chin Trick, who has become ever more powerful under the mentorship of @botrac4r and @huck` }]) t.equal(called, 1, "should only look up the member list once") }) +test("message2event: written @mentions may match part of the name", async t => { + let called = 0 + const events = await messageToEvent({ + ...data.message.advanced_written_at_mention_for_matrix, + content: "I wonder if @cadence saw this?" + }, data.guild.general, {}, { + api: { + async getJoinedMembers(roomID) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + return new Promise(resolve => { + setTimeout(() => { + resolve({ + joined: { + "@secret:cadence.moe": { + display_name: "cadence [they]", + avatar_url: "whatever" + }, + "@huckleton:cadence.moe": { + display_name: "huck", + avatar_url: "whatever" + }, + "@_ooye_botrac4r:cadence.moe": { + display_name: "botrac4r", + avatar_url: "whatever" + }, + "@_ooye_bot:cadence.moe": { + display_name: "Out Of Your Element", + avatar_url: "whatever" + } + } + }) + }) + }) + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": { + user_ids: [ + "@secret:cadence.moe", + ] + }, + msgtype: "m.text", + body: "I wonder if [@cadence](https://matrix.to/#/@secret:cadence.moe) saw this?", + format: "org.matrix.custom.html", + formatted_body: `I wonder if @cadence saw this?` + }]) +}) + +test("message2event: written @mentions may match part of the mxid", async t => { + let called = 0 + const events = await messageToEvent({ + ...data.message.advanced_written_at_mention_for_matrix, + content: "I wonder if @huck saw this?" + }, data.guild.general, {}, { + api: { + async getJoinedMembers(roomID) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + return new Promise(resolve => { + setTimeout(() => { + resolve({ + joined: { + "@cadence:cadence.moe": { + display_name: "cadence [they]", + avatar_url: "whatever" + }, + "@huckleton:cadence.moe": { + display_name: "wa", + avatar_url: "whatever" + }, + "@_ooye_botrac4r:cadence.moe": { + display_name: "botrac4r", + avatar_url: "whatever" + }, + "@_ooye_bot:cadence.moe": { + display_name: "Out Of Your Element", + avatar_url: "whatever" + } + } + }) + }) + }) + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": { + user_ids: [ + "@huckleton:cadence.moe", + ] + }, + msgtype: "m.text", + body: "I wonder if [@huck](https://matrix.to/#/@huckleton:cadence.moe) saw this?", + format: "org.matrix.custom.html", + formatted_body: `I wonder if @huck saw this?` + }]) +}) + +test("message2event: entire message may match elaborate display name", async t => { + let called = 0 + const events = await messageToEvent({ + ...data.message.advanced_written_at_mention_for_matrix, + content: "@Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆" + }, data.guild.general, {}, { + api: { + async getJoinedMembers(roomID) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + return new Promise(resolve => { + setTimeout(() => { + resolve({ + joined: { + "@wa:cadence.moe": { + display_name: "Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆", + avatar_url: "whatever" + }, + "@huckleton:cadence.moe": { + display_name: "huck", + avatar_url: "whatever" + }, + "@_ooye_botrac4r:cadence.moe": { + display_name: "botrac4r", + avatar_url: "whatever" + }, + "@_ooye_bot:cadence.moe": { + display_name: "Out Of Your Element", + avatar_url: "whatever" + } + } + }) + }) + }) + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": { + user_ids: [ + "@wa:cadence.moe", + ] + }, + msgtype: "m.text", + body: "[@Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆](https://matrix.to/#/@wa:cadence.moe)", + format: "org.matrix.custom.html", + formatted_body: `@Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆` + }]) +}) + test("message2event: spoilers are removed from plaintext body", async t => { const events = await messageToEvent({ content: "||**beatrice**||" diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 5cdf4aff..551cbd06 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -3314,6 +3314,47 @@ test("event2message: mentioning matrix users works", async t => { ) }) +test("event2message: matrix mentions are not double-escaped when embed links permission is denied", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `I'm just ▲ testing mentions` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }, { + id: "123", + roles: [{ + id: "123", + name: "@everyone", + permissions: DiscordTypes.PermissionFlagsBits.SendMessages + }] + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "I'm just [@▲]() testing mentions", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + test("event2message: multiple mentions are both escaped", async t => { t.deepEqual( await eventToMessage({ diff --git a/test/test.js b/test/test.js index 81c079a4..0bb1da4a 100644 --- a/test/test.js +++ b/test/test.js @@ -158,6 +158,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/d2m/actions/register-user.test") require("../src/d2m/converters/edit-to-changes.test") require("../src/d2m/converters/emoji-to-key.test") + require("../src/d2m/converters/find-mentions.test") require("../src/d2m/converters/lottie.test") require("../src/d2m/converters/message-to-event.test") require("../src/d2m/converters/message-to-event.test.components") From 238e911d13d60b83a6d74456581ad7445230ea61 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 3 Feb 2026 22:26:00 +1300 Subject: [PATCH 104/153] Fix m->d double-escaping of Matrix mentions --- src/m2d/converters/event-to-message.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index b03de95a..90ac0856 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -138,7 +138,9 @@ turndownService.addRule("inlineLink", { if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}` if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>` const href = node.getAttribute("href") - const suppressedHref = node.hasAttribute("data-suppress") ? "<" + href + ">" : href + let shouldSuppress = node.hasAttribute("data-suppress") + if (href.match(/^https?:\/\/matrix.to\//)) shouldSuppress = false // avoid double-escaping + const suppressedHref = shouldSuppress ? "<" + href + ">" : href content = content.replace(/ @.*/, "") if (href === content) return suppressedHref if (decodeURIComponent(href).startsWith("https://matrix.to/#/@") && content[0] !== "@") content = "@" + content From c73800f785d02b1a2c375453cd63c37d48def316 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 3 Feb 2026 22:58:42 +1300 Subject: [PATCH 105/153] Fix U+FE0F and tweak decay to fix tie result --- src/d2m/converters/find-mentions.js | 5 +++-- src/d2m/converters/find-mentions.test.js | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/d2m/converters/find-mentions.js b/src/d2m/converters/find-mentions.js index 0f529920..9db6355f 100644 --- a/src/d2m/converters/find-mentions.js +++ b/src/d2m/converters/find-mentions.js @@ -52,7 +52,7 @@ function scoreLocalpart(localpart, input, displayname) { return {score, matchedInputTokens: [fakeToken]} } -const decayDistance = 10 +const decayDistance = 20 const decayValue = 0.33 /** * Score by how many tokens in sequence (not necessarily back to back) at the start of input are in display name tokens. Score each token on its length. 2x if it matches at the start. +1 tiebreaker bonus if it matches all @@ -90,11 +90,12 @@ function scoreName(displaynameTokens, inputTokens) { * @returns {Token[]} */ function tokenise(name) { + name = name.replaceAll("\ufe0f", "").normalize().toLowerCase() let index = 0 let result = [] for (const part of name.split(/(_|\s|\b)/g)) { if (part.trim()) { - result.push({text: part.toLowerCase(), index, end: index + part.length}) + result.push({text: part, index, end: index + part.length}) } index += part.length } diff --git a/src/d2m/converters/find-mentions.test.js b/src/d2m/converters/find-mentions.test.js index fc950e33..0d02285d 100644 --- a/src/d2m/converters/find-mentions.test.js +++ b/src/d2m/converters/find-mentions.test.js @@ -1,7 +1,7 @@ // @ts-check const {test} = require("supertape") -const {scoreLocalpart, scoreName, tokenise} = require("./find-mentions") +const {processJoined, scoreLocalpart, scoreName, tokenise, findMention} = require("./find-mentions") test("score localpart: score against cadence", t => { const localparts = [ @@ -87,7 +87,7 @@ test("score name: prefers earlier match", t => { test("score name: matches lots of tokens", t => { t.deepEqual( Math.round(scoreName(tokenise("Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆"), tokenise("cadence maid of creation eye of clarity empress of hope")).score), - 50 + 65 ) }) @@ -116,3 +116,14 @@ test("score name: finds match location", t => { const endLocation = result.matchedInputTokens.at(-1).end t.equal(message.slice(startLocation, endLocation), "evil lillith") }) + +test("find mention: test various tiebreakers", t => { + const found = findMention(processJoined([{ + mxid: "@emma:conduit.rory.gay", + displayname: "Emma [it/its] ⚡️" + }, { + mxid: "@emma:rory.gay", + displayname: "Emma [it/its]" + }]), "emma ⚡ curious which one this prefers", 0, "@", "@emma ⚡ curious which one this prefers") + t.equal(found.mxid, "@emma:conduit.rory.gay") +}) From b52b2de205772c0b452256233853a1830f0b1be3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 4 Feb 2026 00:45:23 +1300 Subject: [PATCH 106/153] Customise format for Klipy GIFs --- src/d2m/converters/message-to-event.js | 26 +++++++- .../message-to-event.test.embeds.js | 12 ++++ src/matrix/utils.js | 7 ++- test/data.js | 63 +++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index c049f181..5778c028 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -630,6 +630,16 @@ async function messageToEvent(message, guild, options = {}, di) { message.content = `added a new emoji, ${message.content} :${name}:` } + // Send Klipy GIFs in customised form + let isKlipyGIF = false + let isOnlyKlipyGIF = false + if (message.embeds?.length === 1 && message.embeds[0].provider?.name === "Klipy" && message.embeds[0].video?.url) { + isKlipyGIF = true + if (message.content.match(/^https?:\/\/klipy\.com[^ \n]+$/)) { + isOnlyKlipyGIF = true + } + } + // Forwarded content appears first if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_snapshots?.length) { // Forwarded notice @@ -682,7 +692,7 @@ async function messageToEvent(message, guild, options = {}, di) { } // Then text content - if (message.content) { + if (message.content && !isOnlyKlipyGIF) { // Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention. let content = message.content if (options.scanTextForMentions !== false) { @@ -905,6 +915,20 @@ async function messageToEvent(message, guild, options = {}, di) { // Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once const rep = new mxUtils.MatrixStringBuilder() + if (isKlipyGIF) { + rep.add("[GIF] ", "➿ ") + if (embed.title) { + rep.add(`${embed.title} ${embed.video.url}`, tag`${embed.title}`) + } else { + rep.add(embed.video.url) + } + + let {body, formatted_body: html} = rep.get() + html = `
    ${html}
    ` + await addTextEvent(body, html, "m.text") + continue + } + // Provider if (embed.provider?.name && embed.provider.name !== "Tenor") { if (embed.provider.url) { diff --git a/src/d2m/converters/message-to-event.test.embeds.js b/src/d2m/converters/message-to-event.test.embeds.js index 48959930..cfb2f960 100644 --- a/src/d2m/converters/message-to-event.test.embeds.js +++ b/src/d2m/converters/message-to-event.test.embeds.js @@ -346,6 +346,18 @@ test("message2event embeds: tenor gif should show a video link without a provide }]) }) +test("message2event embeds: klipy gif should send in customised format", async t => { + const events = await messageToEvent(data.message_with_embeds.klipy_gif, data.guild.general, {}, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "[GIF] Cute Corgi Waddle https://static.klipy.com/ii/d7aec6f6f171607374b2065c836f92f4/5b/5b/7ndEhcilPNKJ8O.mp4", + format: "org.matrix.custom.html", + formatted_body: "
    ➿ Cute Corgi Waddle
    ", + "m.mentions": {} + }]) +}) + test("message2event embeds: if discord creates an embed preview for a discord channel link, don't copy that embed", async t => { const events = await messageToEvent(data.message_with_embeds.discord_server_included_punctuation_bad_discord, data.guild.general, {}, { api: { diff --git a/src/matrix/utils.js b/src/matrix/utils.js index b1315102..d89c9682 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -2,6 +2,7 @@ const assert = require("assert").strict const Ty = require("../types") +const {tag} = require("@cloudrac3r/html-template-tag") const passthrough = require("../passthrough") const {db} = passthrough @@ -72,7 +73,7 @@ class MatrixStringBuilder { */ add(body, formattedBody, condition = true) { if (condition) { - if (formattedBody == undefined) formattedBody = body + if (formattedBody == undefined) formattedBody = tag`${body}` this.body += body this.formattedBody += formattedBody } @@ -86,7 +87,7 @@ class MatrixStringBuilder { */ addLine(body, formattedBody, condition = true) { if (condition) { - if (formattedBody == undefined) formattedBody = body + if (formattedBody == undefined) formattedBody = tag`${body}` if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n" this.body += body const match = this.formattedBody.match(/<\/?([a-zA-Z]+[a-zA-Z0-9]*)[^>]*>\s*$/) @@ -103,7 +104,7 @@ class MatrixStringBuilder { */ addParagraph(body, formattedBody, condition = true) { if (condition) { - if (formattedBody == undefined) formattedBody = body + if (formattedBody == undefined) formattedBody = tag`${body}` if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n\n" this.body += body const match = formattedBody.match(/^<([a-zA-Z]+[a-zA-Z0-9]*)/) diff --git a/test/data.js b/test/data.js index 22056767..4854f6a5 100644 --- a/test/data.js +++ b/test/data.js @@ -4916,6 +4916,69 @@ module.exports = { flags: 0, components: [] }, + klipy_gif: { + type: 0, + content: "https://klipy.com/gifs/cute-15", + mentions: [], + mention_roles: [], + attachments: [], + embeds: [ + { + type: "gifv", + url: "https://klipy.com/gifs/cute-15", + title: "Cute Corgi Waddle", + provider: { + name: "Klipy", + url: "https://klipy.com" + }, + thumbnail: { + url: "https://static.klipy.com/ii/d7aec6f6f171607374b2065c836f92f4/5b/5b/xHVF6sVV.webp", + proxy_url: "https://images-ext-1.discordapp.net/external/Z54QmlQflPPb6NoXikflBHGmttgRm3_jhzmcILXHhcA/https/static.klipy.com/ii/d7aec6f6f171607374b2065c836f92f4/5b/5b/xHVF6sVV.webp", + width: 277, + height: 498, + placeholder: "3gcGDAJV+WZYl3RpZ2gGeFBxBw==", + placeholder_version: 1, + flags: 0 + }, + video: { + url: "https://static.klipy.com/ii/d7aec6f6f171607374b2065c836f92f4/5b/5b/7ndEhcilPNKJ8O.mp4", + proxy_url: "https://images-ext-1.discordapp.net/external/xZspzkQPUKBa74pBhJDpBf3v2d3d0lC943xaB9_JnoM/https/static.klipy.com/ii/d7aec6f6f171607374b2065c836f92f4/5b/5b/7ndEhcilPNKJ8O.mp4", + width: 356, + height: 640, + placeholder: "3gcGDAJV+WZYl3RpZ2gGeFBxBw==", + placeholder_version: 1, + flags: 0 + }, + content_scan_version: 4 + } + ], + timestamp: "2026-02-03T11:11:50.070000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + id: "1468202316233707613", + channel_id: "1370776315266859131", + author: { + id: "304655299631906816", + username: "witterson", + avatar: "47ec94a1b2b4cc41ce0329b3575e9b66", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "wit", + avatar_decoration_data: null, + collectibles: null, + display_name_styles: null, + banner_color: null, + clan: null, + primary_guild: null + }, + pinned: false, + mention_everyone: false, + tts: false + }, tenor_gif: { type: 0, content: "<@&1182745800661540927> get real https://tenor.com/view/get-real-gif-26176788", From f5d50fc14ec4b5c45ff6854172fbc975222302ac Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 4 Feb 2026 00:59:23 +1300 Subject: [PATCH 107/153] Properly stop PluralKit users typing after sending --- src/d2m/actions/send-message.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index 3005ca8c..a6426b88 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -69,7 +69,8 @@ async function sendMessage(message, channel, guild, row) { const eventIDs = [] if (events.length) { db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(message.id, historicalRoomIndex) - if (senderMxid) api.sendTyping(roomID, false, senderMxid).catch(() => {}) + const typingMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() + if (typingMxid) api.sendTyping(roomID, false, typingMxid).catch(() => {}) } for (const event of events) { const part = event === events[0] ? 0 : 1 From 6032ba41999326312446dee3c8c3b6496c4e9b9f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 4 Feb 2026 01:27:31 +1300 Subject: [PATCH 108/153] Support MSC3725-style spoilers --- src/m2d/converters/event-to-message.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 90ac0856..9ac174e9 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -563,8 +563,8 @@ async function eventToMessage(event, guild, channel, di) { let shouldProcessTextEvent = event.type === "m.room.message" && (event.content.msgtype === "m.text" || event.content.msgtype === "m.emote") if (event.type === "m.room.message" && (event.content.msgtype === "m.file" || event.content.msgtype === "m.video" || event.content.msgtype === "m.audio" || event.content.msgtype === "m.image")) { // Build message content in addition to the uploaded file - const fileIsSpoiler = event.content["page.codeberg.everypizza.msc4193.spoiler"] - const fileSpoilerReason = event.content["page.codeberg.everypizza.msc4193.spoiler.reason"] + const fileIsSpoiler = event.content["page.codeberg.everypizza.msc4193.spoiler"] || event.content["town.robin.msc3725.content_warning"] + const fileSpoilerReason = event.content["page.codeberg.everypizza.msc4193.spoiler.reason"] || event.content["town.robin.msc3725.content_warning"]?.description content = "" const captionContent = new mxUtils.MatrixStringBuilder() From c01e347e7b9d4a43f2aa268554e9ac99e6f8eeec Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 4 Feb 2026 12:11:46 +1300 Subject: [PATCH 109/153] Allow more characters at start of scanned mentions --- src/d2m/converters/message-to-event.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 5778c028..9236c893 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -696,7 +696,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention. let content = message.content if (options.scanTextForMentions !== false) { - const matches = [...content.matchAll(/(@ ?)([a-z0-9_.][^@\n]+)/gi)] + const matches = [...content.matchAll(/(@ ?)([a-z0-9_.#$][^@\n]+)/gi)] for (let i = matches.length; i--;) { const m = matches[i] const prefix = m[1] From aa7222c4ed7c2ceecf8fadbf676874d909dc3b04 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 4 Feb 2026 12:56:52 +1300 Subject: [PATCH 110/153] Print d->m errors when there is no room --- src/d2m/event-dispatcher.js | 6 ++++-- src/m2d/event-dispatcher.js | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index c25d1c6f..3392eb98 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -51,13 +51,15 @@ module.exports = { * @param {import("cloudstorm").IGatewayMessage} gatewayMessage */ async onError(client, e, gatewayMessage) { + if (gatewayMessage.t === "TYPING_START") return + + matrixEventDispatcher.printError(gatewayMessage.t, "Discord", e, gatewayMessage) + const channelID = gatewayMessage.d["channel_id"] if (!channelID) return const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() if (!roomID) return - if (gatewayMessage.t === "TYPING_START") return - await matrixEventDispatcher.sendError(roomID, "Discord", gatewayMessage.t, e, gatewayMessage) }, diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 48bff119..57c0fa64 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -88,6 +88,12 @@ function stringifyErrorStack(err, depth = 0) { return collapsed; } +function printError(type, source, e, payload) { + console.error(`Error while processing a ${type} ${source} event:`) + console.error(e) + console.dir(payload, {depth: null}) +} + /** * @param {string} roomID * @param {"Discord" | "Matrix"} source @@ -96,9 +102,9 @@ function stringifyErrorStack(err, depth = 0) { * @param {any} payload */ async function sendError(roomID, source, type, e, payload) { - console.error(`Error while processing a ${type} ${source} event:`) - console.error(e) - console.dir(payload, {depth: null}) + if (source === "Matrix") { + printError(type, source, e, payload) + } if (Date.now() - lastReportedEvent < 5000) return null lastReportedEvent = Date.now() @@ -457,3 +463,4 @@ async event => { module.exports.stringifyErrorStack = stringifyErrorStack module.exports.sendError = sendError +module.exports.printError = printError From 52d9c6fea812a4fd92635476ae5b5be27900c233 Mon Sep 17 00:00:00 2001 From: Ellie Algase Date: Tue, 3 Feb 2026 18:00:53 -0600 Subject: [PATCH 111/153] Fix poll results being double-bridged Oddly, this would only occur for the first poll in a channel. --- src/d2m/actions/send-message.js | 6 ++++-- src/d2m/converters/edit-to-changes.js | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index a6426b88..283f2ec8 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -55,13 +55,14 @@ async function sendMessage(message, channel, guild, row) { } } + let sentResultsMessage if (message.type === DiscordTypes.MessageType.PollResult) { // ensure all Discord-side votes were pushed to Matrix before a poll is closed const detailedResultsMessage = await pollEnd.endPoll(message) if (detailedResultsMessage) { const threadParent = select("channel_room", "thread_parent", {channel_id: message.channel_id}).pluck().get() const channelID = threadParent ? threadParent : message.channel_id const threadID = threadParent ? message.channel_id : undefined - var sentResultsMessage = await channelWebhook.sendMessageWithWebhook(channelID, detailedResultsMessage, threadID) + sentResultsMessage = await channelWebhook.sendMessageWithWebhook(channelID, detailedResultsMessage, threadID) } } @@ -118,7 +119,8 @@ async function sendMessage(message, channel, guild, row) { db.transaction(() => { db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(sentResultsMessage.id, historicalRoomIndex) db.prepare("UPDATE event_message SET reaction_part = 1 WHERE event_id = ?").run(eventID) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, sentResultsMessage.id, 1, 0) // part = 1, reaction_part = 0 + // part = 1, reaction_part = 0, source = 0 as the results are "from Matrix" and doing otherwise breaks things when that message gets updated by Discord (it just does that sometimes) + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(eventID, eventType, event.msgtype || null, sentResultsMessage.id, 1, 0) })() } diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index 86a40287..82b9417f 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -45,7 +45,7 @@ async function editToChanges(message, guild, api) { // Since an update in August 2024, the system always provides the full data of message updates. // Now, this code path is only used by generated embeds for messages that were originally sent from Matrix. - const originallyFromMatrix = oldEventRows.find(r => r.part === 0)?.source === 0 + const originallyFromMatrix = oldEventRows.some(r => r.source === 0) const mightBeGeneratedEmbed = !("content" in message) || originallyFromMatrix // Figure out who to send as From 564d564490548d5ef062de7512bf8d9059e84c86 Mon Sep 17 00:00:00 2001 From: Ellie Algase Date: Tue, 3 Feb 2026 19:23:01 -0600 Subject: [PATCH 112/153] Add command to see Matrix results mid-poll Co-authored-by: Cadence Ember --- src/d2m/actions/poll-end.js | 36 ++------- src/discord/interactions/poll-responses.js | 94 ++++++++++++++++++++++ src/discord/register-interactions.js | 13 ++- 3 files changed, 112 insertions(+), 31 deletions(-) create mode 100644 src/discord/interactions/poll-responses.js diff --git a/src/d2m/actions/poll-end.js b/src/d2m/actions/poll-end.js index 936dedf2..9ffcaf68 100644 --- a/src/d2m/actions/poll-end.js +++ b/src/d2m/actions/poll-end.js @@ -9,22 +9,15 @@ const {discord, sync, db, select, from} = passthrough const {reg} = require("../../matrix/read-registration") /** @type {import("./poll-vote")} */ const vote = sync.require("../actions/poll-vote") -/** @type {import("../../m2d/converters/poll-components")} */ -const pollComponents = sync.require("../../m2d/converters/poll-components") - -// This handles, in the following order: -// * verifying Matrix-side votes are accurate for a poll originating on Discord, sending missed votes to Matrix if necessary -// * sending a message to Discord if a vote in that poll has been cast on Matrix -// This does *not* handle bridging of poll closures on Discord to Matrix; that takes place in converters/message-to-event.js. +/** @type {import("../../discord/interactions/poll-responses")} */ +const pollResponses = sync.require("../../discord/interactions/poll-responses") /** - * @param {number} percent + * @file This handles, in the following order: + * * verifying Matrix-side votes are accurate for a poll originating on Discord, sending missed votes to Matrix if necessary + * * sending a message to Discord if a vote in that poll has been cast on Matrix + * This does *not* handle bridging of poll closures on Discord to Matrix; that takes place in converters/message-to-event.js. */ -function barChart(percent) { - const width = 12 - const bars = Math.floor(percent*width) - return "█".repeat(bars) + "▒".repeat(width-bars) -} /** * @param {string} channelID @@ -114,22 +107,9 @@ async function endPoll(closeMessage) { })) } - /** @type {{matrix_option: string, option_text: string, count: number}[]} */ - const pollResults = db.prepare("SELECT matrix_option, option_text, seq, count(discord_or_matrix_user_id) as count FROM poll_option LEFT JOIN poll_vote USING (message_id, matrix_option) WHERE message_id = ? GROUP BY matrix_option ORDER BY seq").all(pollMessageID) - const combinedVotes = pollResults.reduce((a, c) => a + c.count, 0) - const totalVoters = db.prepare("SELECT count(DISTINCT discord_or_matrix_user_id) as count FROM poll_vote WHERE message_id = ?").pluck().get(pollMessageID) + const {combinedVotes, messageString} = pollResponses.getCombinedResults(pollMessageID, true) - if (combinedVotes !== totalVotes) { // This means some votes were cast on Matrix! - // Now that we've corrected the vote totals, we can get the results again and post them to Discord! - const topAnswers = pollResults.toSorted((a, b) => b.count - a.count) - let messageString = "" - for (const option of pollResults) { - const medal = pollComponents.getMedal(topAnswers, option.count) - const countString = `${String(option.count).padStart(String(topAnswers[0].count).length)}` - const votesString = option.count === 1 ? "vote " : "votes" - const label = medal === "🥇" ? `**${option.option_text}**` : option.option_text - messageString += `\`\u200b${countString} ${votesString}\u200b\` ${barChart(option.count/totalVoters)} ${label} ${medal}\n` - } + if (combinedVotes !== totalVotes) { // This means some votes were cast on Matrix. Now that we've corrected the vote totals, we can get the results again and post them to Discord. return { username: "Total results including Matrix votes", avatar_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png`, diff --git a/src/discord/interactions/poll-responses.js b/src/discord/interactions/poll-responses.js new file mode 100644 index 00000000..86277c28 --- /dev/null +++ b/src/discord/interactions/poll-responses.js @@ -0,0 +1,94 @@ +// @ts-check + +const DiscordTypes = require("discord-api-types/v10") +const {discord, sync, db, select, from} = require("../../passthrough") +const {id: botID} = require("../../../addbot") +const {InteractionMethods} = require("snowtransfer") + +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../m2d/converters/poll-components")} */ +const pollComponents = sync.require("../../m2d/converters/poll-components") +const {reg} = require("../../matrix/read-registration") + +/** + * @param {number} percentc + */ +function barChart(percent) { + const width = 12 + const bars = Math.floor(percent*width) + return "█".repeat(bars) + "▒".repeat(width-bars) +} + +/** + * @param {string} pollMessageID + * @param {boolean} isClosed + */ +function getCombinedResults(pollMessageID, isClosed) { + /** @type {{matrix_option: string, option_text: string, count: number}[]} */ + const pollResults = db.prepare("SELECT matrix_option, option_text, seq, count(discord_or_matrix_user_id) as count FROM poll_option LEFT JOIN poll_vote USING (message_id, matrix_option) WHERE message_id = ? GROUP BY matrix_option ORDER BY seq").all(pollMessageID) + const combinedVotes = pollResults.reduce((a, c) => a + c.count, 0) + const totalVoters = db.prepare("SELECT count(DISTINCT discord_or_matrix_user_id) as count FROM poll_vote WHERE message_id = ?").pluck().get(pollMessageID) + const topAnswers = pollResults.toSorted((a, b) => b.count - a.count) + + let messageString = "" + for (const option of pollResults) { + const medal = isClosed ? pollComponents.getMedal(topAnswers, option.count) : "" + const countString = `${String(option.count).padStart(String(topAnswers[0].count).length)}` + const votesString = option.count === 1 ? "vote " : "votes" + const label = medal === "🥇" ? `**${option.option_text}**` : option.option_text + messageString += `\`\u200b${countString} ${votesString}\u200b\` ${barChart(option.count/totalVoters)} ${label} ${medal}\n` + } + + return {messageString, combinedVotes, totalVoters} +} + +/** + * @param {DiscordTypes.APIMessageApplicationCommandGuildInteraction} interaction + * @param {{api: typeof api}} di + * @returns {AsyncGenerator<{[k in keyof InteractionMethods]?: Parameters[2]}>} + */ +async function* _interact({data}, {api}) { + const row = select("poll", "is_closed", {message_id: data.target_id}).get() + + if (!row) { + return yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, + data: { + content: "This poll hasn't been bridged to Matrix.", + flags: DiscordTypes.MessageFlags.Ephemeral + } + }} + } + + const {messageString} = getCombinedResults(data.target_id, !!row.is_closed) + + return yield {createInteractionResponse: { + type: DiscordTypes.InteractionResponseType.ChannelMessageWithSource, + data: { + embeds: [{ + author: { + name: "Current results including Matrix votes", + icon_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png` + }, + description: messageString + }], + flags: DiscordTypes.MessageFlags.Ephemeral + } + }} +} + +/* c8 ignore start */ + +/** @param {DiscordTypes.APIMessageApplicationCommandGuildInteraction} interaction */ +async function interact(interaction) { + for await (const response of _interact(interaction, {api})) { + if (response.createInteractionResponse) { + await discord.snow.interaction.createInteractionResponse(interaction.id, interaction.token, response.createInteractionResponse) + } + } +} + +module.exports.interact = interact +module.exports._interact = _interact +module.exports.getCombinedResults = getCombinedResults \ No newline at end of file diff --git a/src/discord/register-interactions.js b/src/discord/register-interactions.js index b37f28e9..a9093938 100644 --- a/src/discord/register-interactions.js +++ b/src/discord/register-interactions.js @@ -10,6 +10,7 @@ const permissions = sync.require("./interactions/permissions.js") const reactions = sync.require("./interactions/reactions.js") const privacy = sync.require("./interactions/privacy.js") const poll = sync.require("./interactions/poll.js") +const pollResponses = sync.require("./interactions/poll-responses.js") const ping = sync.require("./interactions/ping.js") // User must have EVERY permission in default_member_permissions to be able to use the command @@ -24,7 +25,7 @@ discord.snow.interaction.bulkOverwriteApplicationCommands(id, [{ type: DiscordTypes.ApplicationCommandType.Message, default_member_permissions: String(DiscordTypes.PermissionFlagsBits.KickMembers | DiscordTypes.PermissionFlagsBits.ManageRoles) }, { - name: "Reactions", + name: "Responses", contexts: [DiscordTypes.InteractionContextType.Guild], type: DiscordTypes.ApplicationCommandType.Message }, { @@ -107,8 +108,14 @@ async function dispatchInteraction(interaction) { await permissions.interact(interaction) } else if (interactionId === "permissions_edit") { await permissions.interactEdit(interaction) - } else if (interactionId === "Reactions") { - await reactions.interact(interaction) + } else if (interactionId === "Responses") { + /** @type {DiscordTypes.APIMessageApplicationCommandGuildInteraction} */ // @ts-ignore + const messageInteraction = interaction + if (messageInteraction.data.resolved.messages[messageInteraction.data.target_id]?.poll) { + await pollResponses.interact(messageInteraction) + } else { + await reactions.interact(messageInteraction) + } } else if (interactionId === "ping") { await ping.interact(interaction) } else if (interactionId === "privacy") { From b463e1173ba7209d79447b1703e3e6e54f96d64c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 5 Feb 2026 01:00:06 +1300 Subject: [PATCH 113/153] Fallback text for Matrix poll end events Right now this doesn't seem to show up on any clients because extensible events is a total mess, but if you did want to code a client that shows this fallback without bothering to code real support for polls, you are easily able to do that. Just pretend the poll end event is a m.room.message and render it like usual. --- src/d2m/actions/poll-end.js | 3 ++- src/d2m/converters/edit-to-changes.js | 8 ++++++-- src/d2m/converters/message-to-event.js | 24 ++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/d2m/actions/poll-end.js b/src/d2m/actions/poll-end.js index 9ffcaf68..b0d29b9a 100644 --- a/src/d2m/actions/poll-end.js +++ b/src/d2m/actions/poll-end.js @@ -113,7 +113,8 @@ async function endPoll(closeMessage) { return { username: "Total results including Matrix votes", avatar_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png`, - content: messageString + content: messageString, + flags: DiscordTypes.MessageFlags.SuppressEmbeds } } } diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index 82b9417f..b73d6e00 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -16,8 +16,12 @@ function eventCanBeEdited(ev) { if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") { return false } - // Discord does not allow stickers to be edited. Poll closures are sent as "edits", but not in a way we care about. - if (ev.old.event_type === "m.sticker" || ev.old.event_type === "org.matrix.msc3381.poll.start") { + // Discord does not allow stickers to be edited. + if (ev.old.event_type === "m.sticker") { + return false + } + // Discord does not allow the data of polls to be edited, they may only be responded to. + if (ev.old.event_type === "org.matrix.msc3381.poll.start" || ev.old.event_type === "org.matrix.msc3381.poll.end") { return false } // Anything else is fair game. diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 9236c893..2f129580 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -20,6 +20,8 @@ const mxUtils = sync.require("../../matrix/utils") const dUtils = sync.require("../../discord/utils") /** @type {import("./find-mentions")} */ const findMentions = sync.require("./find-mentions") +/** @type {import("../../discord/interactions/poll-responses")} */ +const pollResponses = sync.require("../../discord/interactions/poll-responses") const {reg} = require("../../matrix/read-registration") /** @@ -269,7 +271,21 @@ async function messageToEvent(message, guild, options = {}, di) { } if (message.type === DiscordTypes.MessageType.PollResult) { - const event_id = select("event_message", "event_id", {message_id: message.message_reference?.message_id}).pluck().get() + const pollMessageID = message.message_reference?.message_id + if (!pollMessageID) return [] + const event_id = select("event_message", "event_id", {message_id: pollMessageID}).pluck().get() + const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get() + const pollQuestionText = select("poll", "question_text", {message_id: pollMessageID}).pluck().get() + if (!event_id || !roomID || !pollQuestionText) return [] // drop it if the corresponding poll start was not bridged + + const rep = new mxUtils.MatrixStringBuilder() + rep.addLine(`The poll ${pollQuestionText} has closed.`, tag`The poll ${pollQuestionText} has closed.`) + + const {messageString} = pollResponses.getCombinedResults(pollMessageID, true) // poll results have already been double-checked before this point, so these totals will be accurate + rep.addLine(markdown.toHTML(messageString, {discordOnly: true, escapeHTML: false}), markdown.toHTML(messageString, {})) + + const {body, formatted_body} = rep.get() + return [{ $type: "org.matrix.msc3381.poll.end", "m.relates_to": { @@ -277,7 +293,11 @@ async function messageToEvent(message, guild, options = {}, di) { event_id }, "org.matrix.msc3381.poll.end": {}, - body: "This poll has ended.", + "org.matrix.msc1767.text": body, + "org.matrix.msc1767.html": formatted_body, + body: body, + format: "org.matrix.custom.html", + formatted_body: formatted_body, msgtype: "m.text" }] } From e66822e94b3b766a63a1fe88de9295c8a40835ad Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 9 Feb 2026 13:22:36 +1300 Subject: [PATCH 114/153] Make sure written mentions do not match in URLs --- src/d2m/converters/message-to-event.js | 3 ++- src/d2m/converters/message-to-event.test.js | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 2f129580..460b743f 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -721,7 +721,8 @@ async function messageToEvent(message, guild, options = {}, di) { const m = matches[i] const prefix = m[1] const maximumWrittenSection = m[2].toLowerCase() - if (maximumWrittenSection.match(/^!?&?[0-9]+>/) || maximumWrittenSection.match(/^everyone\b/) || maximumWrittenSection.match(/^here\b/)) continue + if (m.index > 0 && !content[m.index-1].match(/ |\(|\n/)) continue // must have space before it + if (maximumWrittenSection.match(/^everyone\b/) || maximumWrittenSection.match(/^here\b/)) continue // ignore @everyone/@here var roomID = roomID ?? select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get() assert(roomID) diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index 7a7d86f3..84cc1e04 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -947,6 +947,21 @@ test("message2event: written @mentions may match part of the mxid", async t => { }]) }) +test("message2event: written @mentions do not match in URLs", async t => { + const events = await messageToEvent({ + ...data.message.advanced_written_at_mention_for_matrix, + content: "the fucking around with pixel composer continues https://pub.mastodon.sleeping.town/@exa/116037641900024965" + }, data.guild.general, {}, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "the fucking around with pixel composer continues https://pub.mastodon.sleeping.town/@exa/116037641900024965", + format: "org.matrix.custom.html", + formatted_body: `the fucking around with pixel composer continues https://pub.mastodon.sleeping.town/@exa/116037641900024965` + }]) +}) + test("message2event: entire message may match elaborate display name", async t => { let called = 0 const events = await messageToEvent({ From 64369f1054818189747fb1a6ae3a5c6f3e125964 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 9 Feb 2026 13:22:45 +1300 Subject: [PATCH 115/153] Fix test --- src/d2m/converters/message-to-event.js | 4 ++-- test/ooye-test-data.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 460b743f..a23d9c9c 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -280,10 +280,10 @@ async function messageToEvent(message, guild, options = {}, di) { const rep = new mxUtils.MatrixStringBuilder() rep.addLine(`The poll ${pollQuestionText} has closed.`, tag`The poll ${pollQuestionText} has closed.`) - + const {messageString} = pollResponses.getCombinedResults(pollMessageID, true) // poll results have already been double-checked before this point, so these totals will be accurate rep.addLine(markdown.toHTML(messageString, {discordOnly: true, escapeHTML: false}), markdown.toHTML(messageString, {})) - + const {body, formatted_body} = rep.get() return [{ diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 04e6b9b1..216581c2 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -98,8 +98,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part ('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 0, 1), ('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 0, 1), ('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1, 1), -('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 1, 0), -('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0, 0), +('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 1, 1), +('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0, 1), ('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999', 'm.room.message', 'm.text', '1106366167788044451', 0, 0, 1), ('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZI999', 'm.room.message', 'm.image', '1106366167788044451', 0, 0, 1), ('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd999', 'm.sticker', NULL, '1106366167788044451', 0, 0, 1), From 279e379d77541ec40a9f5197f4716b4502022ce8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 10 Feb 2026 16:34:47 +1300 Subject: [PATCH 116/153] The database really works better if you query it --- src/d2m/event-dispatcher.js | 2 +- src/discord/interactions/matrix-info.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 3392eb98..2e005adb 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -307,7 +307,7 @@ module.exports = { */ async MESSAGE_REACTION_ADD(client, data) { if (data.user_id === client.user.id) return // m2d reactions are added by the discord bot user - do not reflect them back to matrix. - if (data.emoji.name === "❓" && select("event_message", "message_id", {message_id: data.message_id, source: 0})) { + if (data.emoji.name === "❓" && select("event_message", "message_id", {message_id: data.message_id, source: 0, part: 0}).get()) { // source 0 = matrix const guild_id = data.guild_id ?? client.channels.get(data.channel_id)["guild_id"] await Promise.all([ client.snow.channel.deleteReaction(data.channel_id, data.message_id, data.emoji.name).catch(() => {}), diff --git a/src/discord/interactions/matrix-info.js b/src/discord/interactions/matrix-info.js index ca7da5ba..79300a39 100644 --- a/src/discord/interactions/matrix-info.js +++ b/src/discord/interactions/matrix-info.js @@ -20,7 +20,7 @@ const webGuild = sync.require("../../web/routes/guild") */ async function _interact({guild_id, data}, {api}) { const message = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") - .select("source", "reference_channel_id", "room_id", "event_id").where({message_id: data.target_id, part: 0}).get() + .select("source", "reference_channel_id", "room_id", "event_id").where({message_id: data.target_id}).and("ORDER BY part").get() if (!message) { return { From 0ed3ef68f17155892ab259415a3a321e492e3237 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 10 Feb 2026 16:35:03 +1300 Subject: [PATCH 117/153] Fix PluralKit replies --- src/d2m/converters/message-to-event.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index a23d9c9c..38d16012 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -259,6 +259,7 @@ async function pollToEvent(poll) { * @returns {Promise<{$type: string, $sender?: string, [x: string]: any}[]>} */ async function messageToEvent(message, guild, options = {}, di) { + message = {...message} const events = [] /* c8 ignore next 7 */ @@ -325,8 +326,7 @@ async function messageToEvent(message, guild, options = {}, di) { let content = message.content if (content) content = `\n${content}` else if ((message.flags || 0) & DiscordTypes.MessageFlags.Loading) content = " — interaction loading..." - content = `> ↪️ <@${interaction.user.id}> used \`/${interaction.name}\`${content}` - message = {...message, content} // editToChanges reuses the object so we can't mutate it. have to clone it + message.content = `> ↪️ <@${interaction.user.id}> used \`/${interaction.name}\`${content}` } /** From dbfa9d0f2b797de442b9654ca28b10240d405604 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 10 Feb 2026 16:42:02 +1300 Subject: [PATCH 118/153] Sync PK member profile on first message First time a PK member sends a message in the channel, Discord sends a MESSAGE_UPDATE with the proper avatar data for them. OOYE's speedbump means sending this message will actually take the edit message path. The edit message path previously did not force a profile sync. This is why the Matrix profile did always show up after their second message, because that message was not updated and took the send path. --- src/d2m/actions/edit-message.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2m/actions/edit-message.js b/src/d2m/actions/edit-message.js index 57b8f410..f86a9c82 100644 --- a/src/d2m/actions/edit-message.js +++ b/src/d2m/actions/edit-message.js @@ -30,7 +30,7 @@ async function editMessage(message, guild, row) { if (row && row.speedbump_webhook_id === message.webhook_id) { // Handle the PluralKit public instance if (row.speedbump_id === "466378653216014359") { - senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, false) + senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, true) } } From 6b4123b84595407a5cf44a3d6f8cab5dec7f41b8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 01:10:31 +1300 Subject: [PATCH 119/153] More accurate flags check in setup --- scripts/setup.js | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/setup.js b/scripts/setup.js index 07ded566..26272a6f 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -184,17 +184,21 @@ function defineEchoHandler() { } }) - const mandatoryIntentFlags = DiscordTypes.ApplicationFlags.GatewayMessageContent | DiscordTypes.ApplicationFlags.GatewayMessageContentLimited - if (!(client.flags & mandatoryIntentFlags)) { + const intentFlagPossibilities = [ + DiscordTypes.ApplicationFlags.GatewayMessageContent | DiscordTypes.ApplicationFlags.GatewayPresence, + DiscordTypes.ApplicationFlags.GatewayMessageContentLimited | DiscordTypes.ApplicationFlags.GatewayPresenceLimited + ] + const intentFlagMask = intentFlagPossibilities.reduce((a, c) => a | c, 0) + if (!intentFlagPossibilities.includes(client.flags & intentFlagMask)) { console.log(`On that same page, scroll down to Privileged Gateway Intents and enable all switches.`) await prompt({ type: "invisible", name: "intents", message: "Press Enter when you've enabled them", - validate: async token => { + validate: async () => { process.stdout.write(magenta("checking, please wait...")) client = await snow.requestHandler.request(`/applications/@me`, {}, "get", "json") - if (client.flags & mandatoryIntentFlags) { + if (intentFlagPossibilities.includes(client.flags & intentFlagMask)) { return true } else { return "Switches have not been enabled yet" @@ -220,7 +224,7 @@ function defineEchoHandler() { type: "invisible", name: "description", message: "Press Enter to acknowledge", - validate: async token => { + validate: async () => { process.stdout.write(magenta("checking, please wait...")) client = await snow.requestHandler.request(`/applications/@me`, {}, "get", "json") if (client.description?.match(/out.of.your.element/i)) { @@ -237,7 +241,8 @@ function defineEchoHandler() { const clientSecretResponse = await prompt({ type: "input", name: "discord_client_secret", - message: "Client secret" + message: "Client secret", + validate: secret => !!secret }) const expectedUri = `${bridgeOriginResponse.bridge_origin}/oauth` @@ -247,7 +252,7 @@ function defineEchoHandler() { type: "invisible", name: "redirect_uri", message: "Press Enter when you've added it", - validate: async token => { + validate: async () => { process.stdout.write(magenta("checking, please wait...")) client = await snow.requestHandler.request(`/applications/@me`, {}, "get", "json") if (client.redirect_uris.includes(expectedUri)) { From c0d82754b01652a01c96fbe139afd2bf455f78c8 Mon Sep 17 00:00:00 2001 From: abdul <32655037-CamperThumper@users.noreply.gitlab.com> Date: Sun, 8 Feb 2026 17:50:27 +0300 Subject: [PATCH 120/153] Link instead of upload emoji sprite sheets --- src/m2d/converters/event-to-message.js | 43 ++++++++++++++++++------- src/matrix/utils.js | 25 +++++++++------ src/web/routes/download-matrix.js | 44 +++++++++++++++++++++++--- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 9ac174e9..c8879ca9 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -348,25 +348,44 @@ function getUserOrProxyOwnerMention(mxid) { /** * At the time of this executing, we know what the end of message emojis are, and we know that at least one of them is unknown. - * This function will strip them from the content and generate the correct pending file of the sprite sheet. + * This function will strip them from the content and generate a link in it's place for proxying a sprite sheet that discord previews. * @param {string} content - * @param {{id: string, filename: string}[]} attachments - * @param {({name: string, mxc: string} | {name: string, mxc: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} pendingFiles - * @param {(mxc: string) => Promise} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock. */ -async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, mxcDownloader) { +async function uploadEndOfMessageSpriteSheet(content) { if (!content.includes("<::>")) return content // No unknown emojis, nothing to do // Remove known and unknown emojis from the end of the message const r = /\s*$/ + while (content.match(r)) { content = content.replace(r, "") } - // Create a sprite sheet of known and unknown emojis from the end of the message - const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis, mxcDownloader) - // Attach it - const filename = "emojis.png" - attachments.push({id: String(attachments.length), filename}) - pendingFiles.push({name: filename, buffer}) + + let emojiSheetUrl = new URL('emoji/matrix', reg.ooye.bridge_origin) + for(let mxc of endOfMessageEmojis) { + const emojiSheetUrlString = emojiSheetUrl.toString() + + // Discord will not preview the URL if it is longer than 2000 characters. + // Longer URLs can preview but it's very inconsistant so ignoring any additional emojis + if(emojiSheetUrlString.length + mxc.length > 2000) { + break + } + + // Ignoring any additional emojis if it would exceed discords message length + if(emojiSheetUrlString.length + mxc.length + content.length > 4000) { + break + } + + mxUtils.generatePermittedMediaHash(mxc) + emojiSheetUrl.searchParams.append('e', mxc) + } + + const emojiSheetUrlString = emojiSheetUrl.toString() + + // Discord message length check. Using markdown link with placeholder text since discord displays + // the link when there is preceeding text to the emoji + const placeholderText = '.' + content = content.concat(`[${placeholderText}](${emojiSheetUrlString})`) + return content } @@ -937,7 +956,7 @@ async function eventToMessage(event, guild, channel, di) { if (replyLine && content.startsWith("> ")) content = "\n" + content // SPRITE SHEET EMOJIS FEATURE: - content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader) + content = await uploadEndOfMessageSpriteSheet(content) } else { // Looks like we're using the plaintext body! content = event.content.body diff --git a/src/matrix/utils.js b/src/matrix/utils.js index d89c9682..21e954d0 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -205,6 +205,19 @@ async function getViaServersQuery(roomID, api) { return qs } +function generatePermittedMediaHash(mxc) { + assert(hasher, "xxhash is not ready yet") + const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) + if (!mediaParts) return undefined + + const serverAndMediaID = `${mediaParts[1]}/${mediaParts[2]}` + const unsignedHash = hasher.h64(serverAndMediaID) + const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range + db.prepare("INSERT OR IGNORE INTO media_proxy (permitted_hash) VALUES (?)").run(signedHash) + + return serverAndMediaID +} + /** * Since the introduction of authenticated media, this can no longer just be the /_matrix/media/r0/download URL * because Discord and Discord users cannot use those URLs. Media now has to be proxied through the bridge. @@ -219,15 +232,8 @@ async function getViaServersQuery(roomID, api) { * @returns {string | undefined} */ function getPublicUrlForMxc(mxc) { - assert(hasher, "xxhash is not ready yet") - const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) - if (!mediaParts) return undefined - - const serverAndMediaID = `${mediaParts[1]}/${mediaParts[2]}` - const unsignedHash = hasher.h64(serverAndMediaID) - const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range - db.prepare("INSERT OR IGNORE INTO media_proxy (permitted_hash) VALUES (?)").run(signedHash) - + const serverAndMediaID = generatePermittedMediaHash(mxc); + if(!serverAndMediaID) return undefined return `${reg.ooye.bridge_origin}/download/matrix/${serverAndMediaID}` } @@ -358,6 +364,7 @@ async function setUserPowerCascade(spaceID, mxid, power, api) { module.exports.bot = bot module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord +module.exports.generatePermittedMediaHash = generatePermittedMediaHash module.exports.getPublicUrlForMxc = getPublicUrlForMxc module.exports.getEventIDHash = getEventIDHash module.exports.MatrixStringBuilder = MatrixStringBuilder diff --git a/src/web/routes/download-matrix.js b/src/web/routes/download-matrix.js index 8f790c5a..0810f9a6 100644 --- a/src/web/routes/download-matrix.js +++ b/src/web/routes/download-matrix.js @@ -1,7 +1,7 @@ // @ts-check const assert = require("assert/strict") -const {defineEventHandler, getValidatedRouterParams, setResponseStatus, setResponseHeader, sendStream, createError, H3Event} = require("h3") +const {defineEventHandler, getValidatedRouterParams, setResponseStatus, setResponseHeader, createError, H3Event, getValidatedQuery} = require("h3") const {z} = require("zod") /** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore @@ -27,10 +27,8 @@ function getAPI(event) { return event.context.api || sync.require("../../matrix/api") } -as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(async event => { - const params = await getValidatedRouterParams(event, schema.params.parse) - - const serverAndMediaID = `${params.server_name}/${params.media_id}` +function verifyMediaHash(serverName, mediaId) { + const serverAndMediaID = `${serverName}/${mediaId}` const unsignedHash = hasher.h64(serverAndMediaID) const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range @@ -41,7 +39,12 @@ as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(asyn data: `The file you requested isn't permitted by this media proxy.` }) } +} +as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(async event => { + const params = await getValidatedRouterParams(event, schema.params.parse) + + verifyMediaHash(params.server_name, params.media_id) const api = getAPI(event) const res = await api.getMedia(`mxc://${params.server_name}/${params.media_id}`) @@ -53,3 +56,34 @@ as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(asyn setResponseHeader(event, "Transfer-Encoding", "chunked") return res.body })) + +const emojiSchema = z.object({ + 'e': z.array(z.string()).or(z.string()) +}) + +const emojiSheet = sync.require("../../m2d/actions/emoji-sheet") +const emojiSheetConverter = sync.require("../../m2d/converters/emoji-sheet") + +as.router.get(`/emoji/matrix`, defineEventHandler(async event => { + + const query = await getValidatedQuery(event, emojiSchema.parse) + + let mxcs = query.e + if(!Array.isArray(mxcs)) { + mxcs = [mxcs] + } + + for(let mxc of mxcs) { + const mediaParts = mxc.match(/^mxc:\/\/([^/]+)\/(\w+)$/) + if (!mediaParts) return undefined + verifyMediaHash(mediaParts[1], mediaParts[2]) + } + const buffer = await emojiSheetConverter.compositeMatrixEmojis(mxcs, emojiSheet.getAndConvertEmoji) + + const contentType = 'image/png' + + setResponseStatus(event, 200) + setResponseHeader(event, "Content-Type", contentType) + setResponseHeader(event, "Transfer-Encoding", "chunked") + return buffer +})) From d1b0fa48cf86cfed76d28fbb9e00cb3ed36ecf10 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 02:40:18 +1300 Subject: [PATCH 121/153] Add tests for emoji sheet; style and nits --- package.json | 1 - scripts/start-server.js | 4 + src/m2d/converters/event-to-message.js | 45 ++-- src/m2d/converters/event-to-message.test.js | 234 ++++++++++---------- src/matrix/utils.js | 21 +- src/web/routes/download-matrix.js | 51 +++-- src/web/routes/download-matrix.test.js | 51 +++++ test/test.js | 94 ++++---- test/web.js | 3 +- 9 files changed, 280 insertions(+), 224 deletions(-) diff --git a/package.json b/package.json index d0a154c6..cce8204c 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,6 @@ "setup": "node --enable-source-maps scripts/setup.js", "addbot": "node addbot.js", "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js | tap-dot", - "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot", "cover": "c8 -o test/coverage --skip-full -x db/migrations -x src/m2d/event-dispatcher.js -x src/matrix/file.js -x src/matrix/api.js -x src/d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow" } } diff --git a/scripts/start-server.js b/scripts/start-server.js index 0d4753ac..44edbcb9 100755 --- a/scripts/start-server.js +++ b/scripts/start-server.js @@ -34,5 +34,9 @@ passthrough.select = orm.select console.log("Discord gateway started") sync.require("../src/web/server") + discord.cloud.once("ready", () => { + as.listen() + }) + require("../src/stdin") })() diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index c8879ca9..684b79da 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -348,10 +348,11 @@ function getUserOrProxyOwnerMention(mxid) { /** * At the time of this executing, we know what the end of message emojis are, and we know that at least one of them is unknown. - * This function will strip them from the content and generate a link in it's place for proxying a sprite sheet that discord previews. + * This function will strip them from the content and add a link that Discord will preview with a sprite sheet of emojis. * @param {string} content + * @returns {string} new content with emoji sheet link */ -async function uploadEndOfMessageSpriteSheet(content) { +function linkEndOfMessageSpriteSheet(content) { if (!content.includes("<::>")) return content // No unknown emojis, nothing to do // Remove known and unknown emojis from the end of the message const r = /\s*$/ @@ -360,33 +361,25 @@ async function uploadEndOfMessageSpriteSheet(content) { content = content.replace(r, "") } - let emojiSheetUrl = new URL('emoji/matrix', reg.ooye.bridge_origin) - for(let mxc of endOfMessageEmojis) { - const emojiSheetUrlString = emojiSheetUrl.toString() + // Use a markdown link to hide the URL. If this is the only thing in the message, Discord will hide it entirely, same as lone URLs. Good for us. + content = content.trimEnd() + content += " [\u2800](" // U+2800 Braille Pattern Blank is invisible on all known platforms but is digitally not a whitespace character + const afterLink = ")" - // Discord will not preview the URL if it is longer than 2000 characters. - // Longer URLs can preview but it's very inconsistant so ignoring any additional emojis - if(emojiSheetUrlString.length + mxc.length > 2000) { + // Make emojis URL params + const params = new URLSearchParams() + for (const mxc of endOfMessageEmojis) { + // We can do up to 2000 chars max. (In this maximal case it will get chunked to a separate message.) Ignore additional emojis. + const withoutMxc = mxUtils.makeMxcPublic(mxc) + const emojisLength = params.toString().length + encodeURIComponent(withoutMxc).length + 2 + if (content.length + emojisLength + afterLink.length > 2000) { break } - - // Ignoring any additional emojis if it would exceed discords message length - if(emojiSheetUrlString.length + mxc.length + content.length > 4000) { - break - } - - mxUtils.generatePermittedMediaHash(mxc) - emojiSheetUrl.searchParams.append('e', mxc) + params.append("e", withoutMxc) } - const emojiSheetUrlString = emojiSheetUrl.toString() - - // Discord message length check. Using markdown link with placeholder text since discord displays - // the link when there is preceeding text to the emoji - const placeholderText = '.' - content = content.concat(`[${placeholderText}](${emojiSheetUrlString})`) - - return content + const url = `${reg.ooye.bridge_origin}/download/sheet?${params.toString()}` + return content + url + afterLink } /** @@ -543,7 +536,7 @@ async function getL1L2ReplyLine(called = false) { * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_End} event * @param {DiscordTypes.APIGuild} guild * @param {DiscordTypes.APIGuildTextChannel} channel - * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, mxcDownloader: (mxc: string) => Promise, pollEnd?: {messageID: string}}} di simple-as-nails dependency injection for the matrix API + * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, pollEnd?: {messageID: string}}} di simple-as-nails dependency injection for the matrix API */ async function eventToMessage(event, guild, channel, di) { let displayName = event.sender @@ -956,7 +949,7 @@ async function eventToMessage(event, guild, channel, di) { if (replyLine && content.startsWith("> ")) content = "\n" + content // SPRITE SHEET EMOJIS FEATURE: - content = await uploadEndOfMessageSpriteSheet(content) + content = await linkEndOfMessageSpriteSheet(content) } else { // Looks like we're using the plaintext body! content = event.content.body diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 551cbd06..f667d702 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -1,22 +1,11 @@ const assert = require("assert").strict -const fs = require("fs") const {test} = require("supertape") const DiscordTypes = require("discord-api-types/v10") const {eventToMessage} = require("./event-to-message") -const {convertImageStream} = require("./emoji-sheet") const data = require("../../../test/data") const {MatrixServerError} = require("../../matrix/mreq") const {select, discord} = require("../../passthrough") -/* c8 ignore next 7 */ -function slow() { - if (process.argv.includes("--slow")) { - return test - } else { - return test.skip - } -} - /** * @param {string} roomID * @param {string} eventID @@ -49,25 +38,6 @@ function sameFirstContentAndWhitespace(t, a, b) { t.equal(a2, b2) } -/** - * MOCK: Gets the emoji from the filesystem and converts to uncompressed PNG data. - * @param {string} mxc a single mxc:// URL - * @returns {Promise} uncompressed PNG data, or undefined if the downloaded emoji is not valid -*/ -async function mockGetAndConvertEmoji(mxc) { - const id = mxc.match(/\/([^./]*)$/)?.[1] - let s - if (fs.existsSync(`test/res/${id}.png`)) { - s = fs.createReadStream(`test/res/${id}.png`) - } else { - s = fs.createReadStream(`test/res/${id}.gif`) - } - return convertImageStream(s, () => { - s.pause() - s.emit("end") - }) -} - test("event2message: body is used when there is no formatted_body", async t => { t.deepEqual( await eventToMessage({ @@ -5335,102 +5305,122 @@ test("event2message: table", async t => { ) }) -slow()("event2message: unknown emoji at the end is reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "wrong body", - format: "org.matrix.custom.html", - formatted_body: 'a b \":ms_robot_grin:\"' - }, - event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "a b", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAAAPoAAAD6AG1e1JrAAALoklEQVR4nM1ZaVBU2RU+LZSIGnAvFUtcRkSk6abpbkDH" - }) +test("event2message: unknown emoji at the end is used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: 'a b \":ms_robot_grin:\"' + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "a b [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FRLMgJGfgTPjIQtvvWZsYjhjy)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) -slow()("event2message: known emoji from an unreachable server at the end is reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "wrong body", - format: "org.matrix.custom.html", - formatted_body: 'a b \":emoji_from_unreachable_server:\"' - }, - event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "a b", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAOoUlEQVR4nM1aCXBbx3l+Eu8bN0CAuO+TAHGTFAmAJHgT" - }) +test("event2message: known emoji from an unreachable server at the end is used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: 'a b \":emoji_from_unreachable_server:\"' + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "a b [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FbZFuuUSEebJYXUMSxuuSuLTa)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) -slow()("event2message: known and unknown emojis in the end are reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "wrong body", - format: "org.matrix.custom.html", - formatted_body: 'known unknown: \":hippo:\" \":ms_robot_dress:\" and known unknown: \":hipposcope:\" \":ms_robot_cat:\"' - }, - event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "known unknown: <:hippo:230201364309868544> [:ms_robot_dress:](https://bridge.example.org/download/matrix/cadence.moe/wcouHVjbKJJYajkhJLsyeJAA) and known unknown:", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAYAAADuFn/PAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAT/klEQVR4nOVcC3CVRZbuS2KAIMpDQt5PQkIScm/uvYRX" - }) +test("event2message: known and unknown emojis in the end are used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: 'known unknown: \":hippo:\" \":ms_robot_dress:\" and known unknown: \":hipposcope:\" \":ms_robot_cat:\"' + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "known unknown: <:hippo:230201364309868544> [:ms_robot_dress:](https://bridge.example.org/download/matrix/cadence.moe/wcouHVjbKJJYajkhJLsyeJAA) and known unknown: [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FWbYqNlACRuicynBfdnPYtmvc&e=cadence.moe%2FHYcztccFIPgevDvoaWNsEtGJ)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) -slow()("event2message: all unknown chess emojis are reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "testing :chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black::chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black:", - format: "org.matrix.custom.html", - formatted_body: "testing \":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"\":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"" - }, - event_id: "$Me6iE8C8CZyrDEOYYrXKSYRuuh_25Jj9kZaNrf7LKr4", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "testing", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAYAAAABgCAYAAAAU9KWJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAgAElEQVR4nOx9B3hUVdr/KIpKL2nT0pPpLRNQkdXddV1c" - }) +test("event2message: all unknown chess emojis are used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "testing :chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black::chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black:", + format: "org.matrix.custom.html", + formatted_body: "testing \":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"\":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"" + }, + event_id: "$Me6iE8C8CZyrDEOYYrXKSYRuuh_25Jj9kZaNrf7LKr4", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "testing [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj&e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) diff --git a/src/matrix/utils.js b/src/matrix/utils.js index 21e954d0..9e447e70 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -232,11 +232,28 @@ function generatePermittedMediaHash(mxc) { * @returns {string | undefined} */ function getPublicUrlForMxc(mxc) { - const serverAndMediaID = generatePermittedMediaHash(mxc); + const serverAndMediaID = makeMxcPublic(mxc) if(!serverAndMediaID) return undefined return `${reg.ooye.bridge_origin}/download/matrix/${serverAndMediaID}` } +/** + * @param {string} mxc + * @returns {string | undefined} mxc URL with protocol stripped, e.g. "cadence.moe/abcdef1234" + */ +function makeMxcPublic(mxc) { + assert(hasher, "xxhash is not ready yet") + const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) + if (!mediaParts) return undefined + + const serverAndMediaID = `${mediaParts[1]}/${mediaParts[2]}` + const unsignedHash = hasher.h64(serverAndMediaID) + const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range + db.prepare("INSERT OR IGNORE INTO media_proxy (permitted_hash) VALUES (?)").run(signedHash) + + return serverAndMediaID +} + /** * @param {string} roomVersionString * @param {number} desiredVersion @@ -364,7 +381,7 @@ async function setUserPowerCascade(spaceID, mxid, power, api) { module.exports.bot = bot module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord -module.exports.generatePermittedMediaHash = generatePermittedMediaHash +module.exports.makeMxcPublic = makeMxcPublic module.exports.getPublicUrlForMxc = getPublicUrlForMxc module.exports.getEventIDHash = getEventIDHash module.exports.MatrixStringBuilder = MatrixStringBuilder diff --git a/src/web/routes/download-matrix.js b/src/web/routes/download-matrix.js index 0810f9a6..bb6b8508 100644 --- a/src/web/routes/download-matrix.js +++ b/src/web/routes/download-matrix.js @@ -11,10 +11,18 @@ require("xxhash-wasm")().then(h => hasher = h) const {sync, as, select} = require("../../passthrough") +/** @type {import("../../m2d/actions/emoji-sheet")} */ +const emojiSheet = sync.require("../../m2d/actions/emoji-sheet") +/** @type {import("../../m2d/converters/emoji-sheet")} */ +const emojiSheetConverter = sync.require("../../m2d/converters/emoji-sheet") + const schema = { params: z.object({ server_name: z.string(), media_id: z.string() + }), + sheet: z.object({ + e: z.array(z.string()).or(z.string()) }) } @@ -27,8 +35,16 @@ function getAPI(event) { return event.context.api || sync.require("../../matrix/api") } -function verifyMediaHash(serverName, mediaId) { - const serverAndMediaID = `${serverName}/${mediaId}` +/** + * @param {H3Event} event + * @returns {typeof emojiSheet["getAndConvertEmoji"]} + */ +function getMxcDownloader(event) { + /* c8 ignore next */ + return event.context.mxcDownloader || emojiSheet.getAndConvertEmoji +} + +function verifyMediaHash(serverAndMediaID) { const unsignedHash = hasher.h64(serverAndMediaID) const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range @@ -44,7 +60,7 @@ function verifyMediaHash(serverName, mediaId) { as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(async event => { const params = await getValidatedRouterParams(event, schema.params.parse) - verifyMediaHash(params.server_name, params.media_id) + verifyMediaHash(`${params.server_name}/${params.media_id}`) const api = getAPI(event) const res = await api.getMedia(`mxc://${params.server_name}/${params.media_id}`) @@ -57,33 +73,20 @@ as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(asyn return res.body })) -const emojiSchema = z.object({ - 'e': z.array(z.string()).or(z.string()) -}) - -const emojiSheet = sync.require("../../m2d/actions/emoji-sheet") -const emojiSheetConverter = sync.require("../../m2d/converters/emoji-sheet") - -as.router.get(`/emoji/matrix`, defineEventHandler(async event => { - - const query = await getValidatedQuery(event, emojiSchema.parse) +as.router.get(`/download/sheet`, defineEventHandler(async event => { + const query = await getValidatedQuery(event, schema.sheet.parse) + /** remember that these have no mxc:// protocol in the string for space reasons */ let mxcs = query.e - if(!Array.isArray(mxcs)) { + if (!Array.isArray(mxcs)) { mxcs = [mxcs] } - for(let mxc of mxcs) { - const mediaParts = mxc.match(/^mxc:\/\/([^/]+)\/(\w+)$/) - if (!mediaParts) return undefined - verifyMediaHash(mediaParts[1], mediaParts[2]) + for (const serverAndMediaID of mxcs) { + verifyMediaHash(serverAndMediaID) } - const buffer = await emojiSheetConverter.compositeMatrixEmojis(mxcs, emojiSheet.getAndConvertEmoji) - const contentType = 'image/png' - - setResponseStatus(event, 200) - setResponseHeader(event, "Content-Type", contentType) - setResponseHeader(event, "Transfer-Encoding", "chunked") + const buffer = await emojiSheetConverter.compositeMatrixEmojis(mxcs.map(s => `mxc://${s}`), getMxcDownloader(event)) + setResponseHeader(event, "Content-Type", "image/png") return buffer })) diff --git a/src/web/routes/download-matrix.test.js b/src/web/routes/download-matrix.test.js index 421d2da7..49a63496 100644 --- a/src/web/routes/download-matrix.test.js +++ b/src/web/routes/download-matrix.test.js @@ -1,5 +1,7 @@ // @ts-check +const fs = require("fs") +const {convertImageStream} = require("../../m2d/converters/emoji-sheet") const tryToCatch = require("try-to-catch") const {test} = require("supertape") const {router} = require("../../../test/web") @@ -33,3 +35,52 @@ test("web download matrix: works if a known attachment", async t => { t.equal(event.node.res.statusCode, 200) t.equal(event.node.res.getHeader("content-type"), "image/png") }) + +/** + * MOCK: Gets the emoji from the filesystem and converts to uncompressed PNG data. + * @param {string} mxc a single mxc:// URL + * @returns {Promise} uncompressed PNG data, or undefined if the downloaded emoji is not valid +*/ +async function mockGetAndConvertEmoji(mxc) { + const id = mxc.match(/\/([^./]*)$/)?.[1] + let s + if (fs.existsSync(`test/res/${id}.png`)) { + s = fs.createReadStream(`test/res/${id}.png`) + } else { + s = fs.createReadStream(`test/res/${id}.gif`) + } + return convertImageStream(s, () => { + s.pause() + s.emit("end") + }) +} + +test("web sheet: single emoji", async t => { + const event = {} + const sheet = await router.test("get", "/download/sheet?e=cadence.moe%2FRLMgJGfgTPjIQtvvWZsYjhjy", { + event, + mxcDownloader: mockGetAndConvertEmoji + }) + t.equal(event.node.res.statusCode, 200) + t.equal(sheet.subarray(0, 90).toString("base64"), "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAAAPoAAAD6AG1e1JrAAALoklEQVR4nM1ZaVBU2RU+LZSIGnAvFUtcRkSk6abpbkDH") +}) + +test("web sheet: multiple sources", async t => { + const event = {} + const sheet = await router.test("get", "/download/sheet?e=cadence.moe%2FWbYqNlACRuicynBfdnPYtmvc&e=cadence.moe%2FHYcztccFIPgevDvoaWNsEtGJ", { + event, + mxcDownloader: mockGetAndConvertEmoji + }) + t.equal(event.node.res.statusCode, 200) + t.equal(sheet.subarray(0, 90).toString("base64"), "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAYAAADuFn/PAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAT/klEQVR4nOVcC3CVRZbuS2KAIMpDQt5PQkIScm/uvYRX") +}) + +test("web sheet: big sheet", async t => { + const event = {} + const sheet = await router.test("get", "/download/sheet?e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj&e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj", { + event, + mxcDownloader: mockGetAndConvertEmoji + }) + t.equal(event.node.res.statusCode, 200) + t.equal(sheet.subarray(0, 90).toString("base64"), "iVBORw0KGgoAAAANSUhEUgAAAYAAAABgCAYAAAAU9KWJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAgAElEQVR4nOx9B3hUVdr/KIpKL2nT0pPpLRNQkdXddV1c") +}) diff --git a/test/test.js b/test/test.js index 0bb1da4a..e05b687d 100644 --- a/test/test.js +++ b/test/test.js @@ -75,47 +75,45 @@ const file = sync.require("../src/matrix/file") file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) } ;(async () => { - /* c8 ignore start - maybe download some more test files in slow mode */ - if (process.argv.includes("--slow")) { - test("test files: download", async t => { - /** @param {{url: string, to: string}[]} files */ - async function allReporter(files) { - return new Promise(resolve => { - let resolved = 0 - const report = files.map(file => file.to.split("/").slice(-1)[0][0]) - files.map(download).forEach((p, i) => { - p.then(() => { - report[i] = green(".") - process.stderr.write("\r" + report.join("")) - if (++resolved === files.length) resolve(null) - }) + /* c8 ignore start - download some more test files in slow mode */ + test("test files: download", async t => { + /** @param {{url: string, to: string}[]} files */ + async function allReporter(files) { + return new Promise(resolve => { + let resolved = 0 + const report = files.map(file => file.to.split("/").slice(-1)[0][0]) + files.map(download).forEach((p, i) => { + p.then(() => { + report[i] = green(".") + process.stderr.write("\r" + report.join("")) + if (++resolved === files.length) resolve(null) }) }) - } - async function download({url, to}) { - if (await fs.existsSync(to)) return - const res = await fetch(url) - // @ts-ignore - await res.body.pipeTo(Writable.toWeb(fs.createWriteStream(to, {encoding: "binary"}))) - } - await allReporter([ - {url: "https://cadence.moe/friends/ooye_test/RLMgJGfgTPjIQtvvWZsYjhjy.png", to: "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png"}, - {url: "https://cadence.moe/friends/ooye_test/bZFuuUSEebJYXUMSxuuSuLTa.png", to: "test/res/bZFuuUSEebJYXUMSxuuSuLTa.png"}, - {url: "https://cadence.moe/friends/ooye_test/qWmbXeRspZRLPcjseyLmeyXC.png", to: "test/res/qWmbXeRspZRLPcjseyLmeyXC.png"}, - {url: "https://cadence.moe/friends/ooye_test/wcouHVjbKJJYajkhJLsyeJAA.png", to: "test/res/wcouHVjbKJJYajkhJLsyeJAA.png"}, - {url: "https://cadence.moe/friends/ooye_test/WbYqNlACRuicynBfdnPYtmvc.gif", to: "test/res/WbYqNlACRuicynBfdnPYtmvc.gif"}, - {url: "https://cadence.moe/friends/ooye_test/HYcztccFIPgevDvoaWNsEtGJ.png", to: "test/res/HYcztccFIPgevDvoaWNsEtGJ.png"}, - {url: "https://cadence.moe/friends/ooye_test/lHfmJpzgoNyNtYHdAmBHxXix.png", to: "test/res/lHfmJpzgoNyNtYHdAmBHxXix.png"}, - {url: "https://cadence.moe/friends/ooye_test/MtRdXixoKjKKOyHJGWLsWLNU.png", to: "test/res/MtRdXixoKjKKOyHJGWLsWLNU.png"}, - {url: "https://cadence.moe/friends/ooye_test/HXfFuougamkURPPMflTJRxGc.png", to: "test/res/HXfFuougamkURPPMflTJRxGc.png"}, - {url: "https://cadence.moe/friends/ooye_test/ikYKbkhGhMERAuPPbsnQzZiX.png", to: "test/res/ikYKbkhGhMERAuPPbsnQzZiX.png"}, - {url: "https://cadence.moe/friends/ooye_test/AYPpqXzVJvZdzMQJGjioIQBZ.png", to: "test/res/AYPpqXzVJvZdzMQJGjioIQBZ.png"}, - {url: "https://cadence.moe/friends/ooye_test/UVuzvpVUhqjiueMxYXJiFEAj.png", to: "test/res/UVuzvpVUhqjiueMxYXJiFEAj.png"}, - {url: "https://ezgif.com/images/format-demo/butterfly.gif", to: "test/res/butterfly.gif"}, - {url: "https://ezgif.com/images/format-demo/butterfly.png", to: "test/res/butterfly.png"}, - ]) - }, {timeout: 60000}) - } + }) + } + async function download({url, to}) { + if (await fs.existsSync(to)) return + const res = await fetch(url) + // @ts-ignore + await res.body.pipeTo(Writable.toWeb(fs.createWriteStream(to, {encoding: "binary"}))) + } + await allReporter([ + {url: "https://cadence.moe/friends/ooye_test/RLMgJGfgTPjIQtvvWZsYjhjy.png", to: "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png"}, + {url: "https://cadence.moe/friends/ooye_test/bZFuuUSEebJYXUMSxuuSuLTa.png", to: "test/res/bZFuuUSEebJYXUMSxuuSuLTa.png"}, + {url: "https://cadence.moe/friends/ooye_test/qWmbXeRspZRLPcjseyLmeyXC.png", to: "test/res/qWmbXeRspZRLPcjseyLmeyXC.png"}, + {url: "https://cadence.moe/friends/ooye_test/wcouHVjbKJJYajkhJLsyeJAA.png", to: "test/res/wcouHVjbKJJYajkhJLsyeJAA.png"}, + {url: "https://cadence.moe/friends/ooye_test/WbYqNlACRuicynBfdnPYtmvc.gif", to: "test/res/WbYqNlACRuicynBfdnPYtmvc.gif"}, + {url: "https://cadence.moe/friends/ooye_test/HYcztccFIPgevDvoaWNsEtGJ.png", to: "test/res/HYcztccFIPgevDvoaWNsEtGJ.png"}, + {url: "https://cadence.moe/friends/ooye_test/lHfmJpzgoNyNtYHdAmBHxXix.png", to: "test/res/lHfmJpzgoNyNtYHdAmBHxXix.png"}, + {url: "https://cadence.moe/friends/ooye_test/MtRdXixoKjKKOyHJGWLsWLNU.png", to: "test/res/MtRdXixoKjKKOyHJGWLsWLNU.png"}, + {url: "https://cadence.moe/friends/ooye_test/HXfFuougamkURPPMflTJRxGc.png", to: "test/res/HXfFuougamkURPPMflTJRxGc.png"}, + {url: "https://cadence.moe/friends/ooye_test/ikYKbkhGhMERAuPPbsnQzZiX.png", to: "test/res/ikYKbkhGhMERAuPPbsnQzZiX.png"}, + {url: "https://cadence.moe/friends/ooye_test/AYPpqXzVJvZdzMQJGjioIQBZ.png", to: "test/res/AYPpqXzVJvZdzMQJGjioIQBZ.png"}, + {url: "https://cadence.moe/friends/ooye_test/UVuzvpVUhqjiueMxYXJiFEAj.png", to: "test/res/UVuzvpVUhqjiueMxYXJiFEAj.png"}, + {url: "https://ezgif.com/images/format-demo/butterfly.gif", to: "test/res/butterfly.gif"}, + {url: "https://ezgif.com/images/format-demo/butterfly.png", to: "test/res/butterfly.png"}, + ]) + }, {timeout: 60000}) /* c8 ignore stop */ const p = migrate.migrate(db) @@ -135,15 +133,6 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("./addbot.test") require("../src/db/orm.test") require("../src/web/server.test") - require("../src/web/routes/download-discord.test") - require("../src/web/routes/download-matrix.test") - require("../src/web/routes/guild.test") - require("../src/web/routes/guild-settings.test") - require("../src/web/routes/info.test") - require("../src/web/routes/link.test") - require("../src/web/routes/log-in-with-matrix.test") - require("../src/web/routes/oauth.test") - require("../src/web/routes/password.test") require("../src/discord/utils.test") require("../src/matrix/kstate.test") require("../src/matrix/api.test") @@ -178,4 +167,13 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/discord/interactions/permissions.test") require("../src/discord/interactions/privacy.test") require("../src/discord/interactions/reactions.test") + require("../src/web/routes/download-discord.test") + require("../src/web/routes/download-matrix.test") + require("../src/web/routes/guild.test") + require("../src/web/routes/guild-settings.test") + require("../src/web/routes/info.test") + require("../src/web/routes/link.test") + require("../src/web/routes/log-in-with-matrix.test") + require("../src/web/routes/oauth.test") + require("../src/web/routes/password.test") })() diff --git a/test/web.js b/test/web.js index 463c6b1b..250694aa 100644 --- a/test/web.js +++ b/test/web.js @@ -51,7 +51,7 @@ class Router { /** * @param {string} method * @param {string} inputUrl - * @param {{event?: any, params?: any, body?: any, sessionData?: any, getOauth2Token?: any, getClient?: (string) => {user: {getGuilds: () => Promise}}, api?: Partial, snow?: {[k in keyof SnowTransfer]?: Partial}, createRoom?: Partial, createSpace?: Partial, headers?: any}} [options] + * @param {{event?: any, params?: any, body?: any, sessionData?: any, getOauth2Token?: any, getClient?: (string) => {user: {getGuilds: () => Promise}}, api?: Partial, snow?: {[k in keyof SnowTransfer]?: Partial}, createRoom?: Partial, createSpace?: Partial, mxcDownloader?: import("../src/m2d/actions/emoji-sheet")["getAndConvertEmoji"], headers?: any}} [options] */ async test(method, inputUrl, options = {}) { const url = new URL(inputUrl, "http://a") @@ -83,6 +83,7 @@ class Router { }, context: { api: options.api, + mxcDownloader: options.mxcDownloader, params: options.params, snow: options.snow, createRoom: options.createRoom, From c8b20719db824cf277e982dcf0abed084682e108 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 02:57:45 +1300 Subject: [PATCH 122/153] Move poll-star-avatar file endpoint --- src/d2m/actions/poll-end.js | 2 +- src/discord/interactions/poll-responses.js | 6 +++--- src/m2d/converters/event-to-message.js | 2 +- src/web/server.js | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/d2m/actions/poll-end.js b/src/d2m/actions/poll-end.js index b0d29b9a..8ede9e2f 100644 --- a/src/d2m/actions/poll-end.js +++ b/src/d2m/actions/poll-end.js @@ -112,7 +112,7 @@ async function endPoll(closeMessage) { if (combinedVotes !== totalVotes) { // This means some votes were cast on Matrix. Now that we've corrected the vote totals, we can get the results again and post them to Discord. return { username: "Total results including Matrix votes", - avatar_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png`, + avatar_url: `${reg.ooye.bridge_origin}/download/file/poll-star-avatar.png`, content: messageString, flags: DiscordTypes.MessageFlags.SuppressEmbeds } diff --git a/src/discord/interactions/poll-responses.js b/src/discord/interactions/poll-responses.js index 86277c28..bcfa1673 100644 --- a/src/discord/interactions/poll-responses.js +++ b/src/discord/interactions/poll-responses.js @@ -12,7 +12,7 @@ const pollComponents = sync.require("../../m2d/converters/poll-components") const {reg} = require("../../matrix/read-registration") /** - * @param {number} percentc + * @param {number} percent */ function barChart(percent) { const width = 12 @@ -69,7 +69,7 @@ async function* _interact({data}, {api}) { embeds: [{ author: { name: "Current results including Matrix votes", - icon_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png` + icon_url: `${reg.ooye.bridge_origin}/download/file/poll-star-avatar.png` }, description: messageString }], @@ -91,4 +91,4 @@ async function interact(interaction) { module.exports.interact = interact module.exports._interact = _interact -module.exports.getCombinedResults = getCombinedResults \ No newline at end of file +module.exports.getCombinedResults = getCombinedResults diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 684b79da..a99950b0 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -667,7 +667,7 @@ async function eventToMessage(event, guild, channel, di) { pollMessages.push(pollComponents.getPollComponentsFromDatabase(di.pollEnd.messageID)) pollMessages.push({ ...await pollComponents.getPollEndMessageFromDatabase(channel.id, di.pollEnd.messageID), - avatar_url: `${reg.ooye.bridge_origin}/discord/poll-star-avatar.png` + avatar_url: `${reg.ooye.bridge_origin}/download/file/poll-star-avatar.png` }) } else { diff --git a/src/web/server.js b/src/web/server.js index 9d9f5a30..a2148770 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -70,7 +70,7 @@ as.router.get("/icon.png", defineEventHandler(event => { return fs.promises.readFile(join(__dirname, "../../docs/img/icon.png")) })) -as.router.get("/discord/poll-star-avatar.png", defineEventHandler(event => { +as.router.get("/download/file/poll-star-avatar.png", defineEventHandler(event => { handleCacheHeaders(event, {maxAge: 86400}) return fs.promises.readFile(join(__dirname, "../../docs/img/poll-star-avatar.png")) })) From 33eef25cf15fe03bc674a7026bba78e2413cc5ab Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 10:18:32 +1300 Subject: [PATCH 123/153] Restore as.listen() during setup --- scripts/setup.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup.js b/scripts/setup.js index 26272a6f..2d6b2376 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -309,6 +309,7 @@ function defineEchoHandler() { const {as} = require("../src/matrix/appservice") as.router.use("/**", defineEchoHandler()) + await as.listen() console.log("⏳ Waiting for you to register the file with your homeserver... (Ctrl+C to cancel)") process.once("SIGINT", () => { From c4909653aa803a73bad91f3c6dbcc26806976d85 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 11:31:27 +1300 Subject: [PATCH 124/153] Fix typecheck --- jsconfig.json | 1 + src/d2m/actions/poll-vote.js | 9 +++-- src/d2m/actions/retrigger.js | 2 +- src/d2m/actions/speedbump.js | 4 +- src/d2m/converters/find-mentions.js | 11 ++++-- src/d2m/converters/find-mentions.test.js | 4 +- src/d2m/converters/message-to-event.js | 49 +++++++++++++----------- src/d2m/event-dispatcher.js | 2 +- src/discord/interactions/matrix-info.js | 11 ++++-- src/discord/interactions/ping.js | 16 +++++--- src/m2d/actions/add-reaction.js | 2 +- src/m2d/actions/redact.js | 2 +- src/m2d/converters/event-to-message.js | 2 + src/matrix/api.js | 7 ++-- src/matrix/matrix-command-handler.js | 2 +- src/matrix/room-upgrade.js | 4 +- src/matrix/utils.js | 27 +++++++++++-- src/web/routes/download-discord.js | 2 +- src/web/routes/download-matrix.test.js | 2 + src/web/routes/guild.js | 18 ++++++--- src/web/routes/link.js | 5 ++- 21 files changed, 117 insertions(+), 65 deletions(-) diff --git a/jsconfig.json b/jsconfig.json index 4106061c..65a9b504 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "target": "es2024", "module": "nodenext", + "lib": ["ESNext"], "strict": true, "noImplicitAny": false, "useUnknownInCatchVariables": false diff --git a/src/d2m/actions/poll-vote.js b/src/d2m/actions/poll-vote.js index 85a223dd..66918fea 100644 --- a/src/d2m/actions/poll-vote.js +++ b/src/d2m/actions/poll-vote.js @@ -70,13 +70,14 @@ async function sendVotes(userOrID, channelID, pollMessageID, pollEventID) { return } + let userID, senderMxid if (typeof userOrID === "string") { // just a string when double-checking a vote removal - good thing the unvoter is already here from having voted - var userID = userOrID - var senderMxid = from("sim").join("sim_member", "mxid").where({user_id: userOrID, room_id: matchingRoomID}).pluck("mxid").get() + userID = userOrID + senderMxid = from("sim").join("sim_member", "mxid").where({user_id: userOrID, room_id: matchingRoomID}).pluck("mxid").get() if (!senderMxid) return } else { // sent in full when double-checking adding a vote, so we can properly ensure joined - var userID = userOrID.id - var senderMxid = await registerUser.ensureSimJoined(userOrID, matchingRoomID) + userID = userOrID.id + senderMxid = await registerUser.ensureSimJoined(userOrID, matchingRoomID) } const answersArray = select("poll_vote", "matrix_option", {discord_or_matrix_user_id: userID, message_id: pollMessageID}).pluck().all() diff --git a/src/d2m/actions/retrigger.js b/src/d2m/actions/retrigger.js index 7ff04263..66ef19e7 100644 --- a/src/d2m/actions/retrigger.js +++ b/src/d2m/actions/retrigger.js @@ -19,7 +19,7 @@ const emitter = new EventEmitter() * Due to Eventual Consistency(TM) an update/delete may arrive before the original message arrives * (or before the it has finished being bridged to an event). * In this case, wait until the original message has finished bridging, then retrigger the passed function. - * @template {(...args: any[]) => Promise} T + * @template {(...args: any[]) => any} T * @param {string} inputID * @param {T} fn * @param {Parameters} rest diff --git a/src/d2m/actions/speedbump.js b/src/d2m/actions/speedbump.js index 1a6ef63c..218f046e 100644 --- a/src/d2m/actions/speedbump.js +++ b/src/d2m/actions/speedbump.js @@ -54,8 +54,8 @@ async function doSpeedbump(messageID) { debugSpeedbump(`[speedbump] DELETED ${messageID}`) return true } - value = bumping.get(messageID) - 1 - if (value === 0) { + value = (bumping.get(messageID) ?? 0) - 1 + if (value <= 0) { debugSpeedbump(`[speedbump] OK ${messageID}-- = ${value}`) bumping.delete(messageID) return false diff --git a/src/d2m/converters/find-mentions.js b/src/d2m/converters/find-mentions.js index 9db6355f..87268300 100644 --- a/src/d2m/converters/find-mentions.js +++ b/src/d2m/converters/find-mentions.js @@ -9,7 +9,7 @@ const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) * @typedef {{text: string, index: number, end: number}} Token */ -/** @typedef {{mxids: {localpart: string, mxid: string, displayname?: string}[], names: {displaynameTokens: Token[], mxid: string}[]}} ProcessedJoined */ +/** @typedef {{mxids: {localpart: string, mxid: string, displayname?: string | null}[], names: {displaynameTokens: Token[], mxid: string}[]}} ProcessedJoined */ const lengthBonusLengthCap = 50 const lengthBonusValue = 0.5 @@ -18,7 +18,7 @@ const lengthBonusValue = 0.5 * 0 = no match * @param {string} localpart * @param {string} input - * @param {string} [displayname] only for the super tiebreaker + * @param {string | null} [displayname] only for the super tiebreaker * @returns {{score: number, matchedInputTokens: Token[]}} */ function scoreLocalpart(localpart, input, displayname) { @@ -103,7 +103,7 @@ function tokenise(name) { } /** - * @param {{mxid: string, displayname?: string}[]} joined + * @param {{mxid: string, displayname?: string | null}[]} joined * @returns {ProcessedJoined} */ function processJoined(joined) { @@ -120,6 +120,7 @@ function processJoined(joined) { }), names: joined.filter(j => j.displayname).map(j => { return { + // @ts-ignore displaynameTokens: tokenise(j.displayname), mxid: j.mxid } @@ -130,6 +131,8 @@ function processJoined(joined) { /** * @param {ProcessedJoined} pjr * @param {string} maximumWrittenSection lowercase please + * @param {number} baseOffset + * @param {string} prefix * @param {string} content */ function findMention(pjr, maximumWrittenSection, baseOffset, prefix, content) { @@ -142,7 +145,7 @@ function findMention(pjr, maximumWrittenSection, baseOffset, prefix, content) { if (best.scored.score > 4) { // requires in smallest case perfect match of 2 characters, or in largest case a partial middle match of 5+ characters in a row // Highlight the relevant part of the message const start = baseOffset + best.scored.matchedInputTokens[0].index - const end = baseOffset + prefix.length + best.scored.matchedInputTokens.at(-1).end + const end = baseOffset + prefix.length + best.scored.matchedInputTokens.slice(-1)[0].end const newContent = content.slice(0, start) + "[" + content.slice(start, end) + "](https://matrix.to/#/" + best.mxid + ")" + content.slice(end) return { mxid: best.mxid, diff --git a/src/d2m/converters/find-mentions.test.js b/src/d2m/converters/find-mentions.test.js index 0d02285d..8f2be094 100644 --- a/src/d2m/converters/find-mentions.test.js +++ b/src/d2m/converters/find-mentions.test.js @@ -113,7 +113,7 @@ test("score name: finds match location", t => { const message = "evil lillith is an inspiration" const result = scoreName(tokenise("INX | Evil Lillith (she/her)"), tokenise(message)) const startLocation = result.matchedInputTokens[0].index - const endLocation = result.matchedInputTokens.at(-1).end + const endLocation = result.matchedInputTokens.slice(-1)[0].end t.equal(message.slice(startLocation, endLocation), "evil lillith") }) @@ -125,5 +125,5 @@ test("find mention: test various tiebreakers", t => { mxid: "@emma:rory.gay", displayname: "Emma [it/its]" }]), "emma ⚡ curious which one this prefers", 0, "@", "@emma ⚡ curious which one this prefers") - t.equal(found.mxid, "@emma:conduit.rory.gay") + t.equal(found?.mxid, "@emma:conduit.rory.gay") }) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 38d16012..b36bdf5d 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -427,7 +427,7 @@ async function messageToEvent(message, guild, options = {}, di) { * @param {string} [timestampChannelID] */ async function getHistoricalEventRow(messageID, timestampChannelID) { - /** @type {{room_id: string} | {event_id: string, room_id: string, reference_channel_id: string, source: number} | null} */ + /** @type {{room_id: string} | {event_id: string, room_id: string, reference_channel_id: string, source: number} | null | undefined} */ let row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") .select("event_id", "room_id", "reference_channel_id", "source").where({message_id: messageID}).and("ORDER BY part ASC").get() if (!row && timestampChannelID) { @@ -574,6 +574,7 @@ async function messageToEvent(message, guild, options = {}, di) { if (repliedToEventInDifferentRoom || repliedToUnknownEvent) { let referenced = message.referenced_message if (!referenced) { // backend couldn't be bothered to dereference the message, have to do it ourselves + assert(message.message_reference?.message_id) referenced = await discord.snow.channel.getChannelMessage(message.message_reference.channel_id, message.message_reference.message_id) } @@ -661,14 +662,14 @@ async function messageToEvent(message, guild, options = {}, di) { } // Forwarded content appears first - if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_snapshots?.length) { + if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_reference.message_id && message.message_snapshots?.length) { // Forwarded notice const row = await getHistoricalEventRow(message.message_reference.message_id, message.message_reference.channel_id) const room = select("channel_room", ["room_id", "name", "nick"], {channel_id: message.message_reference.channel_id}).get() const forwardedNotice = new mxUtils.MatrixStringBuilder() if (room) { const roomName = room && (room.nick || room.name) - if ("event_id" in row) { + if (row && "event_id" in row) { const via = await getViaServersMemo(row.room_id) forwardedNotice.addLine( `[🔀 Forwarded from #${roomName}]`, @@ -802,20 +803,23 @@ async function messageToEvent(message, guild, options = {}, di) { // Then components if (message.components?.length) { - const stack = [new mxUtils.MatrixStringBuilder()] + const stack = new mxUtils.MatrixStringBuilderStack() /** @param {DiscordTypes.APIMessageComponent} component */ async function processComponent(component) { // Standalone components if (component.type === DiscordTypes.ComponentType.TextDisplay) { const {body, html} = await transformContent(component.content) - stack[0].addParagraph(body, html) + stack.msb.addParagraph(body, html) } else if (component.type === DiscordTypes.ComponentType.Separator) { - stack[0].addParagraph("----", "
    ") + stack.msb.addParagraph("----", "
    ") } else if (component.type === DiscordTypes.ComponentType.File) { - const ev = await attachmentToEvent({}, {...component.file, filename: component.name, size: component.size}, true) - stack[0].addLine(ev.body, ev.formatted_body) + /** @type {{[k in keyof DiscordTypes.APIUnfurledMediaItem]-?: NonNullable}} */ // @ts-ignore + const file = component.file + assert(component.name && component.size && file.content_type) + const ev = await attachmentToEvent({}, {...file, filename: component.name, size: component.size}, true) + stack.msb.addLine(ev.body, ev.formatted_body) } else if (component.type === DiscordTypes.ComponentType.MediaGallery) { const description = component.items.length === 1 ? component.items[0].description || "Image:" : "Image gallery:" @@ -826,43 +830,43 @@ async function messageToEvent(message, guild, options = {}, di) { estimatedName: item.media.url.match(/\/([^/?]+)(\?|$)/)?.[1] || publicURL } }) - stack[0].addLine(`🖼️ ${description} ${images.map(i => i.url).join(", ")}`, tag`🖼️ ${description} $${images.map(i => tag`${i.estimatedName}`).join(", ")}`) + stack.msb.addLine(`🖼️ ${description} ${images.map(i => i.url).join(", ")}`, tag`🖼️ ${description} $${images.map(i => tag`${i.estimatedName}`).join(", ")}`) } // string select, text input, user select, role select, mentionable select, channel select // Components that can have things nested else if (component.type === DiscordTypes.ComponentType.Container) { // May contain action row, text display, section, media gallery, separator, file - stack.unshift(new mxUtils.MatrixStringBuilder()) + stack.bump() for (const innerComponent of component.components) { await processComponent(innerComponent) } let {body, formatted_body} = stack.shift().get() body = body.split("\n").map(l => "| " + l).join("\n") formatted_body = `
    ${formatted_body}
    ` - if (stack[0].body) stack[0].body += "\n\n" - stack[0].add(body, formatted_body) + if (stack.msb.body) stack.msb.body += "\n\n" + stack.msb.add(body, formatted_body) } else if (component.type === DiscordTypes.ComponentType.Section) { // May contain text display, possibly more in the future // Accessory may be button or thumbnail - stack.unshift(new mxUtils.MatrixStringBuilder()) + stack.bump() for (const innerComponent of component.components) { await processComponent(innerComponent) } if (component.accessory) { - stack.unshift(new mxUtils.MatrixStringBuilder()) + stack.bump() await processComponent(component.accessory) const {body, formatted_body} = stack.shift().get() - stack[0].addLine(body, formatted_body) + stack.msb.addLine(body, formatted_body) } const {body, formatted_body} = stack.shift().get() - stack[0].addParagraph(body, formatted_body) + stack.msb.addParagraph(body, formatted_body) } else if (component.type === DiscordTypes.ComponentType.ActionRow) { const linkButtons = component.components.filter(c => c.type === DiscordTypes.ComponentType.Button && c.style === DiscordTypes.ButtonStyle.Link) if (linkButtons.length) { - stack[0].addLine("") + stack.msb.addLine("") for (const linkButton of linkButtons) { await processComponent(linkButton) } @@ -871,15 +875,15 @@ async function messageToEvent(message, guild, options = {}, di) { // Components that can only be inside things else if (component.type === DiscordTypes.ComponentType.Thumbnail) { // May only be a section accessory - stack[0].add(`🖼️ ${component.media.url}`, tag`🖼️ ${component.media.url}`) + stack.msb.add(`🖼️ ${component.media.url}`, tag`🖼️ ${component.media.url}`) } else if (component.type === DiscordTypes.ComponentType.Button) { // May only be a section accessory or in an action row (up to 5) if (component.style === DiscordTypes.ButtonStyle.Link) { if (component.label) { - stack[0].add(`[${component.label} ${component.url}] `, tag`${component.label} `) + stack.msb.add(`[${component.label} ${component.url}] `, tag`${component.label} `) } else { - stack[0].add(component.url) + stack.msb.add(component.url) } } } @@ -891,7 +895,7 @@ async function messageToEvent(message, guild, options = {}, di) { await processComponent(component) } - const {body, formatted_body} = stack[0].get() + const {body, formatted_body} = stack.msb.get() if (body.trim().length) { await addTextEvent(body, formatted_body, "m.text") } @@ -914,7 +918,7 @@ async function messageToEvent(message, guild, options = {}, di) { continue // Matrix's own URL previews are fine for images. } - if (embed.type === "video" && !embed.title && message.content.includes(embed.video?.url)) { + if (embed.type === "video" && embed.video?.url && !embed.title && message.content.includes(embed.video.url)) { continue // Doesn't add extra information and the direct video URL is already there. } @@ -937,6 +941,7 @@ async function messageToEvent(message, guild, options = {}, di) { const rep = new mxUtils.MatrixStringBuilder() if (isKlipyGIF) { + assert(embed.video?.url) rep.add("[GIF] ", "➿ ") if (embed.title) { rep.add(`${embed.title} ${embed.video.url}`, tag`${embed.title}`) diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 2e005adb..af18669d 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -308,7 +308,7 @@ module.exports = { async MESSAGE_REACTION_ADD(client, data) { if (data.user_id === client.user.id) return // m2d reactions are added by the discord bot user - do not reflect them back to matrix. if (data.emoji.name === "❓" && select("event_message", "message_id", {message_id: data.message_id, source: 0, part: 0}).get()) { // source 0 = matrix - const guild_id = data.guild_id ?? client.channels.get(data.channel_id)["guild_id"] + const guild_id = data.guild_id ?? client.channels.get(data.channel_id)?.["guild_id"] await Promise.all([ client.snow.channel.deleteReaction(data.channel_id, data.message_id, data.emoji.name).catch(() => {}), // @ts-ignore - this is all you need for it to do a matrix-side lookup diff --git a/src/discord/interactions/matrix-info.js b/src/discord/interactions/matrix-info.js index 79300a39..c85cec2f 100644 --- a/src/discord/interactions/matrix-info.js +++ b/src/discord/interactions/matrix-info.js @@ -54,8 +54,11 @@ async function _interact({guild_id, data}, {api}) { // from Matrix const event = await api.getEvent(message.room_id, message.event_id) const via = await utils.getViaServersQuery(message.room_id, api) - const inChannels = discord.guildChannelMap.get(guild_id) - .map(cid => discord.channels.get(cid)) + const channelsInGuild = discord.guildChannelMap.get(guild_id) + assert(channelsInGuild) + const inChannels = channelsInGuild + // @ts-ignore + .map(/** @returns {DiscordTypes.APIGuildChannel} */ cid => discord.channels.get(cid)) .sort((a, b) => webGuild._getPosition(a, discord.channels) - webGuild._getPosition(b, discord.channels)) .filter(channel => from("channel_room").join("member_cache", "room_id").select("mxid").where({channel_id: channel.id, mxid: event.sender}).get()) const matrixMember = select("member_cache", ["displayname", "avatar_url"], {room_id: message.room_id, mxid: event.sender}).get() @@ -67,7 +70,7 @@ async function _interact({guild_id, data}, {api}) { author: { name, url: `https://matrix.to/#/${event.sender}`, - icon_url: utils.getPublicUrlForMxc(matrixMember.avatar_url) + icon_url: utils.getPublicUrlForMxc(matrixMember?.avatar_url) }, description: `This Matrix message was delivered to Discord by **Out Of Your Element**.\n[View on Matrix →]()\n\n**User ID**: [${event.sender}]()`, color: 0x0dbd8b, @@ -96,7 +99,7 @@ async function dm(interaction) { const channel = await discord.snow.user.createDirectMessageChannel(interaction.member.user.id) const response = await _interact(interaction, {api}) assert(response.type === DiscordTypes.InteractionResponseType.ChannelMessageWithSource) - response.data.flags &= 0 // not ephemeral + response.data.flags = 0 & 0 // not ephemeral await discord.snow.channel.createMessage(channel.id, response.data) } diff --git a/src/discord/interactions/ping.js b/src/discord/interactions/ping.js index 57b48b1e..45824bed 100644 --- a/src/discord/interactions/ping.js +++ b/src/discord/interactions/ping.js @@ -31,7 +31,7 @@ async function* _interactAutocomplete({data, channel}, {api}) { } // Check it was used in a bridged channel - const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get() + const roomID = select("channel_room", "room_id", {channel_id: channel?.id}).pluck().get() if (!roomID) return yield exit() // Check we are in fact autocompleting the first option, the user @@ -58,9 +58,9 @@ async function* _interactAutocomplete({data, channel}, {api}) { const displaynameMatches = select("member_cache", ["mxid", "displayname"], {room_id: roomID}, "AND displayname IS NOT NULL AND displayname LIKE ? ESCAPE '$' LIMIT 25").all(query) // prioritise matches closer to the start displaynameMatches.sort((a, b) => { - let ai = a.displayname.toLowerCase().indexOf(input.toLowerCase()) + let ai = a.displayname?.toLowerCase().indexOf(input.toLowerCase()) ?? -1 if (ai === -1) ai = 999 - let bi = b.displayname.toLowerCase().indexOf(input.toLowerCase()) + let bi = b.displayname?.toLowerCase().indexOf(input.toLowerCase()) ?? -1 if (bi === -1) bi = 999 return ai - bi }) @@ -132,14 +132,18 @@ async function* _interactCommand({data, channel, guild_id}, {api}) { type: DiscordTypes.InteractionResponseType.DeferredChannelMessageWithSource }} + let member try { /** @type {Ty.Event.M_Room_Member} */ - var member = await api.getStateEvent(roomID, "m.room.member", mxid) + member = await api.getStateEvent(roomID, "m.room.member", mxid) } catch (e) {} if (!member || member.membership !== "join") { - const inChannels = discord.guildChannelMap.get(guild_id) - .map(cid => discord.channels.get(cid)) + const channelsInGuild = discord.guildChannelMap.get(guild_id) + assert(channelsInGuild) + const inChannels = channelsInGuild + // @ts-ignore + .map(/** @returns {DiscordTypes.APIGuildChannel} */ cid => discord.channels.get(cid)) .sort((a, b) => webGuild._getPosition(a, discord.channels) - webGuild._getPosition(b, discord.channels)) .filter(channel => from("channel_room").join("member_cache", "room_id").select("mxid").where({channel_id: channel.id, mxid}).get()) if (inChannels.length) { diff --git a/src/m2d/actions/add-reaction.js b/src/m2d/actions/add-reaction.js index 9ee92769..e4981fbd 100644 --- a/src/m2d/actions/add-reaction.js +++ b/src/m2d/actions/add-reaction.js @@ -17,7 +17,7 @@ const retrigger = sync.require("../../d2m/actions/retrigger") */ async function addReaction(event) { // Wait until the corresponding channel and message have already been bridged - if (retrigger.eventNotFoundThenRetrigger(event.content["m.relates_to"].event_id, as.emit.bind(as, "type:m.reaction", event))) return + if (retrigger.eventNotFoundThenRetrigger(event.content["m.relates_to"].event_id, () => as.emit("type:m.reaction", event))) return // These will exist because it passed retrigger const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") diff --git a/src/m2d/actions/redact.js b/src/m2d/actions/redact.js index 022157de..3135d311 100644 --- a/src/m2d/actions/redact.js +++ b/src/m2d/actions/redact.js @@ -58,7 +58,7 @@ async function handle(event) { await removeReaction(event) // Or, it might be for removing a message or suppressing embeds. But to do that, the message needs to be bridged first. - if (retrigger.eventNotFoundThenRetrigger(event.redacts, as.emit.bind(as, "type:m.room.redaction", event))) return + if (retrigger.eventNotFoundThenRetrigger(event.redacts, () => as.emit("type:m.room.redaction", event))) return const row = select("event_message", ["event_type", "event_subtype", "part"], {event_id: event.redacts}).get() if (row && row.event_type === "m.room.message" && row.event_subtype === "m.notice" && row.part === 1) { diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index a99950b0..f6baf550 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -1,4 +1,5 @@ // @ts-check +/// const Ty = require("../../types") const DiscordTypes = require("discord-api-types/v10") @@ -371,6 +372,7 @@ function linkEndOfMessageSpriteSheet(content) { for (const mxc of endOfMessageEmojis) { // We can do up to 2000 chars max. (In this maximal case it will get chunked to a separate message.) Ignore additional emojis. const withoutMxc = mxUtils.makeMxcPublic(mxc) + assert(withoutMxc) const emojisLength = params.toString().length + encodeURIComponent(withoutMxc).length + 2 if (content.length + emojisLength + afterLink.length > 2000) { break diff --git a/src/matrix/api.js b/src/matrix/api.js index 1cd05d34..70cb50be 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -196,9 +196,10 @@ async function getInviteState(roomID, event) { } // Try calling sliding sync API and extracting from stripped state + let root try { /** @type {Ty.R.SSS} */ - var root = await mreq.mreq("POST", path("/client/unstable/org.matrix.simplified_msc3575/sync", `@${reg.sender_localpart}:${reg.ooye.server_name}`, {timeout: "0"}), { + root = await mreq.mreq("POST", path("/client/unstable/org.matrix.simplified_msc3575/sync", `@${reg.sender_localpart}:${reg.ooye.server_name}`, {timeout: "0"}), { lists: { a: { ranges: [[0, 999]], @@ -239,7 +240,7 @@ async function getInviteState(roomID, event) { name: room.name ?? null, topic: room.topic ?? null, avatar: room.avatar_url ?? null, - type: room.room_type + type: room.room_type ?? null } } @@ -426,7 +427,7 @@ async function profileSetDisplayname(mxid, displayname, inhibitPropagate) { /** * @param {string} mxid - * @param {string} avatar_url + * @param {string | null | undefined} avatar_url * @param {boolean} [inhibitPropagate] */ async function profileSetAvatarUrl(mxid, avatar_url, inhibitPropagate) { diff --git a/src/matrix/matrix-command-handler.js b/src/matrix/matrix-command-handler.js index c1c69f15..e382a329 100644 --- a/src/matrix/matrix-command-handler.js +++ b/src/matrix/matrix-command-handler.js @@ -124,7 +124,7 @@ const commands = [{ if (matrixOnlyReason) { // If uploading to Matrix, check if we have permission const {powerLevels, powers: {[mxUtils.bot]: botPower}} = await mxUtils.getEffectivePower(event.room_id, [mxUtils.bot], api) - const requiredPower = powerLevels.events["im.ponies.room_emotes"] ?? powerLevels.state_default ?? 50 + const requiredPower = powerLevels.events?.["im.ponies.room_emotes"] ?? powerLevels.state_default ?? 50 if (botPower < requiredPower) { return api.sendEvent(event.room_id, "m.room.message", { ...ctx, diff --git a/src/matrix/room-upgrade.js b/src/matrix/room-upgrade.js index 6c344cf1..5a2606ee 100644 --- a/src/matrix/room-upgrade.js +++ b/src/matrix/room-upgrade.js @@ -57,12 +57,12 @@ async function onBotMembership(event, api, createRoom) { // Check if an upgrade is pending for this room const newRoomID = event.room_id const oldRoomID = select("room_upgrade_pending", "old_room_id", {new_room_id: newRoomID}).pluck().get() - if (!oldRoomID) return + if (!oldRoomID) return false const channelRow = from("channel_room").join("guild_space", "guild_id").where({room_id: oldRoomID}).select("space_id", "guild_id", "channel_id").get() assert(channelRow) // this could only fail if the channel was unbridged or something between upgrade and joining // Check if is join/invite - if (event.content.membership !== "invite" && event.content.membership !== "join") return + if (event.content.membership !== "invite" && event.content.membership !== "join") return false return await roomUpgradeSema.request(async () => { // If invited, join diff --git a/src/matrix/utils.js b/src/matrix/utils.js index 9e447e70..9f5cb0f6 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -60,6 +60,26 @@ function getEventIDHash(eventID) { return signedHash } +class MatrixStringBuilderStack { + constructor() { + this.stack = [new MatrixStringBuilder()] + } + + get msb() { + return this.stack[0] + } + + bump() { + this.stack.unshift(new MatrixStringBuilder()) + } + + shift() { + const msb = this.stack.shift() + assert(msb) + return msb + } +} + class MatrixStringBuilder { constructor() { this.body = "" @@ -228,7 +248,7 @@ function generatePermittedMediaHash(mxc) { * @see https://matrix.org/blog/2024/06/26/sunsetting-unauthenticated-media/ background * @see https://matrix.org/blog/2024/06/20/matrix-v1.11-release/ implementation details * @see https://www.sqlite.org/fileformat2.html#record_format SQLite integer field size - * @param {string} mxc + * @param {string | null | undefined} mxc * @returns {string | undefined} */ function getPublicUrlForMxc(mxc) { @@ -238,7 +258,7 @@ function getPublicUrlForMxc(mxc) { } /** - * @param {string} mxc + * @param {string | null | undefined} mxc * @returns {string | undefined} mxc URL with protocol stripped, e.g. "cadence.moe/abcdef1234" */ function makeMxcPublic(mxc) { @@ -289,7 +309,7 @@ function roomHasAtLeastVersion(roomVersionString, desiredVersion) { */ function removeCreatorsFromPowerLevels(roomCreateOuter, powerLevels) { assert(roomCreateOuter.sender) - if (roomHasAtLeastVersion(roomCreateOuter.content.room_version, 12)) { + if (roomHasAtLeastVersion(roomCreateOuter.content.room_version, 12) && powerLevels.users) { for (const creator of (roomCreateOuter.content.additional_creators ?? []).concat(roomCreateOuter.sender)) { delete powerLevels.users[creator] } @@ -385,6 +405,7 @@ module.exports.makeMxcPublic = makeMxcPublic module.exports.getPublicUrlForMxc = getPublicUrlForMxc module.exports.getEventIDHash = getEventIDHash module.exports.MatrixStringBuilder = MatrixStringBuilder +module.exports.MatrixStringBuilderStack = MatrixStringBuilderStack module.exports.getViaServers = getViaServers module.exports.getViaServersQuery = getViaServersQuery module.exports.roomHasAtLeastVersion = roomHasAtLeastVersion diff --git a/src/web/routes/download-discord.js b/src/web/routes/download-discord.js index 3c58a75b..769fc9c0 100644 --- a/src/web/routes/download-discord.js +++ b/src/web/routes/download-discord.js @@ -31,7 +31,7 @@ function getSnow(event) { /** @type {Map>} */ const cache = new Map() -/** @param {string | undefined} url */ +/** @param {string} url */ function timeUntilExpiry(url) { const params = new URL(url).searchParams const ex = params.get("ex") diff --git a/src/web/routes/download-matrix.test.js b/src/web/routes/download-matrix.test.js index 49a63496..ccbcfddf 100644 --- a/src/web/routes/download-matrix.test.js +++ b/src/web/routes/download-matrix.test.js @@ -5,6 +5,7 @@ const {convertImageStream} = require("../../m2d/converters/emoji-sheet") const tryToCatch = require("try-to-catch") const {test} = require("supertape") const {router} = require("../../../test/web") +const streamWeb = require("stream/web") test("web download matrix: access denied if not a known attachment", async t => { const [error] = await tryToCatch(() => @@ -27,6 +28,7 @@ test("web download matrix: works if a known attachment", async t => { }, event, api: { + // @ts-ignore async getMedia(mxc, init) { return new Response("", {status: 200, headers: {"content-type": "image/png"}}) } diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index 0af37e2c..4f140a3d 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -54,8 +54,8 @@ function getAPI(event) { const validNonce = new LRUCache({max: 200}) /** - * @param {{type: number, parent_id?: string, position?: number}} channel - * @param {Map} channels + * @param {{type: number, parent_id?: string | null, position?: number}} channel + * @param {Map} channels */ function getPosition(channel, channels) { let position = 0 @@ -65,9 +65,11 @@ function getPosition(channel, channels) { // Categories are size 2000. let foundCategory = channel while (foundCategory.parent_id) { - foundCategory = channels.get(foundCategory.parent_id) + const f = channels.get(foundCategory.parent_id) + assert(f) + foundCategory = f } - if (foundCategory.type === DiscordTypes.ChannelType.GuildCategory) position = (foundCategory.position + 1) * 2000 + if (foundCategory.type === DiscordTypes.ChannelType.GuildCategory) position = ((foundCategory.position || 0) + 1) * 2000 // Categories always appear above what they contain. if (channel.type === DiscordTypes.ChannelType.GuildCategory) position -= 0.5 @@ -81,7 +83,7 @@ function getPosition(channel, channels) { // Threads appear below their channel. if ([DiscordTypes.ChannelType.PublicThread, DiscordTypes.ChannelType.PrivateThread, DiscordTypes.ChannelType.AnnouncementThread].includes(channel.type)) { position += 0.5 - let parent = channels.get(channel.parent_id) + let parent = channels.get(channel.parent_id || "") if (parent && parent["position"]) position += parent["position"] } @@ -98,7 +100,11 @@ function getChannelRoomsLinks(guild, rooms, roles) { assert(channelIDs) let linkedChannels = select("channel_room", ["channel_id", "room_id", "name", "nick"], {channel_id: channelIDs}).all() - let linkedChannelsWithDetails = linkedChannels.map(c => ({channel: discord.channels.get(c.channel_id), ...c})) + let linkedChannelsWithDetails = linkedChannels.map(c => ({ + // @ts-ignore + /** @type {DiscordTypes.APIGuildChannel} */ channel: discord.channels.get(c.channel_id), + ...c + })) let removedUncachedChannels = dUtils.filterTo(linkedChannelsWithDetails, c => c.channel) let linkedChannelIDs = linkedChannelsWithDetails.map(c => c.channel_id) linkedChannelsWithDetails.sort((a, b) => getPosition(a.channel, discord.channels) - getPosition(b.channel, discord.channels)) diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 8649348b..10596f25 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -1,5 +1,6 @@ // @ts-check +const assert = require("assert").strict const {z} = require("zod") const {defineEventHandler, createError, readValidatedBody, setResponseHeader, H3Event} = require("h3") const Ty = require("../../types") @@ -77,7 +78,9 @@ as.router.post("/api/link-space", defineEventHandler(async event => { const existing = select("guild_space", "guild_id", {}, "WHERE guild_id = ? OR space_id = ?").get(guildID, spaceID) if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) - const via = [inviteRow.mxid.match(/:(.*)/)[1]] + const inviteServer = inviteRow.mxid.match(/:(.*)/)?.[1] + assert(inviteServer) + const via = [inviteServer] // Check space exists and bridge is joined try { From cd0b8bff2b358568fb826003193d3122751fab31 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 11:36:51 +1300 Subject: [PATCH 125/153] Add reset web password script --- scripts/reset-web-password.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 scripts/reset-web-password.js diff --git a/scripts/reset-web-password.js b/scripts/reset-web-password.js new file mode 100644 index 00000000..9131efb5 --- /dev/null +++ b/scripts/reset-web-password.js @@ -0,0 +1,17 @@ +// @ts-check + +const {reg, writeRegistration, registrationFilePath} = require("../src/matrix/read-registration") +const {prompt} = require("enquirer") + +;(async () => { + /** @type {{web_password: string}} */ + const passwordResponse = await prompt({ + type: "text", + name: "web_password", + message: "Choose a simple password (optional)" + }) + + reg.ooye.web_password = passwordResponse.web_password + writeRegistration(reg) + console.log("Saved. Restart Out Of Your Element to apply this change.") +})() From 314f37f6402862ee47ecfe275c87a445ee87d7f1 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 11:49:35 +1300 Subject: [PATCH 126/153] Add newline at end of registration to help shells --- src/matrix/read-registration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/read-registration.js b/src/matrix/read-registration.js index 9316158d..6dc64dd4 100644 --- a/src/matrix/read-registration.js +++ b/src/matrix/read-registration.js @@ -22,7 +22,7 @@ function checkRegistration(reg) { /* c8 ignore next 4 */ /** @param {import("../types").AppServiceRegistrationConfig} reg */ function writeRegistration(reg) { - fs.writeFileSync(registrationFilePath, JSON.stringify(reg, null, 2)) + fs.writeFileSync(registrationFilePath, JSON.stringify(reg, null, 2) + "\n") } /** From 6df931f848d01af9f9951d3da8fdd0abdc8452fd Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 11:49:45 +1300 Subject: [PATCH 127/153] Check if we got rugpulled while sending --- src/d2m/actions/send-message.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index 283f2ec8..eb919bb5 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -86,7 +86,19 @@ async function sendMessage(message, channel, guild, row) { const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, part, reactionPart) // source 1 = discord + eventIDs.push(eventID) + + try { + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, part, reactionPart) // source 1 = discord + } catch (e) { + // check if we got rugpulled + if (!select("message_room", "message_id", {message_id: message.id}).get()) { + for (const eventID of eventIDs) { + await api.redactEvent(roomID, eventID) + } + return [] + } + } // The primary event is part = 0 and has the most important and distinct information. It is used to provide reply previews, be pinned, and possibly future uses. // The first event is chosen to be the primary part because it is usually the message text content and is more likely to be distinct. @@ -123,8 +135,6 @@ async function sendMessage(message, channel, guild, row) { db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(eventID, eventType, event.msgtype || null, sentResultsMessage.id, 1, 0) })() } - - eventIDs.push(eventID) } return eventIDs From 7ebe8aa042008cdc25d20edb5dfed5e6bb54823a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 11 Feb 2026 12:21:41 +1300 Subject: [PATCH 128/153] Fix backfill script --- .gitignore | 1 + scripts/backfill.js | 5 +- src/d2m/discord-packets.js | 1 + src/discord/register-interactions.js | 141 ++++++++++++++------------- 4 files changed, 78 insertions(+), 70 deletions(-) diff --git a/.gitignore b/.gitignore index e533dce5..17986439 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ config.js registration.yaml ooye.db* events.db* +backfill.db* # Automatically generated node_modules diff --git a/scripts/backfill.js b/scripts/backfill.js index 12d9da3d..27600f09 100644 --- a/scripts/backfill.js +++ b/scripts/backfill.js @@ -29,6 +29,9 @@ const DiscordClient = require("../src/d2m/discord-client") const discord = new DiscordClient(reg.ooye.discord_token, "half") passthrough.discord = discord +const {as} = require("../src/matrix/appservice") +passthrough.as = as + const orm = sync.require("../src/db/orm") passthrough.from = orm.from passthrough.select = orm.select @@ -69,7 +72,7 @@ async function event(event) { backfill: true, ...message } - await eventDispatcher.onMessageCreate(discord, simulatedGatewayDispatchData) + await eventDispatcher.MESSAGE_CREATE(discord, simulatedGatewayDispatchData) preparedInsert.run(channelID, message.id) } last = messages.at(-1)?.id diff --git a/src/d2m/discord-packets.js b/src/d2m/discord-packets.js index ed45324e..8cf2fde2 100644 --- a/src/d2m/discord-packets.js +++ b/src/d2m/discord-packets.js @@ -47,6 +47,7 @@ const utils = { if (listen === "full") { try { + interactions.registerInteractions() await eventDispatcher.checkMissedExpressions(message.d) await eventDispatcher.checkMissedPins(client, message.d) await eventDispatcher.checkMissedMessages(client, message.d) diff --git a/src/discord/register-interactions.js b/src/discord/register-interactions.js index a9093938..46a7360f 100644 --- a/src/discord/register-interactions.js +++ b/src/discord/register-interactions.js @@ -15,75 +15,77 @@ const ping = sync.require("./interactions/ping.js") // User must have EVERY permission in default_member_permissions to be able to use the command -discord.snow.interaction.bulkOverwriteApplicationCommands(id, [{ - name: "Matrix info", - contexts: [DiscordTypes.InteractionContextType.Guild], - type: DiscordTypes.ApplicationCommandType.Message, -}, { - name: "Permissions", - contexts: [DiscordTypes.InteractionContextType.Guild], - type: DiscordTypes.ApplicationCommandType.Message, - default_member_permissions: String(DiscordTypes.PermissionFlagsBits.KickMembers | DiscordTypes.PermissionFlagsBits.ManageRoles) -}, { - name: "Responses", - contexts: [DiscordTypes.InteractionContextType.Guild], - type: DiscordTypes.ApplicationCommandType.Message -}, { - name: "invite", - contexts: [DiscordTypes.InteractionContextType.Guild], - type: DiscordTypes.ApplicationCommandType.ChatInput, - description: "Invite a Matrix user to this Discord server", - default_member_permissions: String(DiscordTypes.PermissionFlagsBits.CreateInstantInvite), - options: [ - { - type: DiscordTypes.ApplicationCommandOptionType.String, - description: "The Matrix user to invite, e.g. @username:example.org", - name: "user" - } - ], -}, { - name: "ping", - contexts: [DiscordTypes.InteractionContextType.Guild], - type: DiscordTypes.ApplicationCommandType.ChatInput, - description: "Ping a Matrix user.", - options: [ - { - type: DiscordTypes.ApplicationCommandOptionType.String, - description: "Display name or ID of the Matrix user", - name: "user", - autocomplete: true, - required: true - } - ] -}, { - name: "privacy", - contexts: [DiscordTypes.InteractionContextType.Guild], - type: DiscordTypes.ApplicationCommandType.ChatInput, - description: "Change whether Matrix users can join through direct invites, links, or the public directory.", - default_member_permissions: String(DiscordTypes.PermissionFlagsBits.ManageGuild), - options: [ - { - type: DiscordTypes.ApplicationCommandOptionType.String, - description: "Check or set the new privacy level", - name: "level", - choices: [{ - name: "❓️ Check the current privacy level and get more information.", - value: "info" - }, { - name: "🤝 Only allow joining with a direct in-app invite from another user. No shareable invite links.", - value: "invite" - }, { - name: "🔗 Matrix links can be created and shared like Discord's invite links. In-app invites still work.", - value: "link", - }, { - name: "🌏️ Publicly visible in the Matrix directory, like Server Discovery. Invites and links still work.", - value: "directory" - }] - } - ] -}]).catch(e => { - console.error(e) -}) +function registerInteractions() { + discord.snow.interaction.bulkOverwriteApplicationCommands(id, [{ + name: "Matrix info", + contexts: [DiscordTypes.InteractionContextType.Guild], + type: DiscordTypes.ApplicationCommandType.Message, + }, { + name: "Permissions", + contexts: [DiscordTypes.InteractionContextType.Guild], + type: DiscordTypes.ApplicationCommandType.Message, + default_member_permissions: String(DiscordTypes.PermissionFlagsBits.KickMembers | DiscordTypes.PermissionFlagsBits.ManageRoles) + }, { + name: "Responses", + contexts: [DiscordTypes.InteractionContextType.Guild], + type: DiscordTypes.ApplicationCommandType.Message + }, { + name: "invite", + contexts: [DiscordTypes.InteractionContextType.Guild], + type: DiscordTypes.ApplicationCommandType.ChatInput, + description: "Invite a Matrix user to this Discord server", + default_member_permissions: String(DiscordTypes.PermissionFlagsBits.CreateInstantInvite), + options: [ + { + type: DiscordTypes.ApplicationCommandOptionType.String, + description: "The Matrix user to invite, e.g. @username:example.org", + name: "user" + } + ], + }, { + name: "ping", + contexts: [DiscordTypes.InteractionContextType.Guild], + type: DiscordTypes.ApplicationCommandType.ChatInput, + description: "Ping a Matrix user.", + options: [ + { + type: DiscordTypes.ApplicationCommandOptionType.String, + description: "Display name or ID of the Matrix user", + name: "user", + autocomplete: true, + required: true + } + ] + }, { + name: "privacy", + contexts: [DiscordTypes.InteractionContextType.Guild], + type: DiscordTypes.ApplicationCommandType.ChatInput, + description: "Change whether Matrix users can join through direct invites, links, or the public directory.", + default_member_permissions: String(DiscordTypes.PermissionFlagsBits.ManageGuild), + options: [ + { + type: DiscordTypes.ApplicationCommandOptionType.String, + description: "Check or set the new privacy level", + name: "level", + choices: [{ + name: "❓️ Check the current privacy level and get more information.", + value: "info" + }, { + name: "🤝 Only allow joining with a direct in-app invite from another user. No shareable invite links.", + value: "invite" + }, { + name: "🔗 Matrix links can be created and shared like Discord's invite links. In-app invites still work.", + value: "link", + }, { + name: "🌏️ Publicly visible in the Matrix directory, like Server Discovery. Invites and links still work.", + value: "directory" + }] + } + ] + }]).catch(e => { + console.error(e) + }) +} /** @param {DiscordTypes.APIInteraction} interaction */ async function dispatchInteraction(interaction) { @@ -147,3 +149,4 @@ async function dispatchInteraction(interaction) { } module.exports.dispatchInteraction = dispatchInteraction +module.exports.registerInteractions = registerInteractions From 228766cec00e40f976907c3e231bdfff2d401a30 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 12 Feb 2026 01:27:49 +1300 Subject: [PATCH 129/153] Change how edit timestamps are treated again --- src/d2m/converters/edit-to-changes.js | 8 ++++++-- test/data.js | 7 +------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index b73d6e00..cfbd1e2d 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -146,10 +146,14 @@ async function editToChanges(message, guild, api) { } // Don't post new generated embeds for messages if it's been a while since the message was sent. Detached embeds look weird. - const messageTooOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 30 * 1000 // older than 30 seconds ago + const messageQuiteOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 30 * 1000 // older than 30 seconds ago + // Don't send anything new at all if it's been longer since the message was sent. Detached messages are just inappropriate. + const messageReallyOld = message.timestamp && new Date(message.timestamp).getTime() < Date.now() - 2 * 60 * 1000 // older than 2 minutes ago // Don't post new generated embeds for messages if the setting was disabled. const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 - if ((messageTooOld || !embedsEnabled) && !message.author.bot) { + if (messageReallyOld) { + eventsToSend = [] // Only allow edits to change and delete, but not send new. + } else if ((messageQuiteOld || !embedsEnabled) && !message.author.bot) { eventsToSend = eventsToSend.filter(e => e.msgtype !== "m.notice") // Only send events that aren't embeds. } diff --git a/test/data.js b/test/data.js index 4854f6a5..eef3a50d 100644 --- a/test/data.js +++ b/test/data.js @@ -239,7 +239,7 @@ module.exports = { unicode_emoji: null, tags: {}, position: 0, - permissions: '559623605575360', + permissions: '1122573558996672', name: '@everyone', mentionable: false, managed: false, @@ -5474,7 +5474,6 @@ module.exports = { mention_roles: [], mentions: [], pinned: false, - timestamp: "2023-08-16T22:38:38.641000+00:00", tts: false, type: 0 }, @@ -5548,7 +5547,6 @@ module.exports = { mention_roles: [], mentions: [], pinned: false, - timestamp: "2023-08-16T22:38:38.641000+00:00", tts: false, type: 0 }, @@ -5583,7 +5581,6 @@ module.exports = { pinned: false, mention_everyone: false, tts: false, - timestamp: "2023-05-11T23:44:09.690000+00:00", edited_timestamp: "2023-05-11T23:44:19.690000+00:00", flags: 0, components: [], @@ -5624,7 +5621,6 @@ module.exports = { pinned: false, mention_everyone: false, tts: false, - timestamp: "2023-05-11T23:44:09.690000+00:00", edited_timestamp: "2023-05-11T23:44:19.690000+00:00", flags: 0, components: [], @@ -5665,7 +5661,6 @@ module.exports = { pinned: false, mention_everyone: false, tts: false, - timestamp: "2023-05-11T23:44:09.690000+00:00", edited_timestamp: "2023-05-11T23:44:19.690000+00:00", flags: 0, components: [], From 0d574c1370e0041dec78f5c2792fe054dcad0567 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 12 Feb 2026 13:46:50 +1300 Subject: [PATCH 130/153] Fix PluralKit replies (properly) --- src/d2m/converters/message-to-event.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index b36bdf5d..6109b0f9 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -259,7 +259,7 @@ async function pollToEvent(poll) { * @returns {Promise<{$type: string, $sender?: string, [x: string]: any}[]>} */ async function messageToEvent(message, guild, options = {}, di) { - message = {...message} + message = structuredClone(message) const events = [] /* c8 ignore next 7 */ From 8ea29d6c2779da761bf1d091965bdd55928b311e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 12 Feb 2026 16:01:48 +1300 Subject: [PATCH 131/153] Fix link escaping breaking with suppressed links --- src/m2d/converters/event-to-message.js | 2 +- src/m2d/converters/event-to-message.test.js | 63 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index f6baf550..9460ebb2 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -69,7 +69,7 @@ turndownService.escape = function (string) { return string.replace(/\s+|\S+/g, part => { // match chunks of spaces or non-spaces if (part.match(/\s/)) return part // don't process spaces - if (part.match(/^https?:\/\//)) { + if (part.match(/^ { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "the wikimedia commons freaks are gonna love this one https://commons.wikimedia.org/wiki/File:Car_covered_in_traffic_cones.jpg" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "the wikimedia commons freaks are gonna love this one https://commons.wikimedia.org/wiki/File:Car_covered_in_traffic_cones.jpg", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: markdown in link url does not attempt to be escaped (plaintext body, link suppressed)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "the wikimedia commons freaks are gonna love this one https://commons.wikimedia.org/wiki/File:Car_covered_in_traffic_cones.jpg" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message" + }, { + id: "123", + roles: [{ + id: "123", + name: "@everyone", + permissions: DiscordTypes.PermissionFlagsBits.SendMessages + }] + }, {}), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "the wikimedia commons freaks are gonna love this one ", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + test("event2message: embeds are suppressed if the guild does not have embed links permission (formatted body)", async t => { t.deepEqual( await eventToMessage({ From e54536d965d7abd85d0d373900d305ac36921450 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 12 Feb 2026 19:24:50 +1300 Subject: [PATCH 132/153] Check for members gateway intent as well It was reported that this is required for Log in with Discord to work. --- scripts/setup.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/setup.js b/scripts/setup.js index 2d6b2376..696eec94 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -185,8 +185,8 @@ function defineEchoHandler() { }) const intentFlagPossibilities = [ - DiscordTypes.ApplicationFlags.GatewayMessageContent | DiscordTypes.ApplicationFlags.GatewayPresence, - DiscordTypes.ApplicationFlags.GatewayMessageContentLimited | DiscordTypes.ApplicationFlags.GatewayPresenceLimited + DiscordTypes.ApplicationFlags.GatewayMessageContent | DiscordTypes.ApplicationFlags.GatewayPresence | DiscordTypes.ApplicationFlags.GatewayGuildMembers, + DiscordTypes.ApplicationFlags.GatewayMessageContentLimited | DiscordTypes.ApplicationFlags.GatewayPresenceLimited | DiscordTypes.ApplicationFlags.GatewayGuildMembersLimited ] const intentFlagMask = intentFlagPossibilities.reduce((a, c) => a | c, 0) if (!intentFlagPossibilities.includes(client.flags & intentFlagMask)) { From 1defd83fdef31782be2a5e61a1d85e665df1c163 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 12 Feb 2026 19:43:29 +1300 Subject: [PATCH 133/153] Sync create polls permission from Discord --- src/d2m/actions/create-room.js | 9 +++++++-- src/d2m/actions/register-user.js | 4 ++++ src/d2m/event-dispatcher.js | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index acd47d46..735b84c2 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -124,7 +124,11 @@ async function channelToKState(channel, guild, di) { const everyonePermissions = dUtils.getPermissions(guild.id, [], guild.roles, undefined, channel.permission_overwrites) const everyoneCanSend = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendMessages) - const everyoneCanMentionEveryone = dUtils.hasAllPermissions(everyonePermissions, ["MentionEveryone"]) + const everyoneCanMentionEveryone = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.MentionEveryone) + + const pollStartPowerLevel = {} + const everyoneCanCreatePolls = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendPolls) + if (everyoneCanSend && !everyoneCanCreatePolls) pollStartPowerLevel["org.matrix.msc3381.poll.start"] = 10 const spacePowerDetails = await mUtils.getEffectivePower(guildSpaceID, [], di.api) spacePowerDetails.powerLevels.users ??= {} @@ -159,7 +163,8 @@ async function channelToKState(channel, guild, di) { events_default: everyoneCanSend ? 0 : READ_ONLY_ROOM_EVENTS_DEFAULT_POWER, events: { "m.reaction": 0, - "m.room.redaction": 0 // only affects redactions of own events, required to be able to un-react + "m.room.redaction": 0, // only affects redactions of own events, required to be able to un-react + ...pollStartPowerLevel }, notifications: { room: everyoneCanMentionEveryone ? 0 : 20 diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index d20bfb8a..1bdd6e30 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -182,6 +182,10 @@ function memberToPowerLevel(user, member, guild, channel) { const everyoneCanMentionEveryone = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.MentionEveryone) const userCanMentionEveryone = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.MentionEveryone) if (!everyoneCanMentionEveryone && userCanMentionEveryone) return 20 + /* PL 10 = Create Polls for technical reasons. */ + const everyoneCanCreatePolls = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendPolls) + const userCanCreatePolls = dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.SendPolls) + if (!everyoneCanCreatePolls && userCanCreatePolls) return 10 return 0 } diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index af18669d..c3bba33a 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -196,6 +196,21 @@ module.exports = { await createSpace.syncSpace(guild) }, + /** + * @param {import("./discord-client")} client + * @param {DiscordTypes.GatewayGuildRoleUpdateDispatchData} data + */ + async GUILD_ROLE_UPDATE(client, data) { + const guild = client.guilds.get(data.guild_id) + if (!guild) return + const spaceID = select("guild_space", "space_id", {guild_id: data.guild_id}).pluck().get() + if (!spaceID) return + + if (data.role.id === data.guild_id) { // @everyone role changed - find a way to do this more efficiently in the future to handle many role updates + await createSpace.syncSpaceFully(guild) + } + }, + /** * @param {import("./discord-client")} client * @param {DiscordTypes.GatewayChannelUpdateDispatchData} channelOrThread From 35e9c9e1ea2d35301ea102496bb28499cf75e1d2 Mon Sep 17 00:00:00 2001 From: Elliu Date: Fri, 13 Feb 2026 19:12:01 +1300 Subject: [PATCH 134/153] Add unlink space feature Squashed commit of the following: commit bd9fd5cd3cf3f1301df18074c997ec537a81b4f5 Author: Elliu Date: Sat Nov 15 15:32:18 2025 +0900 Revert "fix matrix / db resource cleanup on space unlink" This reverts commit ccc10564f1e33ab277bc15f360b8c65f2d0ea867. commit eec559293861305394770343d501389905fe1650 Author: Cadence Ember Date: Sat Nov 8 13:01:59 2025 +1300 Dependency inject snow for testing commit b45eeb150e0702c201b8f710a3bdaa8e9f7d90be Author: Elliu Date: Wed Nov 5 00:20:20 2025 +0900 manually revert 3597a3b: "Factorize some of the space link/unlink sanity checks" commit 0f2e575df21bf940e4780c30d2701da989f62471 Author: Elliu Date: Wed Nov 5 00:04:38 2025 +0900 on unbriding room, also demote powel level of bridge user in matrix room commit ccc10564f1e33ab277bc15f360b8c65f2d0ea867 Author: Elliu Date: Wed Nov 5 00:04:13 2025 +0900 fix matrix / db resource cleanup on space unlink commit f4c1ea7c7f7d5a265b84ce464cd8e9e26d934a32 Author: Elliu Date: Tue Nov 4 23:54:41 2025 +0900 /unlink-space: properly leave guild and clean DB commit 5f0ec3b2c861cc8b9edc51389d6176c7a22a1135 Author: Cadence Ember Date: Sun Nov 2 22:31:14 2025 +1300 Improve HTML to a state I'm happy with commit 16309f26b3dd72927e05454cee8c63504b447b7f Author: Elliu Date: Sat Nov 1 22:24:51 2025 +0900 add tests from /unlink-space endpoint commit 5aff6f9048330a86eda3b2d1862f42df8d2bad84 Author: Elliu Date: Sat Sep 6 20:05:18 2025 +0900 Add /api/unlink-space implementation commit dfc61594f68db4b52b3553ac7d3561ae9ce13b49 Author: Elliu Date: Sat Sep 6 19:59:44 2025 +0900 Extract /api/unlink code to its own function commit 3597a3b5ce9dde3a9ddfe0853253bfda91a38335 Author: Elliu Date: Sat Sep 6 19:28:42 2025 +0900 Factorize some of the space link/unlink sanity checks commit 05d788e26394106d9be24cef8b38f6c6f1e4c984 Author: Elliu Date: Sat Sep 6 18:23:01 2025 +0900 Add button to unlink a space Co-authored-by: Cadence Ember --- src/d2m/actions/create-room.js | 27 ++--- src/d2m/actions/create-space.js | 2 +- src/d2m/event-dispatcher.js | 2 +- src/web/pug/guild.pug | 22 +++- src/web/pug/guild_not_linked.pug | 19 ++- src/web/pug/includes/template.pug | 7 ++ src/web/routes/link.js | 111 ++++++++++++++---- src/web/routes/link.test.js | 185 +++++++++++++++++++++++++++++- test/ooye-test-data.sql | 2 +- 9 files changed, 322 insertions(+), 55 deletions(-) diff --git a/src/d2m/actions/create-room.js b/src/d2m/actions/create-room.js index 735b84c2..651eaf41 100644 --- a/src/d2m/actions/create-room.js +++ b/src/d2m/actions/create-room.js @@ -439,19 +439,11 @@ function syncRoom(channelID) { return _syncRoom(channelID, true) } -async function unbridgeChannel(channelID) { - /** @ts-ignore @type {DiscordTypes.APIGuildChannel} */ - const channel = discord.channels.get(channelID) - assert.ok(channel) - assert.ok(channel.guild_id) - return unbridgeDeletedChannel(channel, channel.guild_id) -} - /** * @param {{id: string, topic?: string?}} channel channel-ish (just needs an id, topic is optional) * @param {string} guildID */ -async function unbridgeDeletedChannel(channel, guildID) { +async function unbridgeChannel(channel, guildID) { const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get() assert.ok(roomID) const row = from("guild_space").join("guild_active", "guild_id").select("space_id", "autocreate").where({guild_id: guildID}).get() @@ -488,14 +480,13 @@ async function unbridgeDeletedChannel(channel, guildID) { if (!botInRoom) return - // demote admins in room - /** @type {Ty.Event.M_Power_Levels} */ - const powerLevelContent = await api.getStateEvent(roomID, "m.room.power_levels", "") - powerLevelContent.users ??= {} - for (const mxid of Object.keys(powerLevelContent.users)) { - if (powerLevelContent.users[mxid] >= 100 && mUtils.eventSenderIsFromDiscord(mxid) && mxid !== mUtils.bot) { - delete powerLevelContent.users[mxid] - await api.sendState(roomID, "m.room.power_levels", "", powerLevelContent, mxid) + // demote discord sim admins in room + const {powerLevels, allCreators} = await mUtils.getEffectivePower(roomID, [], api) + const powerLevelsUsers = (powerLevels.users ||= {}) + for (const mxid of Object.keys(powerLevelsUsers)) { + if (powerLevelsUsers[mxid] >= (powerLevels.state_default ?? 50) && !allCreators.includes(mxid) && mUtils.eventSenderIsFromDiscord(mxid) && mxid !== mUtils.bot) { + delete powerLevelsUsers[mxid] + await api.sendState(roomID, "m.room.power_levels", "", powerLevels, mxid) // done individually because each user must demote themselves } } @@ -526,6 +517,7 @@ async function unbridgeDeletedChannel(channel, guildID) { } // leave room + await mUtils.setUserPower(roomID, mUtils.bot, 0, api) await api.leaveRoom(roomID) } @@ -589,6 +581,5 @@ module.exports.postApplyPowerLevels = postApplyPowerLevels module.exports._convertNameAndTopic = convertNameAndTopic module.exports._syncSpaceMember = _syncSpaceMember module.exports.unbridgeChannel = unbridgeChannel -module.exports.unbridgeDeletedChannel = unbridgeDeletedChannel module.exports.existsOrAutocreatable = existsOrAutocreatable module.exports.assertExistsOrAutocreatable = assertExistsOrAutocreatable diff --git a/src/d2m/actions/create-space.js b/src/d2m/actions/create-space.js index 1417b2d0..7a751e27 100644 --- a/src/d2m/actions/create-space.js +++ b/src/d2m/actions/create-space.js @@ -203,7 +203,7 @@ async function syncSpaceFully(guildID) { if (discord.channels.has(channelID)) { await createRoom.syncRoom(channelID) } else { - await createRoom.unbridgeDeletedChannel({id: channelID}, guildID) + await createRoom.unbridgeChannel({id: channelID}, guildID) } } diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index c3bba33a..01bbc675 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -250,7 +250,7 @@ module.exports = { const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get() if (!roomID) return // channel wasn't being bridged in the first place // @ts-ignore - await createRoom.unbridgeDeletedChannel(channel, guildID) + await createRoom.unbridgeChannel(channel, guildID) }, /** diff --git a/src/web/pug/guild.pug b/src/web/pug/guild.pug index c219e29d..a9e770b0 100644 --- a/src/web/pug/guild.pug +++ b/src/web/pug/guild.pug @@ -75,6 +75,7 @@ block body button.s-btn(class=space_id ? "s-btn__muted" : "s-btn__filled" hx-get=rel(`/qr?guild_id=${guild_id}`) hx-indicator="closest button" hx-swap="outerHTML" hx-disabled-elt="this") Show QR if space_id + h2.mt48.fs-headline1 Server settings h3.mt32.fs-category Privacy level span#privacy-level-loading .s-card @@ -104,7 +105,7 @@ block body p.s-description.m0 Shareable invite links, like Discord p.s-description.m0 Publicly listed in directory, like Discord server discovery - h2.mt48.fs-headline1 Features + h3.mt32.fs-category Features .s-card.d-grid.px0.g16 form.d-flex.ai-center.g16 #url-preview-loading.p8 @@ -138,13 +139,13 @@ block body h3.mt32.fs-category Linked channels .s-card.bs-sm.p0 - form.s-table-container(method="post" action=rel("/api/unlink") hx-confirm="Do you want to unlink these channels?\nIt may take a moment to clean up Matrix resources.") + form.s-table-container(method="post" action=rel("/api/unlink")) input(type="hidden" name="guild_id" value=guild_id) table.s-table.s-table__bx-simple each row in linkedChannelsWithDetails tr td.w40: +discord(row.channel) - td.p2: button.s-btn.s-btn__muted.s-btn__xs(name="channel_id" value=row.channel.id hx-post=rel("/api/unlink") hx-trigger="click" hx-disabled-elt="this")!= icons.Icons.IconLinkSm + td.p2: button.s-btn.s-btn__muted.s-btn__xs(name="channel_id" cx-prevent-default hx-post=rel("/api/unlink") hx-confirm="Do you want to unlink these channels?\nIt may take a moment to clean up Matrix resources." value=row.channel.id hx-indicator="this" hx-disabled-elt="this")!= icons.Icons.IconLinkSm td: +matrix(row) else tr @@ -185,6 +186,19 @@ block body != icons.Icons.IconMerge = ` Link` + h3.mt32.fs-category Unlink server + form.s-card.d-flex.gx16.ai-center(method="post" action=rel("/api/unlink-space")) + input(type="hidden" name="guild_id" value=guild.id) + .fl-grow1.s-prose.s-prose__sm.lh-lg + p.fc-medium. + Not using this bridge, or just made a mistake? You can unlink the whole server and all its channels.#[br] + This may take a minute to process. Please be patient and wait until the page refreshes. + div + button.s-btn.s-btn__icon.s-btn__danger.s-btn__outlined(cx-prevent-default hx-post=rel("/api/unlink-space") hx-confirm="Do you want to unlink this server and all its channels?\nIt may take a minute to clean up Matrix resources." hx-indicator="this" hx-disabled-elt="this") + != icons.Icons.IconUnsync + span.ml4= ` Unlink` + + if space_id details.mt48 summary Debug room list .d-grid.grid__2.gx24 @@ -205,7 +219,7 @@ block body ul.my8.ml24 each row in removedWrongTypeChannels li: a(href=`https://discord.com/channels/${guild_id}/${row.id}`) (#{row.type}) #{row.name} - h3.mt24 Unavailable channels: Bridge can't access + h3.mt24 Unavailable channels: Discord bot can't access .s-card.p0 ul.my8.ml24 each row in removedPrivateChannels diff --git a/src/web/pug/guild_not_linked.pug b/src/web/pug/guild_not_linked.pug index 61c57e9b..04d2dae3 100644 --- a/src/web/pug/guild_not_linked.pug +++ b/src/web/pug/guild_not_linked.pug @@ -42,12 +42,23 @@ block body | You need to log in with Matrix first. a.s-btn.s-btn__matrix.s-btn__outlined(href=rel(`/log-in-with-matrix`, {next: `./guild?guild_id=${guild_id}`})) Log in with Matrix - h3.mt48.fs-category Auto-create - .s-card + h3.mt48.fs-category Other choices + .s-card.d-grid.g16 form.d-flex.ai-center.g8(method="post" action=rel("/api/autocreate") hx-post=rel("/api/autocreate") hx-indicator="#easy-mode-button") input(type="hidden" name="guild_id" value=guild_id) input(type="hidden" name="autocreate" value="true") label.s-label.fl-grow1 - | Changed your mind? + | Do it automatically p.s-description If you want, OOYE can create and manage the Matrix space so you don't have to. - button.s-btn.s-btn__outlined#easy-mode-button Use easy mode + button.s-btn.s-btn__icon.s-btn__outlined#easy-mode-button + != icons.Icons.IconWand + span.ml4= ` Use easy mode` + + form.d-flex.gx16.ai-center(method="post" action=rel("/api/unlink-space")) + input(type="hidden" name="guild_id" value=guild.id) + label.s-label.fl-grow1 + | Cancel + p.s-description Don't want to link this server after all? Here's the button for you. + button.s-btn.s-btn__icon.s-btn__muted.s-btn__outlined(cx-prevent-default hx-post=rel("/api/unlink-space") hx-indicator="this" hx-disabled-elt="this") + != icons.Icons.IconUnsync + span.ml4= ` Unlink` diff --git a/src/web/pug/includes/template.pug b/src/web/pug/includes/template.pug index 93aaefc4..452f8d53 100644 --- a/src/web/pug/includes/template.pug +++ b/src/web/pug/includes/template.pug @@ -129,6 +129,13 @@ html(lang="en") document.styleSheets[0].insertRule(t, document.styleSheets[0].cssRules.length) }) }) + //- Prevent default + script. + document.querySelectorAll("[cx-prevent-default]").forEach(e => { + e.addEventListener("click", event => { + event.preventDefault() + }) + }) script(src=rel("/static/htmx.js")) //- Error dialog aside.s-modal#server-error(aria-hidden="true") diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 10596f25..3ae6de5f 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -9,11 +9,8 @@ const DiscordTypes = require("discord-api-types/v10") const {discord, db, as, sync, select, from} = require("../../passthrough") /** @type {import("../auth")} */ const auth = sync.require("../auth") -/** @type {import("../../matrix/mreq")} */ -const mreq = sync.require("../../matrix/mreq") /** @type {import("../../matrix/utils")}*/ const utils = sync.require("../../matrix/utils") -const {reg} = require("../../matrix/read-registration") /** * @param {H3Event} event @@ -42,6 +39,15 @@ function getCreateSpace(event) { return event.context.createSpace || sync.require("../../d2m/actions/create-space") } +/** + * @param {H3Event} event + * @returns {import("snowtransfer").SnowTransfer} + */ +function getSnow(event) { + /* c8 ignore next */ + return event.context.snow || discord.snow +} + const schema = { linkSpace: z.object({ guild_id: z.string(), @@ -55,7 +61,37 @@ const schema = { unlink: z.object({ guild_id: z.string(), channel_id: z.string() - }) + }), + unlinkSpace: z.object({ + guild_id: z.string(), + }), +} + +/** + * @param {H3Event} event + * @param {string} channel_id + * @param {string} guild_id + */ +async function validateAndUnbridgeChannel(event, channel_id, guild_id) { + const createRoom = getCreateRoom(event) + + // Check channel is currently bridged + const row = select("channel_room", "channel_id", {channel_id: channel_id}).get() + if (!row) throw createError({status: 400, message: "Bad Request", data: `Channel ID ${channel_id} is not currently bridged`}) + + // Check that the channel (if it exists) is part of this guild + /** @type {any} */ + let channel = discord.channels.get(channel_id) + if (channel) { + if (!("guild_id" in channel) || channel.guild_id !== guild_id) throw createError({status: 400, message: "Bad Request", data: `Channel ID ${channel_id} is not part of guild ${guild_id}`}) + } else { + // Otherwise, if the channel isn't cached, it must have been deleted. + // There's no other authentication here - it's okay for anyone to unlink a deleted channel just by knowing its ID. + channel = {id: channel_id} + } + + // Do it + await createRoom.unbridgeChannel(channel, guild_id) } as.router.post("/api/link-space", defineEventHandler(async event => { @@ -195,7 +231,6 @@ as.router.post("/api/link", defineEventHandler(async event => { as.router.post("/api/unlink", defineEventHandler(async event => { const {channel_id, guild_id} = await readValidatedBody(event, schema.unlink.parse) const managed = await auth.getManagedGuilds(event) - const createRoom = getCreateRoom(event) // Check guild ID or nonce if (!managed.has(guild_id)) throw createError({status: 403, message: "Forbidden", data: "Can't edit a guild you don't have Manage Server permissions in"}) @@ -204,24 +239,56 @@ as.router.post("/api/unlink", defineEventHandler(async event => { const guild = discord.guilds.get(guild_id) if (!guild) throw createError({status: 400, message: "Bad Request", data: "Discord guild does not exist or bot has not joined it"}) - // Check that the channel (if it exists) is part of this guild - /** @type {any} */ - let channel = discord.channels.get(channel_id) - if (channel) { - if (!("guild_id" in channel) || channel.guild_id !== guild_id) throw createError({status: 400, message: "Bad Request", data: `Channel ID ${channel_id} is not part of guild ${guild_id}`}) - } else { - // Otherwise, if the channel isn't cached, it must have been deleted. - // There's no other authentication here - it's okay for anyone to unlink a deleted channel just by knowing its ID. - channel = {id: channel_id} - } - - // Check channel is currently bridged - const row = select("channel_room", "channel_id", {channel_id: channel_id}).get() - if (!row) throw createError({status: 400, message: "Bad Request", data: `Channel ID ${channel_id} is not currently bridged`}) - - // Do it - await createRoom.unbridgeDeletedChannel(channel, guild_id) + await validateAndUnbridgeChannel(event, channel_id, guild_id) setResponseHeader(event, "HX-Refresh", "true") return null // 204 })) + +as.router.post("/api/unlink-space", defineEventHandler(async event => { + const {guild_id} = await readValidatedBody(event, schema.unlinkSpace.parse) + const managed = await auth.getManagedGuilds(event) + const api = getAPI(event) + const snow = getSnow(event) + + // Check guild ID or nonce + if (!managed.has(guild_id)) throw createError({status: 403, message: "Forbidden", data: "Can't edit a guild you don't have Manage Server permissions in"}) + + // Check guild exists + const guild = discord.guilds.get(guild_id) + if (!guild) throw createError({status: 400, message: "Bad Request", data: "Discord guild does not exist or bot has not joined it"}) + + const active = select("guild_active", "guild_id", {guild_id: guild_id}).get() + if (!active) { + throw createError({status: 400, message: "Bad Request", data: "Discord guild has not been considered for bridging"}) + } + + // Check if there are Matrix resources + const spaceID = select("guild_space", "space_id", {guild_id: guild_id}).pluck().get() + if (spaceID) { + // Unlink all rooms + const linkedChannels = select("channel_room", ["channel_id", "room_id", "name", "nick"], {guild_id: guild_id}).all() + for (const channel of linkedChannels) { + await validateAndUnbridgeChannel(event, channel.channel_id, guild_id) + } + + // Verify all rooms were unlinked + const remainingLinkedChannels = select("channel_room", ["channel_id", "room_id", "name", "nick"], {guild_id: guild_id}).all() + if (remainingLinkedChannels.length) { + throw createError({status: 500, message: "Internal Server Error", data: "Failed to unlink some rooms. Please try doing it manually, or report a bug. The space will not be unlinked until all rooms are."}) + } + + // Unlink space + await utils.setUserPower(spaceID, utils.bot, 0, api) + await api.leaveRoom(spaceID) + db.prepare("DELETE FROM guild_space WHERE guild_id = ? AND space_id = ?").run(guild_id, spaceID) + } + + // Mark as not considered for bridging + db.prepare("DELETE FROM guild_active WHERE guild_id = ?").run(guild_id) + db.prepare("DELETE FROM invite WHERE room_id = ?").run(spaceID) + await snow.user.leaveGuild(guild_id) + + setResponseHeader(event, "HX-Redirect", "/") + return null +})) diff --git a/src/web/routes/link.test.js b/src/web/routes/link.test.js index 440bdfc5..e8473f85 100644 --- a/src/web/routes/link.test.js +++ b/src/web/routes/link.test.js @@ -613,7 +613,7 @@ test("web unlink room: checks that the channel is part of the guild", async t => t.equal(error.data, "Channel ID 112760669178241024 is not part of guild 665289423482519565") }) -test("web unlink room: successfully calls unbridgeDeletedChannel when the channel does exist", async t => { +test("web unlink room: successfully calls unbridgeChannel when the channel does exist", async t => { let called = 0 await router.test("post", "/api/unlink", { sessionData: { @@ -624,7 +624,7 @@ test("web unlink room: successfully calls unbridgeDeletedChannel when the channe guild_id: "665289423482519565" }, createRoom: { - async unbridgeDeletedChannel(channel) { + async unbridgeChannel(channel) { called++ t.equal(channel.id, "665310973967597573") } @@ -633,7 +633,7 @@ test("web unlink room: successfully calls unbridgeDeletedChannel when the channe t.equal(called, 1) }) -test("web unlink room: successfully calls unbridgeDeletedChannel when the channel does not exist", async t => { +test("web unlink room: successfully calls unbridgeChannel when the channel does not exist", async t => { let called = 0 await router.test("post", "/api/unlink", { sessionData: { @@ -644,7 +644,7 @@ test("web unlink room: successfully calls unbridgeDeletedChannel when the channe guild_id: "112760669178241024" }, createRoom: { - async unbridgeDeletedChannel(channel) { + async unbridgeChannel(channel) { called++ t.equal(channel.id, "489237891895768942") } @@ -654,7 +654,9 @@ test("web unlink room: successfully calls unbridgeDeletedChannel when the channe }) test("web unlink room: checks that the channel is bridged", async t => { + const row = db.prepare("SELECT * FROM channel_room WHERE channel_id = '665310973967597573'").get() db.prepare("DELETE FROM channel_room WHERE channel_id = '665310973967597573'").run() + const [error] = await tryToCatch(() => router.test("post", "/api/unlink", { sessionData: { managedGuilds: ["665289423482519565"] @@ -665,4 +667,179 @@ test("web unlink room: checks that the channel is bridged", async t => { } })) t.equal(error.data, "Channel ID 665310973967597573 is not currently bridged") + + db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom_avatar, last_bridged_pin_timestamp, speedbump_id, speedbump_checked, speedbump_webhook_id, guild_id, custom_topic) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(row.channel_id, row.room_id, row.name, row.nick, row.thread_parent, row.custom_avatar, row.last_bridged_pin_timestamp, row.speedbump_id, row.speedbump_checked, row.speedbump_webhook_id, row.guild_id, row.custom_topic) + const new_row = db.prepare("SELECT * FROM channel_room WHERE channel_id = '665310973967597573'").get() + t.deepEqual(row, new_row) +}) + +// ***** + +test("web unlink space: access denied if not logged in to Discord", async t => { + const [error] = await tryToCatch(() => router.test("post", "/api/unlink-space", { + body: { + guild_id: "665289423482519565" + } + })) + t.equal(error.data, "Can't edit a guild you don't have Manage Server permissions in") +}) + +test("web unlink space: checks that guild exists", async t => { + const [error] = await tryToCatch(() => router.test("post", "/api/unlink-space", { + sessionData: { + managedGuilds: ["2"] + }, + body: { + guild_id: "2" + } + })) + t.equal(error.data, "Discord guild does not exist or bot has not joined it") +}) + +test("web unlink space: checks that a space is linked to the guild before trying to unlink the space", async t => { + db.exec("BEGIN TRANSACTION") + db.prepare("DELETE FROM guild_active WHERE guild_id = '665289423482519565'").run() + + const [error] = await tryToCatch(() => router.test("post", "/api/unlink-space", { + sessionData: { + managedGuilds: ["665289423482519565"] + }, + body: { + guild_id: "665289423482519565" + } + })) + t.equal(error.data, "Discord guild has not been considered for bridging") + + db.exec("ROLLBACK") // ぬ +}) + +test("web unlink space: correctly abort unlinking if some linked channels remain after trying to unlink them all", async t => { + let unbridgedChannel = false + + const [error] = await tryToCatch(() => router.test("post", "/api/unlink-space", { + sessionData: { + managedGuilds: ["665289423482519565"] + }, + body: { + guild_id: "665289423482519565", + }, + createRoom: { + async unbridgeChannel(channel, guildID) { + unbridgedChannel = true + t.ok(["1438284564815548418", "665310973967597573"].includes(channel.id)) + t.equal(guildID, "665289423482519565") + // Do not actually delete the link from DB, should trigger error later in check + } + }, + api: { + async *generateFullHierarchy(spaceID) { + t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + yield { + room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + children_state: [], + guest_can_join: false, + num_joined_members: 2 + } + /* c8 ignore next */ + }, + } + })) + + t.equal(error.data, "Failed to unlink some rooms. Please try doing it manually, or report a bug. The space will not be unlinked until all rooms are.") + t.equal(unbridgedChannel, true) +}) + +test("web unlink space: successfully calls unbridgeChannel on linked channels in space, self-downgrade power level, leave space, and delete link from DB", async t => { + const {reg} = require("../../matrix/read-registration") + const me = `@${reg.sender_localpart}:${reg.ooye.server_name}` + + const getLinkRowQuery = "SELECT * FROM guild_space WHERE guild_id = '665289423482519565'" + + const row = db.prepare(getLinkRowQuery).get() + t.equal(row.space_id, "!zTMspHVUBhFLLSdmnS:cadence.moe") + + let unbridgedChannel = false + let downgradedPowerLevel = false + let leftRoom = false + await router.test("post", "/api/unlink-space", { + sessionData: { + managedGuilds: ["665289423482519565"] + }, + body: { + guild_id: "665289423482519565", + }, + createRoom: { + async unbridgeChannel(channel, guildID) { + unbridgedChannel = true + t.ok(["1438284564815548418", "665310973967597573"].includes(channel.id)) + t.equal(guildID, "665289423482519565") + + // In order to not simulate channel deletion and not trigger the post unlink channels, pre-unlink space check + db.prepare("DELETE FROM channel_room WHERE channel_id = ?").run(channel.id) + } + }, + snow: { + user: { + // @ts-ignore - snowtransfer or discord-api-types broken, 204 No Content should be mapped to void but is actually mapped to never + async leaveGuild(guildID) { + t.equal(guildID, "665289423482519565") + } + } + }, + api: { + async *generateFullHierarchy(spaceID) { + t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + yield { + room_id: "!NDbIqNpJyPvfKRnNcr:cadence.moe", + children_state: [], + guest_can_join: false, + num_joined_members: 2 + } + /* c8 ignore next */ + }, + + async getStateEvent(roomID, type, key) { // getting power levels from space to apply to room + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return {users: {"@_ooye_bot:cadence.moe": 100, "@example:matrix.org": 50}, events: {"m.room.tombstone": 100}} + }, + + async getStateEventOuter(roomID, type, key) { + t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + t.equal(type, "m.room.create") + t.equal(key, "") + return { + type: "m.room.create", + state_key: "", + sender: "@_ooye_bot:cadence.moe", + room_id: "!zTMspHVUBhFLLSdmnS:cadence.moe", + event_id: "$create", + origin_server_ts: 0, + content: { + room_version: "11" + } + } + }, + + async sendState(roomID, type, key, content) { + downgradedPowerLevel = true + t.equal(roomID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + t.equal(type, "m.room.power_levels") + t.notOk(me in content.users, `got ${JSON.stringify(content)} but expected bot user to not be present`) + return "" + }, + + async leaveRoom(spaceID) { + leftRoom = true + t.equal(spaceID, "!zTMspHVUBhFLLSdmnS:cadence.moe") + }, + } + }) + + t.equal(unbridgedChannel, true) + t.equal(downgradedPowerLevel, true) + t.equal(leftRoom, true) + + const missed_row = db.prepare(getLinkRowQuery).get() + t.equal(missed_row, undefined) }) diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 216581c2..3df59014 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -22,7 +22,7 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom ('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS', '112760669178241024'), ('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL, '66192955777486848'), ('1160894080998461480', '!TqlyQmifxGUggEmdBN:cadence.moe', 'ooyexperiment', NULL, NULL, NULL, '66192955777486848'), -('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL, '665289423482519565'), +('1161864271370666075', '!mHmhQQPwXNananMUqq:cadence.moe', 'updates', NULL, NULL, NULL, '112760669178241024'), ('1438284564815548418', '!MHxNpwtgVqWOrmyoTn:cadence.moe', 'sin-cave', NULL, NULL, NULL, '665289423482519565'), ('598707048112193536', '!JBxeGYnzQwLnaooOLD:cadence.moe', 'winners', NULL, NULL, NULL, '1345641201902288987'); From c971ca3e3d6c1dea88fdc48ce10e787cbdb67a86 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 13 Feb 2026 19:31:28 +1300 Subject: [PATCH 135/153] Use radios/checkboxes for poll voting modal --- src/discord/interactions/poll.js | 20 +++++++------------- src/discord/register-interactions.js | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/discord/interactions/poll.js b/src/discord/interactions/poll.js index 0a4689d5..6d7a0153 100644 --- a/src/discord/interactions/poll.js +++ b/src/discord/interactions/poll.js @@ -43,7 +43,8 @@ async function* _interact({data, message, member, user}, {api}) { default: alreadySelected.includes(option.matrix_option) })) const checkboxGroupExtras = maxSelections === 1 && options.length > 1 ? {} : { - type: 22, // DiscordTypes.ComponentType.CheckboxGroup + /** @type {DiscordTypes.ComponentType.CheckboxGroup} */ + type: DiscordTypes.ComponentType.CheckboxGroup, min_values: 0, max_values: maxSelections } @@ -58,20 +59,12 @@ async function* _interact({data, message, member, user}, {api}) { }, { type: DiscordTypes.ComponentType.Label, label: pollRow.question_text, - component: /* { - type: 21, // DiscordTypes.ComponentType.RadioGroup + component: { + type: DiscordTypes.ComponentType.RadioGroup, custom_id: "POLL_MODAL_SELECTION", options, required: false, ...checkboxGroupExtras - } */ - { - type: DiscordTypes.ComponentType.StringSelect, - custom_id: "POLL_MODAL_SELECTION", - options, - required: false, - min_values: 0, - max_values: maxSelections, } }] } @@ -80,14 +73,15 @@ async function* _interact({data, message, member, user}, {api}) { if (data.custom_id === "POLL_MODAL") { // Clicked options via modal - /** @type {DiscordTypes.APIMessageStringSelectInteractionData} */ // @ts-ignore - close enough to the real thing + /** @type {DiscordTypes.APIModalSubmitRadioGroupComponent | DiscordTypes.APIModalSubmitCheckboxGroupComponent} */ // @ts-ignore - close enough to the real thing const component = data.components[1].component assert.equal(component.custom_id, "POLL_MODAL_SELECTION") + const values = "values" in component ? component.values : [component.value] // Replace votes with selection db.transaction(() => { db.prepare("DELETE FROM poll_vote WHERE message_id = ? AND discord_or_matrix_user_id = ?").run(message.id, userID) - for (const option of component.values) { + for (const option of values) { db.prepare("INSERT OR IGNORE INTO poll_vote (discord_or_matrix_user_id, message_id, matrix_option) VALUES (?, ?, ?)").run(userID, message.id, option) } })() diff --git a/src/discord/register-interactions.js b/src/discord/register-interactions.js index 46a7360f..e3d58c48 100644 --- a/src/discord/register-interactions.js +++ b/src/discord/register-interactions.js @@ -113,7 +113,7 @@ async function dispatchInteraction(interaction) { } else if (interactionId === "Responses") { /** @type {DiscordTypes.APIMessageApplicationCommandGuildInteraction} */ // @ts-ignore const messageInteraction = interaction - if (messageInteraction.data.resolved.messages[messageInteraction.data.target_id]?.poll) { + if (select("poll", "message_id", {message_id: messageInteraction.data.target_id}).get()) { await pollResponses.interact(messageInteraction) } else { await reactions.interact(messageInteraction) From 676cab0dc8c2fc7d32b6fad31a5b4cb4bdb88779 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 13 Feb 2026 20:27:25 +1300 Subject: [PATCH 136/153] Use smalltext for interaction header --- src/d2m/converters/message-to-event.js | 36 +++++++-- .../message-to-event.test.embeds.js | 77 ++++++++----------- 2 files changed, 60 insertions(+), 53 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 6109b0f9..8f7d0eeb 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -247,6 +247,20 @@ async function pollToEvent(poll) { } } +/** + * @param {DiscordTypes.APIMessageInteraction} interaction + * @param {boolean} isThinkingInteraction + */ +function getFormattedInteraction(interaction, isThinkingInteraction) { + const mxid = select("sim", "mxid", {user_id: interaction.user.id}).pluck().get() + const username = interaction.member?.nick || interaction.user.global_name || interaction.user.username + const thinkingText = isThinkingInteraction ? " — interaction loading..." : "" + return { + body: `↪️ ${username} used \`/${interaction.name}\`${thinkingText}`, + html: `
    ↪️ ${mxid ? tag`${username}` : username} used /${interaction.name}${thinkingText}
    ` + } +} + /** * @param {DiscordTypes.APIMessage} message * @param {DiscordTypes.APIGuild} guild @@ -321,13 +335,8 @@ async function messageToEvent(message, guild, options = {}, di) { } const interaction = message.interaction_metadata || message.interaction - if (message.type === DiscordTypes.MessageType.ChatInputCommand && interaction && "name" in interaction) { - // Commands are sent by the responding bot. Need to attach the metadata of the person using the command at the top. - let content = message.content - if (content) content = `\n${content}` - else if ((message.flags || 0) & DiscordTypes.MessageFlags.Loading) content = " — interaction loading..." - message.content = `> ↪️ <@${interaction.user.id}> used \`/${interaction.name}\`${content}` - } + const isInteraction = message.type === DiscordTypes.MessageType.ChatInputCommand && !!interaction && "name" in interaction + const isThinkingInteraction = isInteraction && !!((message.flags || 0) & DiscordTypes.MessageFlags.Loading) /** @type {{room?: boolean, user_ids?: string[]}} @@ -621,6 +630,12 @@ async function messageToEvent(message, guild, options = {}, di) { } } + if (isInteraction && !isThinkingInteraction && events.length === 0) { + const formattedInteraction = getFormattedInteraction(interaction, false) + body = `${formattedInteraction.body}\n${body}` + html = `${formattedInteraction.html}${html}` + } + const newTextMessageEvent = { $type: "m.room.message", "m.mentions": mentions, @@ -712,8 +727,13 @@ async function messageToEvent(message, guild, options = {}, di) { events.push(...forwardedEvents) } + if (isThinkingInteraction) { + const formattedInteraction = getFormattedInteraction(interaction, true) + await addTextEvent(formattedInteraction.body, formattedInteraction.html, "m.notice") + } + // Then text content - if (message.content && !isOnlyKlipyGIF) { + if (message.content && !isOnlyKlipyGIF && !isThinkingInteraction) { // Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention. let content = message.content if (options.scanTextForMentions !== false) { diff --git a/src/d2m/converters/message-to-event.test.embeds.js b/src/d2m/converters/message-to-event.test.embeds.js index cfb2f960..259aa668 100644 --- a/src/d2m/converters/message-to-event.test.embeds.js +++ b/src/d2m/converters/message-to-event.test.embeds.js @@ -4,24 +4,31 @@ const data = require("../../../test/data") const {mockGetEffectivePower} = require("../../matrix/utils.test") const {db} = require("../../passthrough") +test("message2event embeds: interaction loading", async t => { + const events = await messageToEvent(data.interaction_message.thinking_interaction, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + body: "↪️ Brad used `/stats` — interaction loading...", + format: "org.matrix.custom.html", + formatted_body: "
    ↪️ Brad used /stats — interaction loading...
    ", + "m.mentions": {}, + msgtype: "m.notice", + }]) +}) + test("message2event embeds: nothing but a field", async t => { const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {}) t.deepEqual(events, [{ - $type: "m.room.message", - body: "> ↪️ @papiophidian: used `/stats`", - format: "org.matrix.custom.html", - formatted_body: "
    ↪️ @papiophidian used /stats
    ", - "m.mentions": {}, - msgtype: "m.text", - }, { $type: "m.room.message", "m.mentions": {}, msgtype: "m.notice", - body: "| ### Amanda 🎵#2192 :online:" + body: "↪️ PapiOphidian used `/stats`" + + "\n| ### Amanda 🎵#2192 :online:" + "\n| willow tree, branch 0" + "\n| **❯ Uptime:**\n| 3m 55s\n| **❯ Memory:**\n| 64.45MB", format: "org.matrix.custom.html", - formatted_body: '

    Amanda 🎵#2192 \":online:\"' + formatted_body: '

    ↪️ PapiOphidian used /stats
    ' + + '

    Amanda 🎵#2192 \":online:\"' + '
    willow tree, branch 0    ' + '
    ❯ Uptime:
    3m 55s' + '
    ❯ Memory:
    64.45MB

    ' @@ -144,18 +151,13 @@ test("message2event embeds: crazy html is all escaped", async t => { test("message2event embeds: title without url", async t => { const events = await messageToEvent(data.message_with_embeds.title_without_url, data.guild.general) t.deepEqual(events, [{ - $type: "m.room.message", - body: "> ↪️ @papiophidian: used `/stats`", - format: "org.matrix.custom.html", - formatted_body: "
    ↪️ @papiophidian used /stats
    ", - "m.mentions": {}, - msgtype: "m.text", - }, { $type: "m.room.message", msgtype: "m.notice", - body: "| ## Hi, I'm Amanda!\n| \n| I condone pirating music!", + body: "↪️ PapiOphidian used `/stats`" + + "\n| ## Hi, I'm Amanda!\n| \n| I condone pirating music!", format: "org.matrix.custom.html", - formatted_body: `

    Hi, I'm Amanda!

    I condone pirating music!

    `, + formatted_body: '
    ↪️ PapiOphidian used /stats
    ' + + `

    Hi, I'm Amanda!

    I condone pirating music!

    `, "m.mentions": {} }]) }) @@ -163,18 +165,13 @@ test("message2event embeds: title without url", async t => { test("message2event embeds: url without title", async t => { const events = await messageToEvent(data.message_with_embeds.url_without_title, data.guild.general) t.deepEqual(events, [{ - $type: "m.room.message", - body: "> ↪️ @papiophidian: used `/stats`", - format: "org.matrix.custom.html", - formatted_body: "
    ↪️ @papiophidian used /stats
    ", - "m.mentions": {}, - msgtype: "m.text", - }, { $type: "m.room.message", msgtype: "m.notice", - body: "| I condone pirating music!", + body: "↪️ PapiOphidian used `/stats`" + + "\n| I condone pirating music!", format: "org.matrix.custom.html", - formatted_body: `

    I condone pirating music!

    `, + formatted_body: '
    ↪️ PapiOphidian used /stats
    ' + + `

    I condone pirating music!

    `, "m.mentions": {} }]) }) @@ -182,18 +179,13 @@ test("message2event embeds: url without title", async t => { test("message2event embeds: author without url", async t => { const events = await messageToEvent(data.message_with_embeds.author_without_url, data.guild.general) t.deepEqual(events, [{ - $type: "m.room.message", - body: "> ↪️ @papiophidian: used `/stats`", - format: "org.matrix.custom.html", - formatted_body: "
    ↪️ @papiophidian used /stats
    ", - "m.mentions": {}, - msgtype: "m.text", - }, { $type: "m.room.message", msgtype: "m.notice", - body: "| ## Amanda\n| \n| I condone pirating music!", + body: "↪️ PapiOphidian used `/stats`" + + "\n| ## Amanda\n| \n| I condone pirating music!", format: "org.matrix.custom.html", - formatted_body: `

    Amanda

    I condone pirating music!

    `, + formatted_body: '
    ↪️ PapiOphidian used /stats
    ' + + `

    Amanda

    I condone pirating music!

    `, "m.mentions": {} }]) }) @@ -201,18 +193,13 @@ test("message2event embeds: author without url", async t => { test("message2event embeds: author url without name", async t => { const events = await messageToEvent(data.message_with_embeds.author_url_without_name, data.guild.general) t.deepEqual(events, [{ - $type: "m.room.message", - body: "> ↪️ @papiophidian: used `/stats`", - format: "org.matrix.custom.html", - formatted_body: "
    ↪️ @papiophidian used /stats
    ", - "m.mentions": {}, - msgtype: "m.text", - }, { $type: "m.room.message", msgtype: "m.notice", - body: "| I condone pirating music!", + body: "↪️ PapiOphidian used `/stats`" + + "\n| I condone pirating music!", format: "org.matrix.custom.html", - formatted_body: `

    I condone pirating music!

    `, + formatted_body: '
    ↪️ PapiOphidian used /stats
    ' + + `

    I condone pirating music!

    `, "m.mentions": {} }]) }) From 5002f3046a0b23f77405080f65f8e22c55269565 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 13 Feb 2026 20:27:38 +1300 Subject: [PATCH 137/153] Convert emojihax to real emoji --- src/d2m/converters/message-to-event.js | 5 ++++ src/d2m/converters/message-to-event.test.js | 12 ++++++++ test/data.js | 31 +++++++++++++++++++++ test/ooye-test-data.sql | 3 +- 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 8f7d0eeb..7f77b81e 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -757,6 +757,11 @@ async function messageToEvent(message, guild, options = {}, di) { } } + // Scan the content for emojihax and replace them with real emojis + content = content.replaceAll(/\[([a-zA-Z0-9_-]{2,32})(?:~[0-9]+)?\]\(https:\/\/cdn\.discordapp\.com\/emojis\/([0-9]+)\.[^ \n)`]+\)/g, (_, name, id) => { + return `<:${name}:${id}>` + }) + const {body, html} = await transformContent(content) await addTextEvent(body, html, msgtype) } diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index 84cc1e04..1a73aea7 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -1171,6 +1171,18 @@ test("message2event: emoji that hasn't been registered yet", async t => { }]) }) +test("message2event: emojihax", async t => { + const events = await messageToEvent(data.message.emojihax, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "I only violate the don't modify our console part of terms of service :troll:", + format: "org.matrix.custom.html", + formatted_body: `I only violate the don't modify our console part of terms of service :troll:` + }]) +}) + test("message2event: emoji triple long name", async t => { const events = await messageToEvent(data.message.emoji_triple_long_name, data.guild.general, {}) t.deepEqual(events, [{ diff --git a/test/data.js b/test/data.js index eef3a50d..6a53cb01 100644 --- a/test/data.js +++ b/test/data.js @@ -3219,6 +3219,37 @@ module.exports = { flags: 0, components: [] }, + emojihax: { + id: "1126733830494093453", + type: 0, + content: "I only violate the don't modify our console part of terms of service [troll~1](https://cdn.discordapp.com/emojis/1254940125948022915.webp?size=48&name=troll%7E1&lossless=true)", + channel_id: "112760669178241024", + author: { + id: "111604486476181504", + username: "kyuugryphon", + avatar: "e4ce31267ca524d19be80e684d4cafa1", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "KyuuGryphon", + avatar_decoration: null, + display_name: "KyuuGryphon", + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-07-07T04:37:58.892000+00:00", + edited_timestamp: null, + flags: 0, + components: [] + }, emoji_triple_long_name: { id: "1156394116540805170", type: 0, diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 3df59014..1dd9dfed 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -152,7 +152,8 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg', 'mxc://cadence.moe/MRRPDggXQMYkrUjTpxQbmcxB'), ('https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml', 'mxc://cadence.moe/HnQIYQmmlIKwOQsbFsIGpzPP'), ('https://cdn.discordapp.com/attachments/112760669178241024/1296237494987133070/100km.gif', 'mxc://cadence.moe/qDAotmebTfEIfsAIVCEZptLh'), -('https://cdn.discordapp.com/attachments/123/456/my_enemies.txt', 'mxc://cadence.moe/y89EOTRp2lbeOkgdsEleGOge'); +('https://cdn.discordapp.com/attachments/123/456/my_enemies.txt', 'mxc://cadence.moe/y89EOTRp2lbeOkgdsEleGOge'), +('https://cdn.discordapp.com/emojis/1254940125948022915.webp', 'mxc://cadence.moe/bvVJFgOIyNcAknKCbmaHDktG'); INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), From 08323f45129bef87bbc60805d0e4f64ff8c7d4dc Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 13 Feb 2026 21:59:17 +1300 Subject: [PATCH 138/153] More consistency for invite records table - Autojoined child spaces are recorded as invited - Update entry when reinvited - Delete entry when uninvited or removed from room - Allow linking with spaces you moderate, even if you didn't invite - Store power levels immediately for new invited rooms - Mark members as missing profile in this case - Only delete from invite table if it left the space --- ...33-add-missing-profile-to-member-cache.sql | 5 ++ src/db/orm-defs.d.ts | 3 +- src/m2d/converters/event-to-message.js | 6 +-- src/m2d/event-dispatcher.js | 50 +++++++++++++------ src/web/routes/guild.js | 8 ++- src/web/routes/link.js | 2 +- 6 files changed, 54 insertions(+), 20 deletions(-) create mode 100644 src/db/migrations/0033-add-missing-profile-to-member-cache.sql diff --git a/src/db/migrations/0033-add-missing-profile-to-member-cache.sql b/src/db/migrations/0033-add-missing-profile-to-member-cache.sql new file mode 100644 index 00000000..ef937e0e --- /dev/null +++ b/src/db/migrations/0033-add-missing-profile-to-member-cache.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +ALTER TABLE member_cache ADD COLUMN missing_profile INTEGER; + +COMMIT; diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index e36ed491..79f02ade 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -90,6 +90,7 @@ export type Models = { displayname: string | null avatar_url: string | null, power_level: number + missing_profile: number | null } member_power: { @@ -146,7 +147,7 @@ export type Models = { question_text: string is_closed: number } - + poll_option: { message_id: string matrix_option: string diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 9460ebb2..ed8d2c3c 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -290,8 +290,8 @@ function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink * @returns {Promise<{displayname?: string?, avatar_url?: string?}>} */ async function getMemberFromCacheOrHomeserver(roomID, mxid, api) { - const row = select("member_cache", ["displayname", "avatar_url"], {room_id: roomID, mxid}).get() - if (row) return row + const row = select("member_cache", ["displayname", "avatar_url", "missing_profile"], {room_id: roomID, mxid}).get() + if (row && !row.missing_profile) return row return api.getStateEvent(roomID, "m.room.member", mxid).then(event => { const room = select("channel_room", "room_id", {room_id: roomID}).get() if (room) { @@ -299,7 +299,7 @@ async function getMemberFromCacheOrHomeserver(roomID, mxid, api) { // the cache will be kept in sync by the `m.room.member` event listener const displayname = event?.displayname || null const avatar_url = event?.avatar_url || null - db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?) ON CONFLICT DO UPDATE SET displayname = ?, avatar_url = ?").run( + db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?) ON CONFLICT DO UPDATE SET displayname = ?, avatar_url = ?, missing_profile = NULL").run( roomID, mxid, displayname, avatar_url, displayname, avatar_url diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 57c0fa64..70e293b1 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -371,7 +371,18 @@ sync.addTemporaryListener(as, "type:m.space.child", guard("m.space.child", */ async event => { if (Array.isArray(event.content.via) && event.content.via.length) { // space child is being added - await api.joinRoom(event.state_key).catch(() => {}) // try to join if able, it's okay if it doesn't want, bot will still respond to invites + try { + // try to join if able, it's okay if it doesn't want, bot will still respond to invites + await api.joinRoom(event.state_key) + // if autojoined a child space, store it in invite (otherwise the child space will be impossible to use with self-service in the future) + const hierarchy = await api.getHierarchy(event.state_key, {limit: 1}) + const roomProperties = hierarchy.rooms?.[0] + if (roomProperties?.room_id === event.state_key && roomProperties.room_type === "m.space" && roomProperties.name) { + db.prepare("INSERT OR IGNORE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)") + .run(event.sender, event.state_key, roomProperties.room_type, roomProperties.name, roomProperties.topic, roomProperties.avatar_url) + await updateMemberCachePowerLevels(event.state_key) // store privileged users in member_cache so they are also allowed to perform self-service + } + } catch (e) {} } })) @@ -404,22 +415,24 @@ async event => { } if (!inviteRoomState?.name) return await api.leaveRoomWithReason(event.room_id, `Please only invite me to rooms that have a name/avatar set. Update the room details and reinvite.`) await api.joinRoom(event.room_id) - db.prepare("INSERT OR IGNORE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)").run(event.sender, event.room_id, inviteRoomState.type, inviteRoomState.name, inviteRoomState.topic, inviteRoomState.avatar) + db.prepare("REPLACE INTO invite (mxid, room_id, type, name, topic, avatar) VALUES (?, ?, ?, ?, ?, ?)").run(event.sender, event.room_id, inviteRoomState.type, inviteRoomState.name, inviteRoomState.topic, inviteRoomState.avatar) if (inviteRoomState.avatar) utils.getPublicUrlForMxc(inviteRoomState.avatar) // make sure it's available in the media_proxy allowed URLs + await updateMemberCachePowerLevels(event.room_id) // store privileged users in member_cache so they are also allowed to perform self-service } - if (utils.eventSenderIsFromDiscord(event.state_key)) return - if (event.content.membership === "leave" || event.content.membership === "ban") { // Member is gone db.prepare("DELETE FROM member_cache WHERE room_id = ? and mxid = ?").run(event.room_id, event.state_key) - // Unregister room's use as a direct chat if the bot itself left + // Unregister room's use as a direct chat and/or an invite target if the bot itself left if (event.state_key === utils.bot) { db.prepare("DELETE FROM direct WHERE room_id = ?").run(event.room_id) + db.prepare("DELETE FROM invite WHERE room_id = ?").run(event.room_id) } } + if (utils.eventSenderIsFromDiscord(event.state_key)) return + const exists = select("channel_room", "room_id", {room_id: event.room_id}) ?? select("guild_space", "space_id", {space_id: event.room_id}) if (!exists) return // don't cache members in unbridged rooms @@ -428,7 +441,7 @@ async event => { if (memberPower === Infinity) memberPower = tombstone // database storage compatibility const displayname = event.content.displayname || null const avatar_url = event.content.avatar_url - db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url, power_level) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET displayname = ?, avatar_url = ?, power_level = ?").run( + db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url, power_level) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET displayname = ?, avatar_url = ?, power_level = ?, missing_profile = NULL").run( event.room_id, event.state_key, displayname, avatar_url, memberPower, displayname, avatar_url, memberPower @@ -441,16 +454,25 @@ sync.addTemporaryListener(as, "type:m.room.power_levels", guard("m.room.power_le */ async event => { if (event.state_key !== "") return - const existingPower = select("member_cache", "mxid", {room_id: event.room_id}).pluck().all() - const {allCreators} = await utils.getEffectivePower(event.room_id, [], api) - const newPower = event.content.users || {} - for (const mxid of existingPower) { - if (!allCreators.includes(mxid)) { - db.prepare("UPDATE member_cache SET power_level = ? WHERE room_id = ? AND mxid = ?").run(newPower[mxid] || 0, event.room_id, mxid) - } - } + await updateMemberCachePowerLevels(event.room_id) })) +/** + * @param {string} roomID + */ +async function updateMemberCachePowerLevels(roomID) { + const existingPower = select("member_cache", "mxid", {room_id: roomID}).pluck().all() + const {powerLevels, allCreators, tombstone} = await utils.getEffectivePower(roomID, [], api) + const newPower = powerLevels.users || {} + const newPowerUsers = Object.keys(newPower) + const relevantUsers = existingPower.concat(newPowerUsers).concat(allCreators) + for (const mxid of [...new Set(relevantUsers)]) { + const level = allCreators.includes(mxid) ? tombstone : newPower[mxid] ?? powerLevels.users_default ?? 0 + db.prepare("INSERT INTO member_cache (room_id, mxid, power_level, missing_profile) VALUES (?, ?, ?, 1) ON CONFLICT DO UPDATE SET power_level = ?") + .run(roomID, mxid, level, level) + } +} + sync.addTemporaryListener(as, "type:m.room.tombstone", guard("m.room.tombstone", /** * @param {Ty.Event.StateOuter} event diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index 4f140a3d..5f9e2d93 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -148,7 +148,13 @@ as.router.get("/guild", defineEventHandler(async event => { // Self-service guild that hasn't been linked yet - needs a special page encouraging the link flow if (!row.space_id && row.autocreate === 0) { - const spaces = db.prepare("SELECT room_id, type, name, topic, avatar FROM invite LEFT JOIN guild_space ON invite.room_id = guild_space.space_id WHERE mxid = ? AND space_id IS NULL AND type = 'm.space'").all(session.data.mxid) + let spaces = + // invited spaces + db.prepare("SELECT room_id, type, name, topic, avatar FROM invite LEFT JOIN guild_space ON invite.room_id = guild_space.space_id WHERE mxid = ? AND space_id IS NULL AND type = 'm.space'").all(session.data.mxid) + // moderated spaces + .concat(db.prepare("SELECT room_id, type, name, topic, avatar FROM invite LEFT JOIN guild_space ON invite.room_id = guild_space.space_id INNER JOIN member_cache USING (room_id) WHERE member_cache.mxid = ? AND power_level >= 50 AND space_id IS NULL AND type = 'm.space'").all(session.data.mxid)) + const seen = new Set(spaces.map(s => s.room_id)) + spaces = spaces.filter(s => seen.delete(s.room_id)) return pugSync.render(event, "guild_not_linked.pug", {guild, guild_id, spaces}) } diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 3ae6de5f..248b4efd 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -282,11 +282,11 @@ as.router.post("/api/unlink-space", defineEventHandler(async event => { await utils.setUserPower(spaceID, utils.bot, 0, api) await api.leaveRoom(spaceID) db.prepare("DELETE FROM guild_space WHERE guild_id = ? AND space_id = ?").run(guild_id, spaceID) + db.prepare("DELETE FROM invite WHERE room_id = ?").run(spaceID) } // Mark as not considered for bridging db.prepare("DELETE FROM guild_active WHERE guild_id = ?").run(guild_id) - db.prepare("DELETE FROM invite WHERE room_id = ?").run(spaceID) await snow.user.leaveGuild(guild_id) setResponseHeader(event, "HX-Redirect", "/") From b5143bfe1f7e53e3c381c02fbd4a48f63afd967f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Feb 2026 00:33:02 +1300 Subject: [PATCH 139/153] Use same invite logic for display and for linking --- src/m2d/converters/event-to-message.js | 1 - src/web/routes/guild.js | 23 ++++++++++++++++------- src/web/routes/link.js | 8 ++++++-- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index ed8d2c3c..2add279f 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -844,7 +844,6 @@ async function eventToMessage(event, guild, channel, di) { // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. // But I should not count it if it's between block elements. input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => { - // console.error(beforeContext, beforeTag, afterContext, afterTag) if (typeof beforeTag !== "string" && typeof afterTag !== "string") { return "
    " } diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index 5f9e2d93..dfb393b7 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -133,6 +133,20 @@ function getChannelRoomsLinks(guild, rooms, roles) { } } +/** + * @param {string} mxid + */ +function getInviteTargetSpaces(mxid) { + /** @type {{room_id: string, mxid: string, type: string, name: string, topic: string?, avatar: string?}[]} */ + const spaces = + // invited spaces + db.prepare("SELECT room_id, invite.mxid, type, name, topic, avatar FROM invite LEFT JOIN guild_space ON invite.room_id = guild_space.space_id WHERE mxid = ? AND space_id IS NULL AND type = 'm.space'").all(mxid) + // moderated spaces + .concat(db.prepare("SELECT room_id, invite.mxid, type, name, topic, avatar FROM invite LEFT JOIN guild_space ON invite.room_id = guild_space.space_id INNER JOIN member_cache USING (room_id) WHERE member_cache.mxid = ? AND power_level >= 50 AND space_id IS NULL AND type = 'm.space'").all(mxid)) + const seen = new Set(spaces.map(s => s.room_id)) + return spaces.filter(s => seen.delete(s.room_id)) +} + as.router.get("/guild", defineEventHandler(async event => { const {guild_id} = await getValidatedQuery(event, schema.guild.parse) const session = await auth.useSession(event) @@ -148,13 +162,7 @@ as.router.get("/guild", defineEventHandler(async event => { // Self-service guild that hasn't been linked yet - needs a special page encouraging the link flow if (!row.space_id && row.autocreate === 0) { - let spaces = - // invited spaces - db.prepare("SELECT room_id, type, name, topic, avatar FROM invite LEFT JOIN guild_space ON invite.room_id = guild_space.space_id WHERE mxid = ? AND space_id IS NULL AND type = 'm.space'").all(session.data.mxid) - // moderated spaces - .concat(db.prepare("SELECT room_id, type, name, topic, avatar FROM invite LEFT JOIN guild_space ON invite.room_id = guild_space.space_id INNER JOIN member_cache USING (room_id) WHERE member_cache.mxid = ? AND power_level >= 50 AND space_id IS NULL AND type = 'm.space'").all(session.data.mxid)) - const seen = new Set(spaces.map(s => s.room_id)) - spaces = spaces.filter(s => seen.delete(s.room_id)) + const spaces = session.data.mxid ? getInviteTargetSpaces(session.data.mxid) : [] return pugSync.render(event, "guild_not_linked.pug", {guild, guild_id, spaces}) } @@ -257,3 +265,4 @@ as.router.post("/api/invite", defineEventHandler(async event => { })) module.exports._getPosition = getPosition +module.exports.getInviteTargetSpaces = getInviteTargetSpaces diff --git a/src/web/routes/link.js b/src/web/routes/link.js index 248b4efd..43995fcd 100644 --- a/src/web/routes/link.js +++ b/src/web/routes/link.js @@ -11,6 +11,8 @@ const {discord, db, as, sync, select, from} = require("../../passthrough") const auth = sync.require("../auth") /** @type {import("../../matrix/utils")}*/ const utils = sync.require("../../matrix/utils") +/** @type {import("./guild")}*/ +const guildRoute = sync.require("./guild") /** * @param {H3Event} event @@ -107,13 +109,15 @@ as.router.post("/api/link-space", defineEventHandler(async event => { // Check space ID if (!session.data.mxid) throw createError({status: 403, message: "Forbidden", data: "Can't link with your Matrix space if you aren't logged in to Matrix"}) const spaceID = parsedBody.space_id - const inviteRow = select("invite", ["mxid", "type"], {mxid: session.data.mxid, room_id: spaceID}).get() - if (!inviteRow || inviteRow.type !== "m.space") throw createError({status: 403, message: "Forbidden", data: "You personally must invite OOYE to that space on Matrix"}) // Check they are not already bridged const existing = select("guild_space", "guild_id", {}, "WHERE guild_id = ? OR space_id = ?").get(guildID, spaceID) if (existing) throw createError({status: 400, message: "Bad Request", data: `Guild ID ${guildID} or space ID ${spaceID} are already bridged and cannot be reused`}) + // Check space ID is a valid invite target + const inviteRow = guildRoute.getInviteTargetSpaces(session.data.mxid).find(s => s.room_id === spaceID) + if (!inviteRow) throw createError({status: 403, message: "Forbidden", data: "You personally must invite OOYE to that space on Matrix"}) + const inviteServer = inviteRow.mxid.match(/:(.*)/)?.[1] assert(inviteServer) const via = [inviteServer] From 14de436054e6c053e65c5fa9021541928b0e3355 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Feb 2026 20:13:46 +1300 Subject: [PATCH 140/153] Add docker policy --- docs/docker.md | 76 +++++++++++++++++++++++++++++++++++++++++++++ docs/get-started.md | 2 ++ 2 files changed, 78 insertions(+) create mode 100644 docs/docker.md diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 00000000..7e3eb7d2 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,76 @@ +# Docker policy + +**Out Of Your Element has no official support for Docker. There are no official files or images. If you choose to run Out Of Your Element in Docker, you must disclose this when asking for support. I may refuse to provide support/advice at any time. I may refuse to acknowledge issue reports.** + +This also goes for Podman, Nix, and other similar technology that upends a program's understanding of what it's running on. + +## What I recommend + +I recommend [following the official setup guide,](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md) which does not use Docker. + +Ultimately, though, do what makes you happy. I can't stop you from doing what you want. As long as you read this page and understand my perspective, that's good enough for me. + +## Why I advise against Docker + +When misconfigured, Docker has terrible impacts. It can cause messages to go missing or even permanent data loss. These have happened to people. + +Docker also makes it much harder for me to advise on debugging because it puts barriers between you and useful debugging tools, such as stdin, the database file, a shell, and the inspector. It's also not clear which version of the source code is running in the container, as there are many pieces of Docker (builder, container, image) that can cache old data, often making it so you didn't actually update when you thought you did. This has happened to people. + +## Why I don't provide a good configuration myself + +It is not possible for Docker to be correctly configured by default. The defaults are broken and will cause data loss. + +It is also not possible for me to provide a correct configuration for everyone. Even if I provided a correct image, the YAMLs and command-line arguments must be written by individual end users. Incorrect YAMLs and command-line arguments may cause connection issues or permanent data loss. + +## Why I don't provide assistance if you run OOYE in Docker + +Problems you encounter, especially with the initial setup, are much more likely to be caused by nuances in your Docker setup than problems in my code. Therefore, my code is not responsible for the problem. The cause of the problem is different code that I can't advise on. + +Also, if you reported an issue and I asked for additional information to help find the cause, you might be unable to provide it because of the debugging barriers discussed above. + +## Why I don't provide Docker resources + +I create OOYE unpaid in my spare time because I enjoy the process. I find great enjoyment in creating code and none at all in creating infrastructure. + +## Why you're probably fine without Docker + +### If you care about system footprint + +OOYE was designed to be simple and courteous: + +* It only creates files in its working directory +* It does not require any other processes to be running (e.g., no dependency on a Postgres process) +* It only requires node/npm executables in PATH, which you can store in any folder if you don't want to use your package manager + +### If you care about ease of setup + +In my opinion, the [official setup process](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md) is straightforward. After installing prerequisites (Node.js and the repo clone), the rest of the process interactively guides you through providing necessary information. Your input is checked for correctness so the bridge will definitely work when you run it. + +I find this easier than the usual Docker workflow of pasting values into a YAML and rolling the dice on whether it will start up or not. + +### If you care about security in the case of compromise/RCE + +There are no known vulnerabilities in dependencies. I [carefully selected simple, light dependencies](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/developer-orientation.md#dependency-justification) to reduce attack surface area. + +For defense in depth, I suggest running OOYE as a different user. + +### If you want to see all the processes when you run docker ps + +Well, you got me there. + +## Unofficial, independent, community-provided container setups + +I acknowledge the demand for using OOYE in a container, so I will still point you in the right direction. + +I had no hand in creating these and have not used or tested them whatsoever. I make no assurance that these will work reliably, or even at all. If you use these, you must do so with the understanding that if you run into any problems, **you must ask for support from the author of that setup, not from me, because you're running their code, not mine.** + +***The following list is distributed for your information, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.*** + +- by melody: https://git.shork.ch/docker-images/out-of-your-element +- by sim2kid: https://github.com/sim2kid/ooye-docker +- by Katharos Technology: https://github.com/katharostech/docker_ooye +- by Emma: https://cgit.rory.gay/nix/OOYE-module.git/tree + +## Making your own Docker setup + +If you decide to make your own, I may provide advice or indicate problems at my discretion. You acknowledge that I am not required to provide evidence of problems I indicate, nor solutions to them. You acknowledge that it is not possible for me to exhaustively indicate every problem, so I cannot indicate correctness. Even if I have provided advice to an unofficial, independent, community-provided setup, I do not endorse it. diff --git a/docs/get-started.md b/docs/get-started.md index a819b478..5b14b2a7 100644 --- a/docs/get-started.md +++ b/docs/get-started.md @@ -1,5 +1,7 @@ # Setup +If you want Docker, [please read this first.](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/docker.md) + If you get stuck, you're welcome to message [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) or [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) to ask for help setting up OOYE! You'll need: From c55e6c611585f4c1dbfd8c767e5f872fbeb0c66a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Feb 2026 20:20:19 +1300 Subject: [PATCH 141/153] v3.4 --- package-lock.json | 294 ++++++++-------------------------------------- package.json | 6 +- 2 files changed, 49 insertions(+), 251 deletions(-) diff --git a/package-lock.json b/package-lock.json index dd0cbbff..5d183b7c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "out-of-your-element", - "version": "3.2.0", + "version": "3.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "out-of-your-element", - "version": "3.2.0", + "version": "3.4.0", "license": "AGPL-3.0-or-later", "dependencies": { "@chriscdn/promise-semaphore": "^3.0.1", @@ -24,7 +24,7 @@ "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.15.2", - "discord-api-types": "^0.38.36", + "discord-api-types": "^0.38.38", "domino": "^2.1.6", "enquirer": "^2.4.1", "entities": "^5.0.0", @@ -50,7 +50,7 @@ "supertape": "^12.0.12" }, "engines": { - "node": ">=20" + "node": ">=22" } }, "../extended-errors/enhance-errors": { @@ -764,101 +764,14 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", + "integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==", "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, + "license": "BlueOak-1.0.0", "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + "node": ">=18" } }, "node_modules/@istanbuljs/schema": { @@ -1171,19 +1084,6 @@ "node": ">=6" } }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -1616,9 +1516,9 @@ } }, "node_modules/discord-api-types": { - "version": "0.38.37", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.37.tgz", - "integrity": "sha512-Cv47jzY1jkGkh5sv0bfHYqGgKOWO1peOrGMkDFM4UmaGMOTgOW8QSexhvixa9sVOiz8MnVOBryWYyw/CEVhj7w==", + "version": "0.38.38", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.38.tgz", + "integrity": "sha512-7qcM5IeZrfb+LXW07HvoI5L+j4PQeMZXEkSm1htHAHh4Y9JSMXBWjy/r7zmUCOj4F7zNjMcm7IMWr131MT2h0Q==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" @@ -1634,13 +1534,6 @@ "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz", "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true, - "license": "MIT" - }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -1860,14 +1753,40 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/glob/node_modules/balanced-match": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.2.tgz", + "integrity": "sha512-x0K50QvKQ97fdEz2kPehIerj+YTeptKF9hyYkKf6egnwmMWAkADiO0QCzSp0R5xN8FTZgYaBfSaue46Ej62nMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "jackspeak": "^4.2.3" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz", + "integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, "node_modules/glob/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.0.tgz", + "integrity": "sha512-ugkC31VaVg9cF0DFVoADH12k6061zNZkZON+aX8AWsR9GhPcErkcMBceb6znR8wLERM2AkkOxy2nWRLpT9Jq5w==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" + "brace-expansion": "^5.0.2" }, "engines": { "node": "20 || >=22" @@ -2054,13 +1973,13 @@ } }, "node_modules/jackspeak": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", - "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", + "integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "@isaacs/cliui": "^8.0.2" + "@isaacs/cliui": "^9.0.0" }, "engines": { "node": "20 || >=22" @@ -2813,45 +2732,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/string-width/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2873,46 +2753,6 @@ "node": ">=8" } }, - "node_modules/strip-ansi": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", - "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", @@ -3257,48 +3097,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/wrap-ansi/node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", diff --git a/package.json b/package.json index cce8204c..2ace5bf4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "out-of-your-element", - "version": "3.2.0", + "version": "3.4.0", "description": "A bridge between Matrix and Discord", "main": "index.js", "repository": { @@ -12,7 +12,7 @@ "discord", "bridge" ], - "author": "Cadence, PapiOphidian", + "author": "Cadence", "license": "AGPL-3.0-or-later", "engines": { "node": ">=22" @@ -33,7 +33,7 @@ "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.15.2", - "discord-api-types": "^0.38.36", + "discord-api-types": "^0.38.38", "domino": "^2.1.6", "enquirer": "^2.4.1", "entities": "^5.0.0", From 09ea94230769c078a54f556655db9486ff6b7233 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Feb 2026 22:47:38 +1300 Subject: [PATCH 142/153] Remove deprecated db management --- scripts/setup.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/scripts/setup.js b/scripts/setup.js index 696eec94..ecc57fd3 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -17,22 +17,6 @@ const {SnowTransfer} = require("snowtransfer") const DiscordTypes = require("discord-api-types/v10") const {createApp, defineEventHandler, toNodeListener} = require("h3") -// Move database file if it's still in the old location -if (fs.existsSync("db")) { - if (fs.existsSync("db/ooye.db")) { - fs.renameSync("db/ooye.db", "ooye.db") - } - const files = fs.readdirSync("db") - if (files.length) { - console.error("The db folder is deprecated and must be removed. Your ooye.db database file has already been moved to the root of the repo. You must manually move or delete the remaining files:") - for (const file of files) { - console.error(file) - } - process.exit(1) - } - fs.rmSync("db", {recursive: true}) -} - const passthrough = require("../src/passthrough") const db = new sqlite("ooye.db") const migrate = require("../src/db/migrate") From e779b4107285f8c6af8a9ac1918a264db74a3137 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 15 Feb 2026 12:34:08 +1300 Subject: [PATCH 143/153] Fix possible undefined property access --- src/d2m/converters/edit-to-changes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/d2m/converters/edit-to-changes.js b/src/d2m/converters/edit-to-changes.js index cfbd1e2d..4f743eb8 100644 --- a/src/d2m/converters/edit-to-changes.js +++ b/src/d2m/converters/edit-to-changes.js @@ -153,7 +153,7 @@ async function editToChanges(message, guild, api) { const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1 if (messageReallyOld) { eventsToSend = [] // Only allow edits to change and delete, but not send new. - } else if ((messageQuiteOld || !embedsEnabled) && !message.author.bot) { + } else if ((messageQuiteOld || !embedsEnabled) && !message.author?.bot) { eventsToSend = eventsToSend.filter(e => e.msgtype !== "m.notice") // Only send events that aren't embeds. } From 0cd7e1c336ae12ea39431926913c2d5f11728839 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 17 Feb 2026 12:54:50 +1300 Subject: [PATCH 144/153] Allow for custom additions to webroot --- .gitignore | 1 + docs/developer-orientation.md | 4 +- package-lock.json | 2 + package.json | 1 + src/web/pug-sync.js | 9 +++ src/web/pug/home.pug | 22 +++---- src/web/pug/includes/template.pug | 30 ++++++++-- src/web/server.js | 98 ++++++++++++++++++++++++------- 8 files changed, 127 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 17986439..c38dd88f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ registration.yaml ooye.db* events.db* backfill.db* +custom-webroot # Automatically generated node_modules diff --git a/docs/developer-orientation.md b/docs/developer-orientation.md index 056fe7e5..dbb19f39 100644 --- a/docs/developer-orientation.md +++ b/docs/developer-orientation.md @@ -89,7 +89,7 @@ Whether you read those or not, I'm more than happy to help you 1-on-1 with codin # Dependency justification -Total transitive production dependencies: 137 +Total transitive production dependencies: 134 ### 🦕 @@ -119,8 +119,8 @@ Total transitive production dependencies: 137 * (0) entities: Looks fine. No dependencies. * (0) get-relative-path: Looks fine. No dependencies. * (1) heatsync: Module hot-reloader that I trust. -* (1) js-yaml: Will be removed in the future after registration.yaml is converted to JSON. * (0) lru-cache: For holding unused nonce in memory and letting them be overwritten later if never used. +* (0) mime-type: File extension to mime type mapping that's already pulled in by stream-mime-type. * (0) prettier-bytes: It does what I want and has no dependencies. * (0) snowtransfer: Discord API library with bring-your-own-caching that I trust. * (0) try-to-catch: Not strictly necessary, but it's already pulled in by supertape, so I may as well. diff --git a/package-lock.json b/package-lock.json index 5d183b7c..98474007 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,7 @@ "heatsync": "^2.7.2", "htmx.org": "^2.0.4", "lru-cache": "^11.0.2", + "mime-types": "^2.1.35", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", "snowtransfer": "^0.17.1", @@ -2073,6 +2074,7 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", "dependencies": { "mime-db": "1.52.0" }, diff --git a/package.json b/package.json index 2ace5bf4..afbb90ab 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "heatsync": "^2.7.2", "htmx.org": "^2.0.4", "lru-cache": "^11.0.2", + "mime-types": "^2.1.35", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", "snowtransfer": "^0.17.1", diff --git a/src/web/pug-sync.js b/src/web/pug-sync.js index f49f5a28..f87550df 100644 --- a/src/web/pug-sync.js +++ b/src/web/pug-sync.js @@ -31,7 +31,15 @@ function addGlobals(obj) { */ function render(event, filename, locals) { const path = join(__dirname, "pug", filename) + return renderPath(event, path, locals) +} +/** + * @param {import("h3").H3Event} event + * @param {string} path + * @param {Record} locals + */ +function renderPath(event, path, locals) { function compile() { try { const template = compileFile(path, {pretty}) @@ -89,4 +97,5 @@ function createRoute(router, url, filename) { module.exports.addGlobals = addGlobals module.exports.render = render +module.exports.renderPath = renderPath module.exports.createRoute = createRoute diff --git a/src/web/pug/home.pug b/src/web/pug/home.pug index d5622502..8b865331 100644 --- a/src/web/pug/home.pug +++ b/src/web/pug/home.pug @@ -41,16 +41,18 @@ block body = ` Set up self-service` .s-prose - h2 What is this? - p #[a(href="https://gitdab.com/cadence/out-of-your-element") Out Of Your Element] is a bridge between the Discord and Matrix chat apps. It lets people on both platforms chat with each other without needing to get everyone on the same app. - p Just chat like usual, and the bridge will forward messages back and forth between the two platforms, so everyone sees the whole conversation. - p All kinds of content are supported, including pictures, threads, emojis, and @mentions. - p It's really easy to set up, even if you only have Discord. Just add the bot to your server, and it'll make everything available on Matrix automatically. + block bridge-info + h2 What is this? + p #[a(href="https://gitdab.com/cadence/out-of-your-element") Out Of Your Element] is a bridge between the Discord and Matrix chat apps. It lets people on both platforms chat with each other without needing to get everyone on the same app. + p Just chat like usual, and the bridge will forward messages back and forth between the two platforms, so everyone sees the whole conversation. + p All kinds of content are supported, including pictures, threads, emojis, and @mentions. + p It's really easy to set up, even if you only have Discord. Just add the bot to your server, and it'll make everything available on Matrix automatically. if locked - h2 This is a private instance - p Anybody can run their own instance of the Out Of Your Element software. The person running this instance has made it private, so you can't add it to your server just yet. If you know who's in charge of #{reg.ooye.server_name}, ask them for the password. + block locked-info + h2 This is a private instance + p Anybody can run their own instance of the Out Of Your Element software. The person running this instance has made it private, so you can't add it to your server just yet. If you know who's in charge of #{reg.ooye.server_name}, ask them for the password. - h2 Run your own instance - p You can still use Out Of Your Element by running your own copy of the software, but this requires some technical skill. - p To get started, #[a(href="https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md") check the installation instructions.] + h2 Run your own instance + p You can still use Out Of Your Element by running your own copy of the software, but this requires some technical skill. + p To get started, #[a(href="https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md") check the installation instructions.] diff --git a/src/web/pug/includes/template.pug b/src/web/pug/includes/template.pug index 452f8d53..9fe80aad 100644 --- a/src/web/pug/includes/template.pug +++ b/src/web/pug/includes/template.pug @@ -1,4 +1,10 @@ -mixin guild(guild) +mixin guild-menuitem(guild) + - let bridgedRoomCount = from("channel_room").selectUnsafe("count(*) as count").where({guild_id: guild.id}).and("AND thread_parent IS NULL").get().count + li(role="menuitem") + a.s-topbar--item.s-user-card.d-flex.p4(href=rel(`/guild?guild_id=${guild.id}`) class={"bg-purple-200": bridgedRoomCount === 0, "h:bg-purple-300": bridgedRoomCount === 0}) + +guild(guild, bridgedRoomCount) + +mixin guild(guild, bridgedRoomCount) span.s-avatar.s-avatar__32.s-user-card--avatar if guild.icon img.s-avatar--image(src=`https://cdn.discordapp.com/icons/${guild.id}/${guild.icon}.png?size=32` alt="") @@ -6,8 +12,12 @@ mixin guild(guild) .s-avatar--letter.bg-silver-400.bar-md(aria-hidden="true")= guild.name[0] .s-user-card--info.ai-start strong= guild.name - ul.s-user-card--awards - li #{discord.guildChannelMap.get(guild.id).filter(c => [0, 5, 15, 16].includes(discord.channels.get(c).type)).length} channels + if bridgedRoomCount != null + ul.s-user-card--awards + if bridgedRoomCount + li #{bridgedRoomCount} bridged rooms + else + li.fc-purple Not yet linked mixin define-theme(name, h, s, l) style. @@ -58,6 +68,8 @@ html(lang="en") title Out Of Your Element link(rel="stylesheet" type="text/css" href=rel("/static/stacks.min.css")) + //- Please use responsibly!!!!! + link(rel="stylesheet" type="text/css" href=rel("/custom.css")) meta(name="htmx-config" content='{"requestClass":"is-loading"}') style. @@ -79,6 +91,14 @@ html(lang="en") .s-btn__dropdown:has(+ :popover-open) { background-color: var(--theme-topbar-item-background-hover, var(--black-200)) !important; } + @media (prefers-color-scheme: dark) { + body.theme-system .s-popover { + --_po-bg: var(--black-100); + --_po-bc: var(--bc-light); + --_po-bs: var(--bs-lg); + --_po-arrow-fc: var(--black-100); + } + } +define-themed-button("matrix", "black") body.themed.theme-system header.s-topbar @@ -114,9 +134,7 @@ html(lang="en") .s-popover--content.overflow-y-auto.overflow-x-hidden ul.s-menu(role="menu") each guild in [...managed].map(id => discord.guilds.get(id)).filter(g => g).sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? -1 : 1) - li(role="menuitem") - a.s-topbar--item.s-user-card.d-flex.p4(href=rel(`/guild?guild_id=${guild.id}`)) - +guild(guild) + +guild-menuitem(guild) //- Body .mx-auto.w100.wmx9.py24.px8.fs-body1#content block body diff --git a/src/web/server.js b/src/web/server.js index a2148770..dc13cf0d 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -4,13 +4,14 @@ const assert = require("assert") const fs = require("fs") const {join} = require("path") const h3 = require("h3") -const {defineEventHandler, defaultContentType, getRequestHeader, setResponseHeader, handleCacheHeaders} = h3 +const mimeTypes = require("mime-types") +const {defineEventHandler, defaultContentType, getRequestHeader, setResponseHeader, handleCacheHeaders, serveStatic} = h3 const icons = require("@stackoverflow/stacks-icons") const DiscordTypes = require("discord-api-types/v10") const dUtils = require("../discord/utils") const reg = require("../matrix/read-registration") -const {sync, discord, as, select} = require("../passthrough") +const {sync, discord, as, select, from} = require("../passthrough") /** @type {import("./pug-sync")} */ const pugSync = sync.require("./pug-sync") /** @type {import("../matrix/utils")} */ @@ -19,21 +20,7 @@ const {id} = require("../../addbot") // Pug -pugSync.addGlobals({id, h3, discord, select, DiscordTypes, dUtils, mUtils, icons, reg: reg.reg}) -pugSync.createRoute(as.router, "/", "home.pug") -pugSync.createRoute(as.router, "/ok", "ok.pug") - -// Routes - -sync.require("./routes/download-matrix") -sync.require("./routes/download-discord") -sync.require("./routes/guild-settings") -sync.require("./routes/guild") -sync.require("./routes/info") -sync.require("./routes/link") -sync.require("./routes/log-in-with-matrix") -sync.require("./routes/oauth") -sync.require("./routes/password") +pugSync.addGlobals({id, h3, discord, select, from, DiscordTypes, dUtils, mUtils, icons, reg: reg.reg}) // Files @@ -65,12 +52,79 @@ as.router.get("/static/htmx.js", defineEventHandler({ } })) -as.router.get("/icon.png", defineEventHandler(event => { - handleCacheHeaders(event, {maxAge: 86400}) - return fs.promises.readFile(join(__dirname, "../../docs/img/icon.png")) -})) - as.router.get("/download/file/poll-star-avatar.png", defineEventHandler(event => { handleCacheHeaders(event, {maxAge: 86400}) return fs.promises.readFile(join(__dirname, "../../docs/img/poll-star-avatar.png")) })) + +// Custom files + +const publicDir = "custom-webroot" + +/** + * @param {h3.H3Event} event + * @param {boolean} fallthrough + */ +function tryStatic(event, fallthrough) { + return serveStatic(event, { + indexNames: ["/index.html", "/index.pug"], + fallthrough, + getMeta: async id => { + // Check + const stats = await fs.promises.stat(join(publicDir, id)).catch(() => {}); + if (!stats || !stats.isFile()) { + return + } + // Pug + if (id.match(/\.pug$/)) { + defaultContentType(event, "text/html; charset=utf-8") + return {} + } + // Everything else + else { + const mime = mimeTypes.lookup(id) + if (typeof mime === "string") defaultContentType(event, mime) + return { + size: stats.size + } + } + }, + getContents: id => { + if (id.match(/\.pug$/)) { + const path = join(publicDir, id) + return pugSync.renderPath(event, path, {}) + } else { + return fs.promises.readFile(join(publicDir, id)) + } + } + }) +} + +as.router.get("/**", defineEventHandler(event => { + return tryStatic(event, false) +})) + +as.router.get("/", defineEventHandler(async event => { + return (await tryStatic(event, true)) || pugSync.render(event, "home.pug", {}) +})) + +as.router.get("/icon.png", defineEventHandler(async event => { + const s = await tryStatic(event, true) + if (s) return s + handleCacheHeaders(event, {maxAge: 86400}) + return fs.promises.readFile(join(__dirname, "../../docs/img/icon.png")) +})) + +// Routes + +pugSync.createRoute(as.router, "/ok", "ok.pug") + +sync.require("./routes/download-matrix") +sync.require("./routes/download-discord") +sync.require("./routes/guild-settings") +sync.require("./routes/guild") +sync.require("./routes/info") +sync.require("./routes/link") +sync.require("./routes/log-in-with-matrix") +sync.require("./routes/oauth") +sync.require("./routes/password") From ee583fddbdaf55a68041040541426ae32cabfcd3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 17 Feb 2026 12:56:18 +1300 Subject: [PATCH 145/153] Fix server names with numbers in them --- scripts/setup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup.js b/scripts/setup.js index ecc57fd3..4e6de0a0 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -86,7 +86,7 @@ function defineEchoHandler() { type: "input", name: "server_name", message: "Homeserver name", - validate: serverName => !!serverName.match(/[a-z][a-z.]+[a-z]/) + validate: serverName => !!serverName.match(/[a-z0-9][.a-z0-9-]+[a-z]/) }) console.log("What is the URL of your homeserver?") From 9f9cfdb53493eefbed4c7023d6ad937046ac9f1a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 17 Feb 2026 14:03:57 +1300 Subject: [PATCH 146/153] Allow namespace prefix to be empty string --- src/matrix/read-registration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/read-registration.js b/src/matrix/read-registration.js index 6dc64dd4..114bf756 100644 --- a/src/matrix/read-registration.js +++ b/src/matrix/read-registration.js @@ -11,7 +11,7 @@ const registrationFilePath = path.join(process.cwd(), "registration.yaml") function checkRegistration(reg) { reg["ooye"].invite = reg.ooye.invite.filter(mxid => mxid.endsWith(`:${reg.ooye.server_name}`)) // one day I will understand why typescript disagrees with dot notation on this line assert(reg.ooye?.max_file_size) - assert(reg.ooye?.namespace_prefix) + assert(reg.ooye?.namespace_prefix != null) assert(reg.ooye?.server_name) assert(reg.sender_localpart?.startsWith(reg.ooye.namespace_prefix), "appservice's localpart must be in the namespace it controls") assert(reg.ooye?.server_origin.match(/^https?:\/\//), "server origin must start with http or https") From 411491b405c92c7817ee552745429f6e59d16958 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 21 Feb 2026 12:04:42 +1300 Subject: [PATCH 147/153] Remove live dependency on cadence.moe --- scripts/setup.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/setup.js b/scripts/setup.js index 4e6de0a0..69b62a29 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -1,6 +1,7 @@ #!/usr/bin/env node // @ts-check +const Ty = require("../src/types") const assert = require("assert").strict const fs = require("fs") const sqlite = require("better-sqlite3") @@ -285,8 +286,8 @@ function defineEchoHandler() { console.log() // Done with user prompts, reg is now guaranteed to be valid + const mreq = require("../src/matrix/mreq") const api = require("../src/matrix/api") - const file = require("../src/matrix/file") const DiscordClient = require("../src/d2m/discord-client") const discord = new DiscordClient(reg.ooye.discord_token, "no") passthrough.discord = discord @@ -343,7 +344,13 @@ function defineEchoHandler() { await api.register(reg.sender_localpart) // upload initial images... - const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png") + const avatarBuffer = await fs.promises.readFile(join(__dirname, "..", "docs", "img", "icon.png"), null) + /** @type {Ty.R.FileUploaded} */ + const root = await mreq.mreq("POST", "/media/v3/upload", avatarBuffer, { + headers: {"Content-Type": "image/png"} + }) + const avatarUrl = root.content_uri + assert(avatarUrl) console.log("✅ Matrix appservice login works...") @@ -352,8 +359,7 @@ function defineEchoHandler() { console.log("✅ Emojis are ready...") // set profile data on discord... - const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer()) - await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")}) + await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + avatarBuffer.toString("base64")}) console.log("✅ Discord profile updated...") // set profile data on homeserver... From 9b3707baa1dc7f9308f3152068d69f0678d16d30 Mon Sep 17 00:00:00 2001 From: Abdul <32655037-CamperThumper@users.noreply.gitlab.com> Date: Wed, 25 Feb 2026 01:03:30 +0300 Subject: [PATCH 148/153] Link sticker instead of file upload --- src/m2d/actions/sticker.js | 53 ++++++++++++++++++++++++++ src/m2d/converters/event-to-message.js | 25 +++++------- src/web/routes/download-matrix.js | 19 +++++++++ 3 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 src/m2d/actions/sticker.js diff --git a/src/m2d/actions/sticker.js b/src/m2d/actions/sticker.js new file mode 100644 index 00000000..f9c06bfb --- /dev/null +++ b/src/m2d/actions/sticker.js @@ -0,0 +1,53 @@ +// @ts-check + +const streamr = require("stream") +const {pipeline} = require("stream").promises + +const {sync} = require("../../passthrough") +const sharp = require("sharp") +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/mreq")} */ +const mreq = sync.require("../../matrix/mreq") +const streamMimeType = require("stream-mime-type") + +const WIDTH = 160 +const HEIGHT = 160 +/** + * Downloads the sticker from the web and converts to webp data. + * @param {string} mxc a single mxc:// URL + * @returns {Promise} sticker webp data, or undefined if the downloaded sticker is not valid + */ +async function getAndResizeSticker(mxc) { + const res = await api.getMedia(mxc) + if (res.status !== 200) { + const root = await res.json() + throw new mreq.MatrixServerError(root, {mxc}) + } + const streamIn = streamr.Readable.fromWeb(res.body) + + const { stream, mime } = await streamMimeType.getMimeType(streamIn) + let animated = false + if (mime === "image/gif" || mime === "image/webp") { + animated = true + } + + const result = await new Promise((resolve, reject) => { + const transformer = sharp({animated: animated}) + .resize(WIDTH, HEIGHT, {fit: "inside", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .webp() + .toBuffer((err, buffer, info) => { + /* c8 ignore next */ + if (err) return reject(err) + resolve({info, buffer}) + }) + pipeline( + stream, + transformer + ) + }) + return result.buffer +} + + +module.exports.getAndResizeSticker = getAndResizeSticker diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 2add279f..91c24008 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -632,23 +632,16 @@ async function eventToMessage(event, guild, channel, di) { if (event.type === "m.sticker") { content = "" - let filename = event.content.body - if (event.type === "m.sticker") { - let mimetype - if (event.content.info?.mimetype?.includes("/")) { - mimetype = event.content.info.mimetype - } else { - const res = await di.api.getMedia(event.content.url, {method: "HEAD"}) - if (res.status === 200) { - mimetype = res.headers.get("content-type") - } - if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`) - } - filename += "." + mimetype.split("/")[1] - } - attachments.push({id: "0", filename}) - pendingFiles.push({name: filename, mxc: event.content.url}) + content += `[${event.content.body}](` // sticker title for fallback if the url preview fails + const afterLink = ")" + // Make sticker URL params + const params = new URLSearchParams() + const withoutMxc = mxUtils.makeMxcPublic(event.content.url) + assert(withoutMxc) + params.append("mxc", withoutMxc) + const url = `${reg.ooye.bridge_origin}/download/sticker.webp?${params.toString()}` + content += url + afterLink } else if (event.type === "org.matrix.msc3381.poll.start") { const pollContent = event.content["org.matrix.msc3381.poll.start"] // just for convenience const isClosed = false; diff --git a/src/web/routes/download-matrix.js b/src/web/routes/download-matrix.js index bb6b8508..1fbc1d6a 100644 --- a/src/web/routes/download-matrix.js +++ b/src/web/routes/download-matrix.js @@ -16,6 +16,9 @@ const emojiSheet = sync.require("../../m2d/actions/emoji-sheet") /** @type {import("../../m2d/converters/emoji-sheet")} */ const emojiSheetConverter = sync.require("../../m2d/converters/emoji-sheet") +/** @type {import("../../m2d/actions/sticker")} */ +const sticker = sync.require("../../m2d/actions/sticker") + const schema = { params: z.object({ server_name: z.string(), @@ -23,6 +26,9 @@ const schema = { }), sheet: z.object({ e: z.array(z.string()).or(z.string()) + }), + sticker: z.object({ + mxc: z.string() }) } @@ -90,3 +96,16 @@ as.router.get(`/download/sheet`, defineEventHandler(async event => { setResponseHeader(event, "Content-Type", "image/png") return buffer })) + +as.router.get(`/download/sticker.webp`, defineEventHandler(async event => { + const query = await getValidatedQuery(event, schema.sticker.parse) + + /** remember that these have no mxc:// protocol in the string */ + verifyMediaHash(query.mxc) + const mxc = `mxc://${query.mxc}` + + setResponseHeader(event, "Content-Type", 'image/webp') + const buffer = await sticker.getAndResizeSticker(mxc) + return buffer +})) + From d1aa8f01e70e78d7ae438059872d4f7ef52bff1a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 25 Feb 2026 18:21:35 +1300 Subject: [PATCH 149/153] Change sticker URL and stream response --- src/m2d/actions/sticker.js | 33 ++++++++------------------ src/m2d/converters/event-to-message.js | 12 +++------- src/web/routes/download-matrix.js | 21 ++++++++-------- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/src/m2d/actions/sticker.js b/src/m2d/actions/sticker.js index f9c06bfb..341d8b04 100644 --- a/src/m2d/actions/sticker.js +++ b/src/m2d/actions/sticker.js @@ -1,7 +1,7 @@ // @ts-check -const streamr = require("stream") -const {pipeline} = require("stream").promises +const {Readable} = require("stream") +const {ReadableStream} = require("stream/web") const {sync} = require("../../passthrough") const sharp = require("sharp") @@ -16,7 +16,7 @@ const HEIGHT = 160 /** * Downloads the sticker from the web and converts to webp data. * @param {string} mxc a single mxc:// URL - * @returns {Promise} sticker webp data, or undefined if the downloaded sticker is not valid + * @returns {Promise} sticker webp data, or undefined if the downloaded sticker is not valid */ async function getAndResizeSticker(mxc) { const res = await api.getMedia(mxc) @@ -24,29 +24,16 @@ async function getAndResizeSticker(mxc) { const root = await res.json() throw new mreq.MatrixServerError(root, {mxc}) } - const streamIn = streamr.Readable.fromWeb(res.body) + const streamIn = Readable.fromWeb(res.body) const { stream, mime } = await streamMimeType.getMimeType(streamIn) - let animated = false - if (mime === "image/gif" || mime === "image/webp") { - animated = true - } + const animated = ["image/gif", "image/webp"].includes(mime) - const result = await new Promise((resolve, reject) => { - const transformer = sharp({animated: animated}) - .resize(WIDTH, HEIGHT, {fit: "inside", background: {r: 0, g: 0, b: 0, alpha: 0}}) - .webp() - .toBuffer((err, buffer, info) => { - /* c8 ignore next */ - if (err) return reject(err) - resolve({info, buffer}) - }) - pipeline( - stream, - transformer - ) - }) - return result.buffer + const transformer = sharp({animated: animated}) + .resize(WIDTH, HEIGHT, {fit: "inside", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .webp() + stream.pipe(transformer) + return Readable.toWeb(transformer) } diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 91c24008..81ad48c4 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -631,17 +631,11 @@ async function eventToMessage(event, guild, channel, di) { } if (event.type === "m.sticker") { - content = "" - content += `[${event.content.body}](` // sticker title for fallback if the url preview fails - const afterLink = ")" - - // Make sticker URL params - const params = new URLSearchParams() const withoutMxc = mxUtils.makeMxcPublic(event.content.url) assert(withoutMxc) - params.append("mxc", withoutMxc) - const url = `${reg.ooye.bridge_origin}/download/sticker.webp?${params.toString()}` - content += url + afterLink + const url = `${reg.ooye.bridge_origin}/download/sticker/${withoutMxc}/_.webp` + content = `[${event.content.body || "\u2800"}](${url})` + } else if (event.type === "org.matrix.msc3381.poll.start") { const pollContent = event.content["org.matrix.msc3381.poll.start"] // just for convenience const isClosed = false; diff --git a/src/web/routes/download-matrix.js b/src/web/routes/download-matrix.js index 1fbc1d6a..82e2f7e6 100644 --- a/src/web/routes/download-matrix.js +++ b/src/web/routes/download-matrix.js @@ -28,7 +28,8 @@ const schema = { e: z.array(z.string()).or(z.string()) }), sticker: z.object({ - mxc: z.string() + server_name: z.string().regex(/^[^/]+$/), + media_id: z.string().regex(/^[A-Za-z0-9_-]+$/) }) } @@ -97,15 +98,13 @@ as.router.get(`/download/sheet`, defineEventHandler(async event => { return buffer })) -as.router.get(`/download/sticker.webp`, defineEventHandler(async event => { - const query = await getValidatedQuery(event, schema.sticker.parse) +as.router.get(`/download/sticker/:server_name/:media_id/_.webp`, defineEventHandler(async event => { + const {server_name, media_id} = await getValidatedRouterParams(event, schema.sticker.parse) + /** remember that this has no mxc:// protocol in the string */ + const mxc = server_name + "/" + media_id + verifyMediaHash(mxc) - /** remember that these have no mxc:// protocol in the string */ - verifyMediaHash(query.mxc) - const mxc = `mxc://${query.mxc}` - - setResponseHeader(event, "Content-Type", 'image/webp') - const buffer = await sticker.getAndResizeSticker(mxc) - return buffer + const stream = await sticker.getAndResizeSticker(`mxc://${mxc}`) + setResponseHeader(event, "Content-Type", "image/webp") + return stream })) - From ea261e825b8b9a80ebae44d3048042af44df8e55 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 27 Feb 2026 18:33:29 +1300 Subject: [PATCH 150/153] Slashes not allowed in MXID --- src/d2m/converters/user-to-mxid.js | 6 +++--- src/d2m/converters/user-to-mxid.test.js | 8 ++++++-- src/db/migrations/0034-slash-not-allowed-in-mxid.sql | 5 +++++ 3 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 src/db/migrations/0034-slash-not-allowed-in-mxid.sql diff --git a/src/d2m/converters/user-to-mxid.js b/src/d2m/converters/user-to-mxid.js index 12891c09..7705aff1 100644 --- a/src/d2m/converters/user-to-mxid.js +++ b/src/d2m/converters/user-to-mxid.js @@ -20,10 +20,10 @@ const SPECIAL_USER_MAPPINGS = new Map([ function downcaseUsername(user) { // First, try to convert the username to the set of allowed characters let downcased = user.username.toLowerCase() - // spaces to underscores... - .replace(/ /g, "_") + // spaces and slashes to underscores... + .replace(/[ /]/g, "_") // remove disallowed characters... - .replace(/[^a-z0-9._=/-]*/g, "") + .replace(/[^a-z0-9._=-]*/g, "") // remove leading and trailing dashes and underscores... .replace(/(?:^[_-]*|[_-]*$)/g, "") // If requested, also make the Discord user ID part of the username diff --git a/src/d2m/converters/user-to-mxid.test.js b/src/d2m/converters/user-to-mxid.test.js index 387d4726..f8cf16a1 100644 --- a/src/d2m/converters/user-to-mxid.test.js +++ b/src/d2m/converters/user-to-mxid.test.js @@ -21,8 +21,12 @@ test("user2name: works on single emoji at the end", t => { t.equal(userToSimName({username: "Melody 🎵", discriminator: "2192"}), "melody") }) -test("user2name: works on crazy name", t => { - t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7//") +test("user2name: works on really weird name", t => { + t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7") +}) + +test("user2name: treats slashes", t => { + t.equal(userToSimName({username: "Evil Lillith (she/her)", discriminator: "5892"}), "evil_lillith_she_her") }) test("user2name: adds discriminator if name is unavailable (old tag format)", t => { diff --git a/src/db/migrations/0034-slash-not-allowed-in-mxid.sql b/src/db/migrations/0034-slash-not-allowed-in-mxid.sql new file mode 100644 index 00000000..ea2d031a --- /dev/null +++ b/src/db/migrations/0034-slash-not-allowed-in-mxid.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +DELETE FROM sim WHERE sim_name like '%/%'; + +COMMIT; From 780154fd09beda223a21bf5d17d407d07b2c7192 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 27 Feb 2026 18:34:30 +1300 Subject: [PATCH 151/153] Bots with Administrator may access all channels --- src/web/routes/guild.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/routes/guild.js b/src/web/routes/guild.js index dfb393b7..a5508c4a 100644 --- a/src/web/routes/guild.js +++ b/src/web/routes/guild.js @@ -115,7 +115,7 @@ function getChannelRoomsLinks(guild, rooms, roles) { let removedWrongTypeChannels = dUtils.filterTo(unlinkedChannels, c => c && [0, 5].includes(c.type)) let removedPrivateChannels = dUtils.filterTo(unlinkedChannels, c => { const permissions = dUtils.getPermissions(guild.id, roles, guild.roles, botID, c["permission_overwrites"]) - return dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel) + return dUtils.hasSomePermissions(permissions, ["Administrator", "ViewChannel"]) }) unlinkedChannels.sort((a, b) => getPosition(a, discord.channels) - getPosition(b, discord.channels)) From e275d4c928b9bcbc7d3c855b7ce6900dae1e8686 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 27 Feb 2026 18:35:48 +1300 Subject: [PATCH 152/153] Add script to estimate total channel file size --- scripts/estimate-size.js | 65 ++++++++++++++++++++++++++++++++++++++++ src/matrix/api.js | 2 +- src/types.d.ts | 8 ++++- 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 scripts/estimate-size.js diff --git a/scripts/estimate-size.js b/scripts/estimate-size.js new file mode 100644 index 00000000..341abc07 --- /dev/null +++ b/scripts/estimate-size.js @@ -0,0 +1,65 @@ +// @ts-check + +const pb = require("prettier-bytes") +const sqlite = require("better-sqlite3") +const HeatSync = require("heatsync") + +const {reg} = require("../src/matrix/read-registration") +const passthrough = require("../src/passthrough") + +const sync = new HeatSync({watchFS: false}) +Object.assign(passthrough, {reg, sync}) + +const DiscordClient = require("../src/d2m/discord-client") + +const discord = new DiscordClient(reg.ooye.discord_token, "no") +passthrough.discord = discord + +const db = new sqlite("ooye.db") +passthrough.db = db + +const api = require("../src/matrix/api") + +const {room: roomID} = require("minimist")(process.argv.slice(2), {string: ["room"]}) +if (!roomID) { + console.error("Usage: ./scripts/estimate-size.js --room=") + process.exit(1) +} + +const {channel_id, guild_id} = db.prepare("SELECT channel_id, guild_id FROM channel_room WHERE room_id = ?").get(roomID) + +const max = 1000 + +;(async () => { + let total = 0 + let size = 0 + let from + + while (total < max) { + const events = await api.getEvents(roomID, "b", {limit: 1000, from}) + total += events.chunk.length + from = events.end + console.log(`Fetched ${total} events so far`) + + for (const e of events.chunk) { + if (e.content?.info?.size) { + size += e.content.info.size + } + } + + if (events.chunk.length === 0 || !events.end) break + } + + console.log(`Total size of uploads: ${pb(size)}`) + + const searchResults = await discord.snow.requestHandler.request(`/guilds/${guild_id}/messages/search`, { + channel_id, + offset: "0", + limit: "1" + }, "get", "json") + + const totalAllTime = searchResults.total_results + const fractionCounted = total / totalAllTime + console.log(`That counts for ${(fractionCounted*100).toFixed(2)}% of the history on Discord (${totalAllTime.toLocaleString()} messages)`) + console.log(`The size of uploads for the whole history would be approx: ${pb(Math.floor(size/total*totalAllTime))}`) +})() diff --git a/src/matrix/api.js b/src/matrix/api.js index 70cb50be..87bbf0cc 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -136,7 +136,7 @@ async function getEventForTimestamp(roomID, ts) { */ async function getEvents(roomID, dir, pagination = {}, filter) { filter = filter && JSON.stringify(filter) - /** @type {Ty.Pagination>} */ + /** @type {Ty.MessagesPagination>} */ const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/messages`, null, {...pagination, dir, filter})) return root } diff --git a/src/types.d.ts b/src/types.d.ts index 6ee2eb13..a85907d5 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -498,7 +498,13 @@ export type Membership = "invite" | "knock" | "join" | "leave" | "ban" export type Pagination = { chunk: T[] next_batch?: string - prev_match?: string + prev_batch?: string +} + +export type MessagesPagination = { + chunk: T[] + start: string + end?: string } export type HierarchyPagination = { From c68bac5476dd6809dffacfd09d467db1f257ba83 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 1 Mar 2026 22:05:46 +1300 Subject: [PATCH 153/153] Document encryption as unsupported --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 8c9fc90c..e8a8e7e7 100644 --- a/readme.md +++ b/readme.md @@ -38,6 +38,7 @@ For more information about features, [see the user guide.](https://gitdab.com/ca * This bridge is not designed for puppetting. * Direct Messaging is not supported until I figure out a good way of doing it. +* Encrypted messages are not supported. Decryption is often unreliable on Matrix, and your messages end up in plaintext on Discord anyway, so there's not much advantage. ## Get started!