From aa263d3eb84a8250847ecd063089f7d71fee36e1 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 3 Sep 2023 16:38:54 +1200 Subject: [PATCH 1/6] m->d avoid using blockquote as reply preview --- m2d/converters/event-to-message.js | 2 +- m2d/converters/event-to-message.test.js | 49 +++++++++++++++++++++++++ notes.md | 7 ++-- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 5ad62c1..1eb1be2 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -210,7 +210,7 @@ async function eventToMessage(event, guild, di) { replyLine += `Ⓜ️**${senderName}**:` } const repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body - const contentPreviewChunks = chunk(repliedToContent.replace(/.*<\/mx-reply>/, "").replace(/(?:\n|
)+/g, " ").replace(/<[^>]+>/g, ""), 50) + const contentPreviewChunks = chunk(repliedToContent.replace(/.*<\/mx-reply>/, "").replace(/.*?<\/blockquote>/, "").replace(/(?:\n|
)+/g, " ").replace(/<[^>]+>/g, ""), 50) const contentPreview = contentPreviewChunks.length > 1 ? contentPreviewChunks[0] + "..." : contentPreviewChunks[0] replyLine = `> ${replyLine}\n> ${contentPreview}\n` })() diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 565d8c4..e9061b7 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -535,6 +535,55 @@ test("event2message: rich reply to a sim user", async t => { ) }) +test("event2message: should avoid using blockquote contents as reply preview in rich reply to a sim user", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "> <@_ooye_kyuugryphon:cadence.moe> > well, you said this, so...\n> \n> that can't be true! there's no way :o\n\nI agree!", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @_ooye_kyuugryphon:cadence.moe
well, you said this, so...

that can't be true! there's no way :o
I agree!", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04" + } + } + }, + event_id: "$BpGx8_vqHyN6UQDARPDU51ftrlRBhleutRSgpAJJ--g", + room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { + "type": "m.room.message", + "sender": "@_ooye_kyuugryphon:cadence.moe", + "content": { + "m.mentions": {}, + "msgtype": "m.text", + "body": "> well, you said this, so...\n\nthat can't be true! there's no way :o", + "format": "org.matrix.custom.html", + "formatted_body": "
well, you said this, so...

that can't be true! there's no way :o" + } + }) + } + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 <@111604486476181504>:" + + "\n> that can't be true! there's no way :o" + + "\nI agree!", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" + }] + } + ) +}) + + + test("event2message: editing a rich reply to a sim user", async t => { const eventsFetched = [] t.deepEqual( diff --git a/notes.md b/notes.md index 0b6b088..458e3bd 100644 --- a/notes.md +++ b/notes.md @@ -2,14 +2,13 @@ ## Known issues -- m->d attachments do not work -- m->d replying to a message that used a blockquote should avoid using the blockquote contents as the preview - d->m emojis do not work at all (inline chat, single emoji size, reactions, bridged state) +- d->m embeds - m->d code blocks have slightly too much spacing -- m->d some reactions don't work because of the variation selector - d->m check whether I implemented deletions - m->d deletions -- rooms will be set up even if the bridge does not have permission for them, then break when it restarts and tries to reach messages +- removing reactions +- rooms will be set up even if the bridge does not have permission for the channels, which breaks when it restarts and tries to fetch messages - test private threads as part of this - solution part 1: calculate the permissions to see if the bot should be able to do stuff - solution part 2: attempt a get messages request anyway before bridging a new room, just to make sure! From 5e6bb0cd2edaa8c340b5f27d5ccfce622c22fc8e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 3 Sep 2023 17:13:04 +1200 Subject: [PATCH 2/6] create syncSpaceFully --- d2m/actions/create-room.js | 5 ++++- d2m/actions/create-space.js | 43 +++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js index e778d22..e6e2cbe 100644 --- a/d2m/actions/create-room.js +++ b/d2m/actions/create-room.js @@ -82,7 +82,7 @@ async function channelToKState(channel, guild) { avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API } - let history_visibility = "invited" + let history_visibility = "shared" if (channel["thread_metadata"]) history_visibility = "world_readable" const channelKState = { @@ -106,6 +106,9 @@ async function channelToKState(channel, guild) { events: { "m.room.avatar": 0 } + }, + "chat.schildi.hide_ui/read_receipts": { + hidden: true } } diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index 87e25eb..2c86f75 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -67,6 +67,7 @@ async function guildToKState(guild) { return guildKState } +/** Efficiently update space name, space avatar, and child room avatars. */ async function syncSpace(guildID) { /** @ts-ignore @type {DiscordTypes.APIGuild} */ const guild = discord.guilds.get(guildID) @@ -77,10 +78,7 @@ async function syncSpace(guildID) { const guildKState = await guildToKState(guild) - if (!spaceID) { - const spaceID = await createSpace(guild, guildKState) - return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here. - } + if (!spaceID) return console.log(`[space sync] to matrix: ${guild.name}`) @@ -111,6 +109,43 @@ async function syncSpace(guildID) { return spaceID } +/** + * Inefficiently force the space and its existing child rooms to be fully updated. + * Should not need to be called as part of the bridge's normal operation. + */ +async function syncSpaceFully(guildID) { + /** @ts-ignore @type {DiscordTypes.APIGuild} */ + const guild = discord.guilds.get(guildID) + assert.ok(guild) + + /** @type {string?} */ + const spaceID = db.prepare("SELECT space_id from guild_space WHERE guild_id = ?").pluck().get(guildID) + + const guildKState = await guildToKState(guild) + + if (!spaceID) return + + console.log(`[space sync] to matrix: ${guild.name}`) + + // sync guild state to space + const spaceKState = await createRoom.roomToKState(spaceID) + const spaceDiff = ks.diffKState(spaceKState, guildKState) + await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff) + + const childRooms = ks.kstateToState(spaceKState).filter(({type, content}) => { + return type === "m.space.child" && "via" in content + }).map(({state_key}) => state_key) + + for (const roomID of childRooms) { + const channelID = db.prepare("SELECT channel_id FROM channel_room WHERE room_id = ?").pluck().get(roomID) + if (!channelID) continue + await createRoom.syncRoom(channelID) + } + + return spaceID +} + module.exports.createSpace = createSpace module.exports.syncSpace = syncSpace +module.exports.syncSpaceFully = syncSpaceFully module.exports.guildToKState = guildToKState From d3f637d7cee71b9c37f334c7c04e605cb6169a6b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 4 Sep 2023 01:36:35 +1200 Subject: [PATCH 3/6] properly support guild update events --- d2m/discord-packets.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index 6ece8ca..b2fb643 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -43,6 +43,15 @@ const utils = { } eventDispatcher.checkMissedMessages(client, message.d) + } else if (message.t === "GUILD_UPDATE") { + const guild = client.guilds.get(message.d.id) + if (guild) { + for (const prop of Object.keys(message.d)) { + if (!["channels", "threads"].includes(prop)) { + guild[prop] = message.d[prop] + } + } + } } else if (message.t === "THREAD_CREATE") { client.channels.set(message.d.id, message.d) From d255f2ab22402293cd23e67ac9921bd7111d9e4e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 4 Sep 2023 01:36:58 +1200 Subject: [PATCH 4/6] fix join rules on old room versions --- d2m/actions/create-room.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js index e6e2cbe..679c0de 100644 --- a/d2m/actions/create-room.js +++ b/d2m/actions/create-room.js @@ -95,6 +95,7 @@ async function channelToKState(channel, guild) { via: [reg.ooye.server_name], canonical: true }, + /** @type {{join_rule: string, [x: string]: any}} */ "m.room.join_rules/": { join_rule: "restricted", allow: [{ @@ -255,6 +256,12 @@ async function _syncRoom(channelID, shouldActuallySync) { // sync channel state to room const roomKState = await roomToKState(roomID) + if (+roomKState["m.room.create/"].room_version <= 8) { + // join_rule `restricted` is not available in room version < 8 and not working properly in version == 8 + // read more: https://spec.matrix.org/v1.8/rooms/v9/ + // we have to use `public` instead, otherwise the room will be unjoinable. + channelKState["m.room.join_rules/"] = {join_rule: "public"} + } const roomDiff = ks.diffKState(roomKState, channelKState) const roomApply = applyKStateDiffToRoom(roomID, roomDiff) db.prepare("UPDATE channel_room SET name = ? WHERE room_id = ?").run(channel.name, roomID) From 5185ae45ab7ec7364b3160d65c26cc7dc2102173 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 4 Sep 2023 01:37:33 +1200 Subject: [PATCH 5/6] m->d fix a few markdown escaping problems --- m2d/converters/event-to-message.js | 43 +++++++++++++-- m2d/converters/event-to-message.test.js | 72 ++++++++++++++++++++++--- 2 files changed, 105 insertions(+), 10 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 1eb1be2..1e66c7a 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -22,13 +22,48 @@ const BLOCK_ELEMENTS = [ "TFOOT", "TH", "THEAD", "TR", "UL" ] +/** @type {[RegExp, string][]} */ +const markdownEscapes = [ + [/\\/g, '\\\\'], + [/\*/g, '\\*'], + [/^-/g, '\\-'], + [/^\+ /g, '\\+ '], + [/^(=+)/g, '\\$1'], + [/^(#{1,6}) /g, '\\$1 '], + [/`/g, '\\`'], + [/^~~~/g, '\\~~~'], + [/\[/g, '\\['], + [/\]/g, '\\]'], + [/^>/g, '\\>'], + [/_/g, '\\_'], + [/^(\d+)\. /g, '$1\\. '] + ] + const turndownService = new TurndownService({ hr: "----", headingStyle: "atx", preformattedCode: true, - codeBlockStyle: "fenced" + codeBlockStyle: "fenced", }) +/** + * Markdown characters in the HTML content need to be escaped, though take care not to escape the middle of bare links + * @param {string} string + */ +// @ts-ignore bad type from turndown +turndownService.escape = function (string) { + const escapedWords = string.split(" ").map(word => { + if (word.match(/^https?:\/\//)) { + return word + } else { + return markdownEscapes.reduce(function (accumulator, escape) { + return accumulator.replace(escape[0], escape[1]) + }, word) + } + }) + return escapedWords.join(" ") +} + turndownService.remove("mx-reply") turndownService.addRule("strikethrough", { @@ -67,7 +102,6 @@ turndownService.addRule("spoiler", { turndownService.addRule("inlineLink", { filter: function (node, options) { return ( - options.linkStyle === "inlined" && node.nodeName === "A" && node.getAttribute("href") ) @@ -275,8 +309,9 @@ async function eventToMessage(event, guild, di) { content = `* ${displayName} ${content}` } - // Markdown needs to be escaped - content = content.replace(/([*_~`#])/g, `\\$1`) + // Markdown needs to be escaped, though take care not to escape the middle of links + // @ts-ignore bad type from turndown + content = turndownService.escape(content) } } else if (event.type === "m.room.message" && (event.content.msgtype === "m.file" || event.content.msgtype === "m.video" || event.content.msgtype === "m.audio" || event.content.msgtype === "m.image")) { content = "" diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index e9061b7..e7f28d4 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -64,11 +64,11 @@ test("event2message: body is used when there is no formatted_body", async t => { ) }) -test("event2message: any markdown in body is escaped", async t => { +test("event2message: any markdown in body is escaped, except strikethrough", async t => { t.deepEqual( await eventToMessage({ content: { - body: "testing **special** ~~things~~ which _should_ *not* `trigger` @any ", + body: "testing **special** ~~things~~ which _should_ *not* `trigger` @any , except strikethrough", msgtype: "m.text" }, event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", @@ -85,7 +85,67 @@ test("event2message: any markdown in body is escaped", async t => { messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "testing \\*\\*special\\*\\* \\~\\~things\\~\\~ which \\_should\\_ \\*not\\* \\`trigger\\` @any ", + content: "testing \\*\\*special\\*\\* ~~things~~ which \\_should\\_ \\*not\\* \\`trigger\\` @any , except strikethrough", + avatar_url: undefined + }] + } + ) +}) + +test("event2message: links in formatted body are not broken", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "kyuugryphon I wonder what the midjourney text description of this photo is https://upload.wikimedia.org/wikipedia/commons/f/f3/After_gay_pride%2C_rainbow_flags_flying_along_Beach_Street_%2814853144744%29.jpg", + format: "org.matrix.custom.html", + formatted_body: "kyuugryphon I wonder what the midjourney text description of this photo is https://upload.wikimedia.org/wikipedia/commons/f/f3/After_gay_pride%2C_rainbow_flags_flying_along_Beach_Street_%2814853144744%29.jpg" + }, + origin_server_ts: 1693739630700, + unsigned: { + age: 39, + transaction_id: "m1693739630587.160" + }, + event_id: "$zANQGOdnHKZj48lrajojsejH86KNYST26imgb2Sw1Jg", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "<@111604486476181504> I wonder what the midjourney text description of this photo is https://upload.wikimedia.org/wikipedia/commons/f/f3/After_gay_pride%2C_rainbow_flags_flying_along_Beach_Street_%2814853144744%29.jpg", + avatar_url: undefined + }] + } + ) +}) + +test("event2message: links in plaintext body are not broken", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "I wonder what the midjourney text description of this photo is https://upload.wikimedia.org/wikipedia/commons/f/f3/After_gay_pride%2C_rainbow_flags_flying_along_Beach_Street_%2814853144744%29.jpg", + }, + origin_server_ts: 1693739630700, + unsigned: { + age: 39, + transaction_id: "m1693739630587.160" + }, + event_id: "$zANQGOdnHKZj48lrajojsejH86KNYST26imgb2Sw1Jg", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "I wonder what the midjourney text description of this photo is https://upload.wikimedia.org/wikipedia/commons/f/f3/After_gay_pride%2C_rainbow_flags_flying_along_Beach_Street_%2814853144744%29.jpg", avatar_url: undefined }] } @@ -99,7 +159,7 @@ test("event2message: basic html is converted to markdown", async t => { msgtype: "m.text", body: "wrong body", format: "org.matrix.custom.html", - formatted_body: "this is a test of formatting" + formatted_body: "this is a test of formatting" }, event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", origin_server_ts: 1688301929913, @@ -115,7 +175,7 @@ test("event2message: basic html is converted to markdown", async t => { messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "this **is** a _**test** __of___ ~~formatting~~", + content: "this **is** a _**test** __of___ ~~_formatting_~~", avatar_url: undefined }] } @@ -449,7 +509,7 @@ test("event2message: m.emote plaintext works", async t => { messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "\\* cadence [they] tests an m.emote message", + content: "\\* cadence \\[they\\] tests an m.emote message", avatar_url: undefined }] } From 39458bd2bf09a79ea8e8fa8783d853ec071c2f11 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 4 Sep 2023 01:38:30 +1200 Subject: [PATCH 6/6] allow migrating from old to new bridge --- d2m/actions/create-space.js | 10 +- d2m/discord-client.js | 8 +- d2m/discord-packets.js | 51 +++++----- d2m/event-dispatcher.js | 20 ++-- index.js | 2 +- matrix/mreq.js | 19 ++++ package-lock.json | 6 ++ package.json | 1 + scripts/capture-message-update-events.js | 4 +- scripts/migrate-from-old-bridge.js | 124 +++++++++++++++++++++++ scripts/save-channel-names-to-db.js | 2 +- test/data.js | 5 +- types.d.ts | 4 + 13 files changed, 214 insertions(+), 42 deletions(-) create mode 100644 scripts/migrate-from-old-bridge.js diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index 2c86f75..9802d48 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -78,7 +78,10 @@ async function syncSpace(guildID) { const guildKState = await guildToKState(guild) - if (!spaceID) return + if (!spaceID) { + const spaceID = await createSpace(guild, guildKState) + return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here. + } console.log(`[space sync] to matrix: ${guild.name}`) @@ -123,7 +126,10 @@ async function syncSpaceFully(guildID) { const guildKState = await guildToKState(guild) - if (!spaceID) return + if (!spaceID) { + const spaceID = await createSpace(guild, guildKState) + return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here. + } console.log(`[space sync] to matrix: ${guild.name}`) diff --git a/d2m/discord-client.js b/d2m/discord-client.js index 5e90d85..b1a1e81 100644 --- a/d2m/discord-client.js +++ b/d2m/discord-client.js @@ -12,9 +12,9 @@ const discordPackets = sync.require("./discord-packets") class DiscordClient { /** * @param {string} discordToken - * @param {boolean} listen whether to set up the event listeners for OOYE to operate + * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate */ - constructor(discordToken, listen = true) { + constructor(discordToken, listen = "full") { this.discordToken = discordToken this.snow = new SnowTransfer(discordToken) this.cloud = new CloudStorm(discordToken, { @@ -44,8 +44,8 @@ class DiscordClient { this.guilds = new Map() /** @type {Map>} */ this.guildChannelMap = new Map() - if (listen) { - this.cloud.on("event", message => discordPackets.onPacket(this, message)) + if (listen !== "no") { + this.cloud.on("event", message => discordPackets.onPacket(this, message, listen)) } this.cloud.on("error", console.error) } diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index b2fb643..1649e68 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -10,8 +10,9 @@ const utils = { /** * @param {import("./discord-client")} client * @param {import("cloudstorm").IGatewayMessage} message + * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate */ - async onPacket(client, message) { + async onPacket(client, message, listen) { // requiring this later so that the client is already constructed by the time event-dispatcher is loaded /** @type {typeof import("./event-dispatcher")} */ const eventDispatcher = sync.require("./event-dispatcher") @@ -41,7 +42,9 @@ const utils = { arr.push(thread.id) client.channels.set(thread.id, thread) } - eventDispatcher.checkMissedMessages(client, message.d) + if (listen === "full") { + eventDispatcher.checkMissedMessages(client, message.d) + } } else if (message.t === "GUILD_UPDATE") { const guild = client.guilds.get(message.d.id) @@ -90,35 +93,37 @@ const utils = { } // Event dispatcher for OOYE bridge operations - try { - if (message.t === "GUILD_UPDATE") { - await eventDispatcher.onGuildUpdate(client, message.d) + if (listen === "full") { + try { + if (message.t === "GUILD_UPDATE") { + await eventDispatcher.onGuildUpdate(client, message.d) - } else if (message.t === "CHANNEL_UPDATE") { - await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false) + } else if (message.t === "CHANNEL_UPDATE") { + await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false) - } else if (message.t === "THREAD_CREATE") { - // @ts-ignore - await eventDispatcher.onThreadCreate(client, message.d) + } else if (message.t === "THREAD_CREATE") { + // @ts-ignore + await eventDispatcher.onThreadCreate(client, message.d) - } else if (message.t === "THREAD_UPDATE") { - await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true) + } else if (message.t === "THREAD_UPDATE") { + await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true) - } else if (message.t === "MESSAGE_CREATE") { - await eventDispatcher.onMessageCreate(client, message.d) + } else if (message.t === "MESSAGE_CREATE") { + await eventDispatcher.onMessageCreate(client, message.d) - } else if (message.t === "MESSAGE_UPDATE") { - await eventDispatcher.onMessageUpdate(client, message.d) + } else if (message.t === "MESSAGE_UPDATE") { + await eventDispatcher.onMessageUpdate(client, message.d) - } else if (message.t === "MESSAGE_DELETE") { - await eventDispatcher.onMessageDelete(client, message.d) + } else if (message.t === "MESSAGE_DELETE") { + await eventDispatcher.onMessageDelete(client, message.d) - } else if (message.t === "MESSAGE_REACTION_ADD") { - await eventDispatcher.onReactionAdd(client, message.d) + } else if (message.t === "MESSAGE_REACTION_ADD") { + await eventDispatcher.onReactionAdd(client, message.d) + } + } catch (e) { + // Let OOYE try to handle errors too + eventDispatcher.onError(client, e, message) } - } catch (e) { - // Let OOYE try to handle errors too - eventDispatcher.onError(client, e, message) } } } diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index c73b95b..f20d1c7 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -23,10 +23,6 @@ const discordCommandHandler = sync.require("./discord-command-handler") let lastReportedEvent = 0 -function isGuildAllowed(guildID) { - return ["112760669178241024", "497159726455455754", "1100319549670301727"].includes(guildID) -} - // Grab Discord events we care about for the bridge, check them, and pass them on module.exports = { @@ -93,12 +89,22 @@ module.exports = { if (latestWasBridged) continue /** More recent messages come first. */ - console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) - const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50}) + // console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) + let messages + try { + messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50}) + } catch (e) { + if (e.message === `{"message": "Missing Access", "code": 50001}`) { // pathetic error handling from SnowTransfer + console.log(`[check missed messages] no permissions to look back in channel ${channel.name} (${channel.id})`) + continue // Sucks. + } else { + throw e // Sucks more. + } + } let latestBridgedMessageIndex = messages.findIndex(m => { return prepared.get(m.id) }) - console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`) + // console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`) if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date. for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) { const simulatedGatewayDispatchData = { diff --git a/index.js b/index.js index 447e944..26f2783 100644 --- a/index.js +++ b/index.js @@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db}) const DiscordClient = require("./d2m/discord-client") -const discord = new DiscordClient(config.discordToken, true) +const discord = new DiscordClient(config.discordToken, "full") passthrough.discord = discord const as = require("./m2d/appservice") diff --git a/matrix/mreq.js b/matrix/mreq.js index df34d91..d5ffeba 100644 --- a/matrix/mreq.js +++ b/matrix/mreq.js @@ -43,5 +43,24 @@ async function mreq(method, url, body, extra = {}) { return root } +/** + * JavaScript doesn't have Racket-like parameters with dynamic scoping, so + * do NOT do anything else at the same time as this. + * @template T + * @param {string} token + * @param {(...arg: any[]) => Promise} callback + * @returns {Promise} + */ +async function withAccessToken(token, callback) { + const prevToken = reg.as_token + reg.as_token = token + try { + return await callback() + } finally { + reg.as_token = prevToken + } +} + module.exports.MatrixServerError = MatrixServerError module.exports.mreq = mreq +module.exports.withAccessToken = withAccessToken diff --git a/package-lock.json b/package-lock.json index dfd21ff..6549c67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@chriscdn/promise-semaphore": "^2.0.1", "better-sqlite3": "^8.3.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.8.0", @@ -51,6 +52,11 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@chriscdn/promise-semaphore": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.1.tgz", + "integrity": "sha512-C0Ku5DNZFbafbSRXagidIaRgzhgGmSHk4aAgPpmmHEostazBiSaMryovC/Aix3vRLNuaeGDKN/DHoNECmMD6jg==" + }, "node_modules/@cloudcmd/stub": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@cloudcmd/stub/-/stub-4.0.1.tgz", diff --git a/package.json b/package.json index 238b9ae..5c99b51 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "author": "Cadence, PapiOphidian", "license": "MIT", "dependencies": { + "@chriscdn/promise-semaphore": "^2.0.1", "better-sqlite3": "^8.3.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.8.0", diff --git a/scripts/capture-message-update-events.js b/scripts/capture-message-update-events.js index 2ff9b49..f345d3f 100644 --- a/scripts/capture-message-update-events.js +++ b/scripts/capture-message-update-events.js @@ -23,9 +23,9 @@ const sync = new HeatSync({watchFS: false}) Object.assign(passthrough, {config, sync}) -const DiscordClient = require("../d2m/discord-client", false) +const DiscordClient = require("../d2m/discord-client") -const discord = new DiscordClient(config.discordToken, false) +const discord = new DiscordClient(config.discordToken, "no") passthrough.discord = discord ;(async () => { diff --git a/scripts/migrate-from-old-bridge.js b/scripts/migrate-from-old-bridge.js new file mode 100644 index 0000000..4504aea --- /dev/null +++ b/scripts/migrate-from-old-bridge.js @@ -0,0 +1,124 @@ +// @ts-check + +const assert = require("assert").strict +/** @type {any} */ // @ts-ignore bad types from semaphore +const Semaphore = require("@chriscdn/promise-semaphore") +const sqlite = require("better-sqlite3") +const HeatSync = require("heatsync") + +const config = require("../config") +const passthrough = require("../passthrough") + +const sync = new HeatSync({watchFS: false}) + +/** @type {import("../matrix/read-registration")} */ +const reg = sync.require("../matrix/read-registration") +assert(reg.old_bridge) +const oldAT = reg.old_bridge.as_token +const newAT = reg.as_token + +const oldDB = new sqlite(reg.old_bridge.database) +const db = new sqlite("db/ooye.db") + +db.exec(`CREATE TABLE IF NOT EXISTS migration ( + discord_channel TEXT NOT NULL, + migrated INTEGER NOT NULL, + PRIMARY KEY("discord_channel") +) WITHOUT ROWID;`) + +Object.assign(passthrough, {config, sync, db}) + +const DiscordClient = require("../d2m/discord-client") +const discord = new DiscordClient(config.discordToken, "half") +passthrough.discord = discord + +/** @type {import("../d2m/actions/create-space")} */ +const createSpace = sync.require("../d2m/actions/create-space") +/** @type {import("../d2m/actions/create-room")} */ +const createRoom = sync.require("../d2m/actions/create-room") +/** @type {import("../matrix/mreq")} */ +const mreq = sync.require("../matrix/mreq") +/** @type {import("../matrix/api")} */ +const api = sync.require("../matrix/api") + +const sema = new Semaphore() + +;(async () => { + await discord.cloud.connect() + console.log("Discord gateway started") + + discord.cloud.on("event", event => onPacket(discord, event)) +})() + +/** @param {DiscordClient} discord */ +function onPacket(discord, event) { + if (event.t === "GUILD_CREATE") { + const guild = event.d + if (["1100319549670301727", "112760669178241024", "497159726455455754"].includes(guild.id)) return + sema.request(() => migrateGuild(guild)) + } +} + +const newBridgeMxid = `@${reg.sender_localpart}:${reg.ooye.server_name}` + +/** @param {import("discord-api-types/v10").GatewayGuildCreateDispatchData} guild */ +async function migrateGuild(guild) { + console.log(`START MIGRATION of ${guild.name} (${guild.id})`) + + // Step 1: Create a new space for the guild (createSpace) + const spaceID = await createSpace.syncSpace(guild.id) + + let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id) + const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all() + oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel)) + console.log("Found these rooms which can be migrated:") + console.log(oldRooms) + + for (const row of oldRooms) { + const roomID = row.matrix_id + const channel = discord.channels.get(row.discord_channel) + assert(channel) + + // Step 2: (Using old bridge access token) Join the new bridge to the old rooms and give it PL 100 + console.log(`-- Joining channel ${channel.name}...`) + await mreq.withAccessToken(oldAT, async () => { + try { + await api.inviteToRoom(roomID, newBridgeMxid) + } catch (e) { + if (e.message.includes("is already in the room")) { + // Great! + } else { + throw e + } + } + await api.setUserPower(roomID, newBridgeMxid, 100) + }) + await api.joinRoom(roomID) + + // Step 3: Remove the old bridge's aliases + console.log(`-- -- Deleting aliases...`) + await mreq.withAccessToken(oldAT, async () => { // have to run as old application service since the AS owns its aliases + const aliases = (await mreq.mreq("GET", `/client/v3/rooms/${roomID}/aliases`)).aliases + for (const alias of aliases) { + if (alias.match(/^#?_?discord/)) { + await mreq.mreq("DELETE", `/client/v3/directory/room/${alias.replace(/#/g, "%23")}`) + } + } + await api.sendState(roomID, "m.room.canonical_alias", "", {}) + }) + + // Step 4: Add old rooms to new database; they are now also the new rooms + db.prepare("REPLACE INTO channel_room (channel_id, room_id, name) VALUES (?, ?, ?)").run(channel.id, row.matrix_id, channel.name) + console.log(`-- -- Added to database`) + + // Step 5: Call syncRoom for each room + await createRoom.syncRoom(row.discord_channel) + console.log(`-- -- Finished syncing`) + + db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id) + } + + // Step 5: Call syncSpace to make sure everything is up to date + await createSpace.syncSpace(guild.id) + console.log(`Finished migrating ${guild.name} to Out Of Your Element`) +} diff --git a/scripts/save-channel-names-to-db.js b/scripts/save-channel-names-to-db.js index 6f5867a..1e5c541 100644 --- a/scripts/save-channel-names-to-db.js +++ b/scripts/save-channel-names-to-db.js @@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db}) const DiscordClient = require("../d2m/discord-client") -const discord = new DiscordClient(config.discordToken, false) +const discord = new DiscordClient(config.discordToken, "no") passthrough.discord = discord ;(async () => { diff --git a/test/data.js b/test/data.js index 98c943a..07415d8 100644 --- a/test/data.js +++ b/test/data.js @@ -25,7 +25,7 @@ module.exports = { "m.room.name/": {name: "main"}, "m.room.topic/": {topic: "#collective-unconscious | https://docs.google.com/document/d/blah/edit | I spread, pipe, and whip because it is my will. :headstone:\n\nChannel ID: 112760669178241024\nGuild ID: 112760669178241024"}, "m.room.guest_access/": {guest_access: "can_join"}, - "m.room.history_visibility/": {history_visibility: "invited"}, + "m.room.history_visibility/": {history_visibility: "shared"}, "m.space.parent/!jjWAGMeQdNrVZSSfvz:cadence.moe": { via: ["cadence.moe"], canonical: true @@ -45,7 +45,8 @@ module.exports = { events: { "m.room.avatar": 0 } - } + }, + "chat.schildi.hide_ui/read_receipts": {hidden: true} } }, guild: { diff --git a/types.d.ts b/types.d.ts index a8c7e68..6aec80c 100644 --- a/types.d.ts +++ b/types.d.ts @@ -21,6 +21,10 @@ export type AppServiceRegistrationConfig = { max_file_size: number server_name: string } + old_bridge?: { + as_token: string + database: string + } } export type WebhookCreds = {