From 39458bd2bf09a79ea8e8fa8783d853ec071c2f11 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 4 Sep 2023 01:38:30 +1200 Subject: [PATCH] allow migrating from old to new bridge --- d2m/actions/create-space.js | 10 +- d2m/discord-client.js | 8 +- d2m/discord-packets.js | 51 +++++----- d2m/event-dispatcher.js | 20 ++-- index.js | 2 +- matrix/mreq.js | 19 ++++ package-lock.json | 6 ++ package.json | 1 + scripts/capture-message-update-events.js | 4 +- scripts/migrate-from-old-bridge.js | 124 +++++++++++++++++++++++ scripts/save-channel-names-to-db.js | 2 +- test/data.js | 5 +- types.d.ts | 4 + 13 files changed, 214 insertions(+), 42 deletions(-) create mode 100644 scripts/migrate-from-old-bridge.js diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index 2c86f75..9802d48 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -78,7 +78,10 @@ async function syncSpace(guildID) { const guildKState = await guildToKState(guild) - if (!spaceID) return + if (!spaceID) { + const spaceID = await createSpace(guild, guildKState) + return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here. + } console.log(`[space sync] to matrix: ${guild.name}`) @@ -123,7 +126,10 @@ async function syncSpaceFully(guildID) { const guildKState = await guildToKState(guild) - if (!spaceID) return + if (!spaceID) { + const spaceID = await createSpace(guild, guildKState) + return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here. + } console.log(`[space sync] to matrix: ${guild.name}`) diff --git a/d2m/discord-client.js b/d2m/discord-client.js index 5e90d85..b1a1e81 100644 --- a/d2m/discord-client.js +++ b/d2m/discord-client.js @@ -12,9 +12,9 @@ const discordPackets = sync.require("./discord-packets") class DiscordClient { /** * @param {string} discordToken - * @param {boolean} listen whether to set up the event listeners for OOYE to operate + * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate */ - constructor(discordToken, listen = true) { + constructor(discordToken, listen = "full") { this.discordToken = discordToken this.snow = new SnowTransfer(discordToken) this.cloud = new CloudStorm(discordToken, { @@ -44,8 +44,8 @@ class DiscordClient { this.guilds = new Map() /** @type {Map>} */ this.guildChannelMap = new Map() - if (listen) { - this.cloud.on("event", message => discordPackets.onPacket(this, message)) + if (listen !== "no") { + this.cloud.on("event", message => discordPackets.onPacket(this, message, listen)) } this.cloud.on("error", console.error) } diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index b2fb643..1649e68 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -10,8 +10,9 @@ const utils = { /** * @param {import("./discord-client")} client * @param {import("cloudstorm").IGatewayMessage} message + * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate */ - async onPacket(client, message) { + async onPacket(client, message, listen) { // requiring this later so that the client is already constructed by the time event-dispatcher is loaded /** @type {typeof import("./event-dispatcher")} */ const eventDispatcher = sync.require("./event-dispatcher") @@ -41,7 +42,9 @@ const utils = { arr.push(thread.id) client.channels.set(thread.id, thread) } - eventDispatcher.checkMissedMessages(client, message.d) + if (listen === "full") { + eventDispatcher.checkMissedMessages(client, message.d) + } } else if (message.t === "GUILD_UPDATE") { const guild = client.guilds.get(message.d.id) @@ -90,35 +93,37 @@ const utils = { } // Event dispatcher for OOYE bridge operations - try { - if (message.t === "GUILD_UPDATE") { - await eventDispatcher.onGuildUpdate(client, message.d) + if (listen === "full") { + try { + if (message.t === "GUILD_UPDATE") { + await eventDispatcher.onGuildUpdate(client, message.d) - } else if (message.t === "CHANNEL_UPDATE") { - await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false) + } else if (message.t === "CHANNEL_UPDATE") { + await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false) - } else if (message.t === "THREAD_CREATE") { - // @ts-ignore - await eventDispatcher.onThreadCreate(client, message.d) + } else if (message.t === "THREAD_CREATE") { + // @ts-ignore + await eventDispatcher.onThreadCreate(client, message.d) - } else if (message.t === "THREAD_UPDATE") { - await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true) + } else if (message.t === "THREAD_UPDATE") { + await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true) - } else if (message.t === "MESSAGE_CREATE") { - await eventDispatcher.onMessageCreate(client, message.d) + } else if (message.t === "MESSAGE_CREATE") { + await eventDispatcher.onMessageCreate(client, message.d) - } else if (message.t === "MESSAGE_UPDATE") { - await eventDispatcher.onMessageUpdate(client, message.d) + } else if (message.t === "MESSAGE_UPDATE") { + await eventDispatcher.onMessageUpdate(client, message.d) - } else if (message.t === "MESSAGE_DELETE") { - await eventDispatcher.onMessageDelete(client, message.d) + } else if (message.t === "MESSAGE_DELETE") { + await eventDispatcher.onMessageDelete(client, message.d) - } else if (message.t === "MESSAGE_REACTION_ADD") { - await eventDispatcher.onReactionAdd(client, message.d) + } else if (message.t === "MESSAGE_REACTION_ADD") { + await eventDispatcher.onReactionAdd(client, message.d) + } + } catch (e) { + // Let OOYE try to handle errors too + eventDispatcher.onError(client, e, message) } - } catch (e) { - // Let OOYE try to handle errors too - eventDispatcher.onError(client, e, message) } } } diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index c73b95b..f20d1c7 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -23,10 +23,6 @@ const discordCommandHandler = sync.require("./discord-command-handler") let lastReportedEvent = 0 -function isGuildAllowed(guildID) { - return ["112760669178241024", "497159726455455754", "1100319549670301727"].includes(guildID) -} - // Grab Discord events we care about for the bridge, check them, and pass them on module.exports = { @@ -93,12 +89,22 @@ module.exports = { if (latestWasBridged) continue /** More recent messages come first. */ - console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) - const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50}) + // console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) + let messages + try { + messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50}) + } catch (e) { + if (e.message === `{"message": "Missing Access", "code": 50001}`) { // pathetic error handling from SnowTransfer + console.log(`[check missed messages] no permissions to look back in channel ${channel.name} (${channel.id})`) + continue // Sucks. + } else { + throw e // Sucks more. + } + } let latestBridgedMessageIndex = messages.findIndex(m => { return prepared.get(m.id) }) - console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`) + // console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`) if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date. for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) { const simulatedGatewayDispatchData = { diff --git a/index.js b/index.js index 447e944..26f2783 100644 --- a/index.js +++ b/index.js @@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db}) const DiscordClient = require("./d2m/discord-client") -const discord = new DiscordClient(config.discordToken, true) +const discord = new DiscordClient(config.discordToken, "full") passthrough.discord = discord const as = require("./m2d/appservice") diff --git a/matrix/mreq.js b/matrix/mreq.js index df34d91..d5ffeba 100644 --- a/matrix/mreq.js +++ b/matrix/mreq.js @@ -43,5 +43,24 @@ async function mreq(method, url, body, extra = {}) { return root } +/** + * JavaScript doesn't have Racket-like parameters with dynamic scoping, so + * do NOT do anything else at the same time as this. + * @template T + * @param {string} token + * @param {(...arg: any[]) => Promise} callback + * @returns {Promise} + */ +async function withAccessToken(token, callback) { + const prevToken = reg.as_token + reg.as_token = token + try { + return await callback() + } finally { + reg.as_token = prevToken + } +} + module.exports.MatrixServerError = MatrixServerError module.exports.mreq = mreq +module.exports.withAccessToken = withAccessToken diff --git a/package-lock.json b/package-lock.json index dfd21ff..6549c67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@chriscdn/promise-semaphore": "^2.0.1", "better-sqlite3": "^8.3.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.8.0", @@ -51,6 +52,11 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@chriscdn/promise-semaphore": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.1.tgz", + "integrity": "sha512-C0Ku5DNZFbafbSRXagidIaRgzhgGmSHk4aAgPpmmHEostazBiSaMryovC/Aix3vRLNuaeGDKN/DHoNECmMD6jg==" + }, "node_modules/@cloudcmd/stub": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@cloudcmd/stub/-/stub-4.0.1.tgz", diff --git a/package.json b/package.json index 238b9ae..5c99b51 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "author": "Cadence, PapiOphidian", "license": "MIT", "dependencies": { + "@chriscdn/promise-semaphore": "^2.0.1", "better-sqlite3": "^8.3.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.8.0", diff --git a/scripts/capture-message-update-events.js b/scripts/capture-message-update-events.js index 2ff9b49..f345d3f 100644 --- a/scripts/capture-message-update-events.js +++ b/scripts/capture-message-update-events.js @@ -23,9 +23,9 @@ const sync = new HeatSync({watchFS: false}) Object.assign(passthrough, {config, sync}) -const DiscordClient = require("../d2m/discord-client", false) +const DiscordClient = require("../d2m/discord-client") -const discord = new DiscordClient(config.discordToken, false) +const discord = new DiscordClient(config.discordToken, "no") passthrough.discord = discord ;(async () => { diff --git a/scripts/migrate-from-old-bridge.js b/scripts/migrate-from-old-bridge.js new file mode 100644 index 0000000..4504aea --- /dev/null +++ b/scripts/migrate-from-old-bridge.js @@ -0,0 +1,124 @@ +// @ts-check + +const assert = require("assert").strict +/** @type {any} */ // @ts-ignore bad types from semaphore +const Semaphore = require("@chriscdn/promise-semaphore") +const sqlite = require("better-sqlite3") +const HeatSync = require("heatsync") + +const config = require("../config") +const passthrough = require("../passthrough") + +const sync = new HeatSync({watchFS: false}) + +/** @type {import("../matrix/read-registration")} */ +const reg = sync.require("../matrix/read-registration") +assert(reg.old_bridge) +const oldAT = reg.old_bridge.as_token +const newAT = reg.as_token + +const oldDB = new sqlite(reg.old_bridge.database) +const db = new sqlite("db/ooye.db") + +db.exec(`CREATE TABLE IF NOT EXISTS migration ( + discord_channel TEXT NOT NULL, + migrated INTEGER NOT NULL, + PRIMARY KEY("discord_channel") +) WITHOUT ROWID;`) + +Object.assign(passthrough, {config, sync, db}) + +const DiscordClient = require("../d2m/discord-client") +const discord = new DiscordClient(config.discordToken, "half") +passthrough.discord = discord + +/** @type {import("../d2m/actions/create-space")} */ +const createSpace = sync.require("../d2m/actions/create-space") +/** @type {import("../d2m/actions/create-room")} */ +const createRoom = sync.require("../d2m/actions/create-room") +/** @type {import("../matrix/mreq")} */ +const mreq = sync.require("../matrix/mreq") +/** @type {import("../matrix/api")} */ +const api = sync.require("../matrix/api") + +const sema = new Semaphore() + +;(async () => { + await discord.cloud.connect() + console.log("Discord gateway started") + + discord.cloud.on("event", event => onPacket(discord, event)) +})() + +/** @param {DiscordClient} discord */ +function onPacket(discord, event) { + if (event.t === "GUILD_CREATE") { + const guild = event.d + if (["1100319549670301727", "112760669178241024", "497159726455455754"].includes(guild.id)) return + sema.request(() => migrateGuild(guild)) + } +} + +const newBridgeMxid = `@${reg.sender_localpart}:${reg.ooye.server_name}` + +/** @param {import("discord-api-types/v10").GatewayGuildCreateDispatchData} guild */ +async function migrateGuild(guild) { + console.log(`START MIGRATION of ${guild.name} (${guild.id})`) + + // Step 1: Create a new space for the guild (createSpace) + const spaceID = await createSpace.syncSpace(guild.id) + + let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id) + const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all() + oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel)) + console.log("Found these rooms which can be migrated:") + console.log(oldRooms) + + for (const row of oldRooms) { + const roomID = row.matrix_id + const channel = discord.channels.get(row.discord_channel) + assert(channel) + + // Step 2: (Using old bridge access token) Join the new bridge to the old rooms and give it PL 100 + console.log(`-- Joining channel ${channel.name}...`) + await mreq.withAccessToken(oldAT, async () => { + try { + await api.inviteToRoom(roomID, newBridgeMxid) + } catch (e) { + if (e.message.includes("is already in the room")) { + // Great! + } else { + throw e + } + } + await api.setUserPower(roomID, newBridgeMxid, 100) + }) + await api.joinRoom(roomID) + + // Step 3: Remove the old bridge's aliases + console.log(`-- -- Deleting aliases...`) + await mreq.withAccessToken(oldAT, async () => { // have to run as old application service since the AS owns its aliases + const aliases = (await mreq.mreq("GET", `/client/v3/rooms/${roomID}/aliases`)).aliases + for (const alias of aliases) { + if (alias.match(/^#?_?discord/)) { + await mreq.mreq("DELETE", `/client/v3/directory/room/${alias.replace(/#/g, "%23")}`) + } + } + await api.sendState(roomID, "m.room.canonical_alias", "", {}) + }) + + // Step 4: Add old rooms to new database; they are now also the new rooms + db.prepare("REPLACE INTO channel_room (channel_id, room_id, name) VALUES (?, ?, ?)").run(channel.id, row.matrix_id, channel.name) + console.log(`-- -- Added to database`) + + // Step 5: Call syncRoom for each room + await createRoom.syncRoom(row.discord_channel) + console.log(`-- -- Finished syncing`) + + db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id) + } + + // Step 5: Call syncSpace to make sure everything is up to date + await createSpace.syncSpace(guild.id) + console.log(`Finished migrating ${guild.name} to Out Of Your Element`) +} diff --git a/scripts/save-channel-names-to-db.js b/scripts/save-channel-names-to-db.js index 6f5867a..1e5c541 100644 --- a/scripts/save-channel-names-to-db.js +++ b/scripts/save-channel-names-to-db.js @@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db}) const DiscordClient = require("../d2m/discord-client") -const discord = new DiscordClient(config.discordToken, false) +const discord = new DiscordClient(config.discordToken, "no") passthrough.discord = discord ;(async () => { diff --git a/test/data.js b/test/data.js index 98c943a..07415d8 100644 --- a/test/data.js +++ b/test/data.js @@ -25,7 +25,7 @@ module.exports = { "m.room.name/": {name: "main"}, "m.room.topic/": {topic: "#collective-unconscious | https://docs.google.com/document/d/blah/edit | I spread, pipe, and whip because it is my will. :headstone:\n\nChannel ID: 112760669178241024\nGuild ID: 112760669178241024"}, "m.room.guest_access/": {guest_access: "can_join"}, - "m.room.history_visibility/": {history_visibility: "invited"}, + "m.room.history_visibility/": {history_visibility: "shared"}, "m.space.parent/!jjWAGMeQdNrVZSSfvz:cadence.moe": { via: ["cadence.moe"], canonical: true @@ -45,7 +45,8 @@ module.exports = { events: { "m.room.avatar": 0 } - } + }, + "chat.schildi.hide_ui/read_receipts": {hidden: true} } }, guild: { diff --git a/types.d.ts b/types.d.ts index a8c7e68..6aec80c 100644 --- a/types.d.ts +++ b/types.d.ts @@ -21,6 +21,10 @@ export type AppServiceRegistrationConfig = { max_file_size: number server_name: string } + old_bridge?: { + as_token: string + database: string + } } export type WebhookCreds = {