allow migrating from old to new bridge

d2m/actions/create-space.js

@@ -78,7 +78,10 @@ async function syncSpace(guildID) {
 
 	const guildKState = await guildToKState(guild)
 
-	if (!spaceID) return
+	if (!spaceID) {
+		const spaceID = await createSpace(guild, guildKState)
+		return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here.
+	}
 
 	console.log(`[space sync] to matrix: ${guild.name}`)
 
@@ -123,7 +126,10 @@ async function syncSpaceFully(guildID) {
 
 	const guildKState = await guildToKState(guild)
 
-	if (!spaceID) return
+	if (!spaceID) {
+		const spaceID = await createSpace(guild, guildKState)
+		return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here.
+	}
 
 	console.log(`[space sync] to matrix: ${guild.name}`)
 

d2m/discord-client.js

@@ -12,9 +12,9 @@ const discordPackets = sync.require("./discord-packets")
 class DiscordClient {
 	/**
 	 * @param {string} discordToken
-	 * @param {boolean} listen whether to set up the event listeners for OOYE to operate
+	 * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
 	 */
-	constructor(discordToken, listen = true) {
+	constructor(discordToken, listen = "full") {
 		this.discordToken = discordToken
 		this.snow = new SnowTransfer(discordToken)
 		this.cloud = new CloudStorm(discordToken, {
@@ -44,8 +44,8 @@ class DiscordClient {
 		this.guilds = new Map()
 		/** @type {Map<string, Array<string>>} */
 		this.guildChannelMap = new Map()
-		if (listen) {
+		if (listen !== "no") {
-			this.cloud.on("event", message => discordPackets.onPacket(this, message))
+			this.cloud.on("event", message => discordPackets.onPacket(this, message, listen))
 		}
 		this.cloud.on("error", console.error)
 	}

d2m/discord-packets.js

@@ -10,8 +10,9 @@ const utils = {
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {import("cloudstorm").IGatewayMessage} message
+	 * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
 	 */
-	async onPacket(client, message) {
+	async onPacket(client, message, listen) {
 		// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
 		/** @type {typeof import("./event-dispatcher")} */
 		const eventDispatcher = sync.require("./event-dispatcher")
@@ -41,7 +42,9 @@ const utils = {
 				arr.push(thread.id)
 				client.channels.set(thread.id, thread)
 			}
+			if (listen === "full") {
 				eventDispatcher.checkMissedMessages(client, message.d)
+			}
 
 		} else if (message.t === "GUILD_UPDATE") {
 			const guild = client.guilds.get(message.d.id)
@@ -90,6 +93,7 @@ const utils = {
 		}
 
 		// Event dispatcher for OOYE bridge operations
+		if (listen === "full") {
 			try {
 				if (message.t === "GUILD_UPDATE") {
 					await eventDispatcher.onGuildUpdate(client, message.d)
@@ -122,5 +126,6 @@ const utils = {
 			}
 		}
 	}
+}
 
 module.exports = utils

d2m/event-dispatcher.js

@@ -23,10 +23,6 @@ const discordCommandHandler = sync.require("./discord-command-handler")
 
 let lastReportedEvent = 0
 
-function isGuildAllowed(guildID) {
-	return ["112760669178241024", "497159726455455754", "1100319549670301727"].includes(guildID)
-}
-
 // Grab Discord events we care about for the bridge, check them, and pass them on
 
 module.exports = {
@@ -93,12 +89,22 @@ module.exports = {
 			if (latestWasBridged) continue
 
 			/** More recent messages come first. */
-			console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
+			// console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
-			const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50})
+			let messages
+			try {
+				messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50})
+			} catch (e) {
+				if (e.message === `{"message": "Missing Access", "code": 50001}`) { // pathetic error handling from SnowTransfer
+					console.log(`[check missed messages] no permissions to look back in channel ${channel.name} (${channel.id})`)
+					continue // Sucks.
+				} else {
+					throw e // Sucks more.
+				}
+			}
 			let latestBridgedMessageIndex = messages.findIndex(m => {
 				return prepared.get(m.id)
 			})
-			console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
+			// console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
 			if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date.
 			for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) {
 				const simulatedGatewayDispatchData = {

index.js

@@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db})
 
 const DiscordClient = require("./d2m/discord-client")
 
-const discord = new DiscordClient(config.discordToken, true)
+const discord = new DiscordClient(config.discordToken, "full")
 passthrough.discord = discord
 
 const as = require("./m2d/appservice")

matrix/mreq.js

@@ -43,5 +43,24 @@ async function mreq(method, url, body, extra = {}) {
 	return root
 }
 
+/**
+ * JavaScript doesn't have Racket-like parameters with dynamic scoping, so
+ * do NOT do anything else at the same time as this.
+ * @template T
+ * @param {string} token
+ * @param {(...arg: any[]) => Promise<T>} callback
+ * @returns {Promise<T>}
+ */
+async function withAccessToken(token, callback) {
+	const prevToken = reg.as_token
+	reg.as_token = token
+	try {
+		return await callback()
+	} finally {
+		reg.as_token = prevToken
+	}
+}
+
 module.exports.MatrixServerError = MatrixServerError
 module.exports.mreq = mreq
+module.exports.withAccessToken = withAccessToken

package-lock.json

@@ -9,6 +9,7 @@
       "version": "1.0.0",
       "license": "MIT",
       "dependencies": {
+        "@chriscdn/promise-semaphore": "^2.0.1",
         "better-sqlite3": "^8.3.0",
         "chunk-text": "^2.0.1",
         "cloudstorm": "^0.8.0",
@@ -51,6 +52,11 @@
       "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==",
       "dev": true
     },
+    "node_modules/@chriscdn/promise-semaphore": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.1.tgz",
+      "integrity": "sha512-C0Ku5DNZFbafbSRXagidIaRgzhgGmSHk4aAgPpmmHEostazBiSaMryovC/Aix3vRLNuaeGDKN/DHoNECmMD6jg=="
+    },
     "node_modules/@cloudcmd/stub": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/@cloudcmd/stub/-/stub-4.0.1.tgz",

package.json

@@ -15,6 +15,7 @@
   "author": "Cadence, PapiOphidian",
   "license": "MIT",
   "dependencies": {
+    "@chriscdn/promise-semaphore": "^2.0.1",
     "better-sqlite3": "^8.3.0",
     "chunk-text": "^2.0.1",
     "cloudstorm": "^0.8.0",

scripts/capture-message-update-events.js

@@ -23,9 +23,9 @@ const sync = new HeatSync({watchFS: false})
 
 Object.assign(passthrough, {config, sync})
 
-const DiscordClient = require("../d2m/discord-client", false)
+const DiscordClient = require("../d2m/discord-client")
 
-const discord = new DiscordClient(config.discordToken, false)
+const discord = new DiscordClient(config.discordToken, "no")
 passthrough.discord = discord
 
 ;(async () => {

scripts/migrate-from-old-bridge.js

@@ -0,0 +1,124 @@
+// @ts-check
+
+const assert = require("assert").strict
+/** @type {any} */ // @ts-ignore bad types from semaphore
+const Semaphore = require("@chriscdn/promise-semaphore")
+const sqlite = require("better-sqlite3")
+const HeatSync = require("heatsync")
+
+const config = require("../config")
+const passthrough = require("../passthrough")
+
+const sync = new HeatSync({watchFS: false})
+
+/** @type {import("../matrix/read-registration")} */
+const reg = sync.require("../matrix/read-registration")
+assert(reg.old_bridge)
+const oldAT = reg.old_bridge.as_token
+const newAT = reg.as_token
+
+const oldDB = new sqlite(reg.old_bridge.database)
+const db = new sqlite("db/ooye.db")
+
+db.exec(`CREATE TABLE IF NOT EXISTS migration (
+	discord_channel	TEXT NOT NULL,
+	migrated	INTEGER NOT NULL,
+	PRIMARY KEY("discord_channel")
+) WITHOUT ROWID;`)
+
+Object.assign(passthrough, {config, sync, db})
+
+const DiscordClient = require("../d2m/discord-client")
+const discord = new DiscordClient(config.discordToken, "half")
+passthrough.discord = discord
+
+/** @type {import("../d2m/actions/create-space")} */
+const createSpace = sync.require("../d2m/actions/create-space")
+/** @type {import("../d2m/actions/create-room")} */
+const createRoom = sync.require("../d2m/actions/create-room")
+/** @type {import("../matrix/mreq")} */
+const mreq = sync.require("../matrix/mreq")
+/** @type {import("../matrix/api")} */
+const api = sync.require("../matrix/api")
+
+const sema = new Semaphore()
+
+;(async () => {
+	await discord.cloud.connect()
+	console.log("Discord gateway started")
+
+	discord.cloud.on("event", event => onPacket(discord, event))
+})()
+
+/** @param {DiscordClient} discord */
+function onPacket(discord, event) {
+	if (event.t === "GUILD_CREATE") {
+		const guild = event.d
+		if (["1100319549670301727", "112760669178241024", "497159726455455754"].includes(guild.id)) return
+		sema.request(() => migrateGuild(guild))
+	}
+}
+
+const newBridgeMxid = `@${reg.sender_localpart}:${reg.ooye.server_name}`
+
+/** @param {import("discord-api-types/v10").GatewayGuildCreateDispatchData} guild */
+async function migrateGuild(guild) {
+	console.log(`START MIGRATION of ${guild.name} (${guild.id})`)
+
+	// Step 1: Create a new space for the guild (createSpace)
+	const spaceID = await createSpace.syncSpace(guild.id)
+
+	let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id)
+	const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all()
+	oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel))
+	console.log("Found these rooms which can be migrated:")
+	console.log(oldRooms)
+
+	for (const row of oldRooms) {
+		const roomID = row.matrix_id
+		const channel = discord.channels.get(row.discord_channel)
+		assert(channel)
+
+		// Step 2: (Using old bridge access token) Join the new bridge to the old rooms and give it PL 100
+		console.log(`-- Joining channel ${channel.name}...`)
+		await mreq.withAccessToken(oldAT, async () => {
+			try {
+				await api.inviteToRoom(roomID, newBridgeMxid)
+			} catch (e) {
+				if (e.message.includes("is already in the room")) {
+					// Great!
+				} else {
+					throw e
+				}
+			}
+			await api.setUserPower(roomID, newBridgeMxid, 100)
+		})
+		await api.joinRoom(roomID)
+
+		// Step 3: Remove the old bridge's aliases
+		console.log(`-- -- Deleting aliases...`)
+		await mreq.withAccessToken(oldAT, async () => { // have to run as old application service since the AS owns its aliases
+			const aliases = (await mreq.mreq("GET", `/client/v3/rooms/${roomID}/aliases`)).aliases
+			for (const alias of aliases) {
+				if (alias.match(/^#?_?discord/)) {
+					await mreq.mreq("DELETE", `/client/v3/directory/room/${alias.replace(/#/g, "%23")}`)
+				}
+			}
+			await api.sendState(roomID, "m.room.canonical_alias", "", {})
+		})
+
+		// Step 4: Add old rooms to new database; they are now also the new rooms
+		db.prepare("REPLACE INTO channel_room (channel_id, room_id, name) VALUES (?, ?, ?)").run(channel.id, row.matrix_id, channel.name)
+		console.log(`-- -- Added to database`)
+
+		// Step 5: Call syncRoom for each room
+		await createRoom.syncRoom(row.discord_channel)
+		console.log(`-- -- Finished syncing`)
+
+		db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
+	}
+
+	// Step 5: Call syncSpace to make sure everything is up to date
+	await createSpace.syncSpace(guild.id)
+	console.log(`Finished migrating ${guild.name} to Out Of Your Element`)
+}

scripts/save-channel-names-to-db.js

@@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db})
 
 const DiscordClient = require("../d2m/discord-client")
 
-const discord = new DiscordClient(config.discordToken, false)
+const discord = new DiscordClient(config.discordToken, "no")
 passthrough.discord = discord
 
 ;(async () => {

test/data.js

@@ -25,7 +25,7 @@ module.exports = {
 			"m.room.name/": {name: "main"},
 			"m.room.topic/": {topic: "#collective-unconscious | https://docs.google.com/document/d/blah/edit | I spread, pipe, and whip because it is my will. :headstone:\n\nChannel ID: 112760669178241024\nGuild ID: 112760669178241024"},
 			"m.room.guest_access/": {guest_access: "can_join"},
-			"m.room.history_visibility/": {history_visibility: "invited"},
+			"m.room.history_visibility/": {history_visibility: "shared"},
 			"m.space.parent/!jjWAGMeQdNrVZSSfvz:cadence.moe": {
 				via: ["cadence.moe"],
 				canonical: true
@@ -45,7 +45,8 @@ module.exports = {
 				events: {
 					"m.room.avatar": 0
 				}
-			}
+			},
+			"chat.schildi.hide_ui/read_receipts": {hidden: true}
 		}
 	},
 	guild: {

types.d.ts

@@ -21,6 +21,10 @@ export type AppServiceRegistrationConfig = {
 		max_file_size: number
 		server_name: string
 	}
+	old_bridge?: {
+		as_token: string
+		database: string
+	}
 }
 
 export type WebhookCreds = {