Wait for homeserver when bridge is first started

docs/developer-orientation.md

@@ -113,6 +113,7 @@ Total transitive production dependencies: 144
 * (0) ansi-colors: Helps with interactive prompting for the initial setup, and it's already pulled in by enquirer.
 * (1) chunk-text: It does what I want.
 * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
+* (0) denque: Much faster than using arrays as a queue.
 * (0) discord-api-types: Bitfields needed at runtime and types needed for development.
 * (0) domino: DOM implementation that's already pulled in by turndown.
 * (2) enquirer: Interactive prompting for the initial setup rather than forcing users to edit YAML non-interactively.

package-lock.json

@@ -24,7 +24,8 @@
         "ansi-colors": "^4.1.3",
         "better-sqlite3": "^12.2.0",
         "chunk-text": "^2.0.1",
-        "cloudstorm": "^0.17.1",
+        "cloudstorm": "^0.19.0",
+        "denque": "^2.1.0",
         "discord-api-types": "^0.38.38",
         "domino": "^2.1.6",
         "enquirer": "^2.4.1",
@@ -37,7 +38,7 @@
         "mime-types": "^2.1.35",
         "prettier-bytes": "^1.0.4",
         "sharp": "^0.34.5",
-        "snowtransfer": "^0.17.5",
+        "snowtransfer": "^0.19.0",
         "try-to-catch": "^4.0.5",
         "uqr": "^0.1.2",
         "xxhash-wasm": "^1.0.2",
@@ -1316,13 +1317,13 @@
       }
     },
     "node_modules/cloudstorm": {
-      "version": "0.17.1",
+      "version": "0.19.0",
-      "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.17.1.tgz",
+      "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.19.0.tgz",
-      "integrity": "sha512-LYUwzHagRYRd93XocOqi+HCHdzPYI9cW7Yf7pYqinxgG+Qka1OiqBKWTCcLiEuiqXaOV30kr8c6aZ/c1QcDP4Q==",
+      "integrity": "sha512-mjuP5o2nS/CgSljajh08KjN6jYMfNBBttSPHc1YUN74f7OkmhUhv8QE8JFDXomxCfdnLNJcqBQtjRLrs8nkfVQ==",
       "license": "MIT",
       "dependencies": {
-        "discord-api-types": "^0.38.47",
+        "discord-api-types": "^0.38.49",
-        "snowtransfer": "^0.17.5"
+        "snowtransfer": "^0.19.0"
       },
       "engines": {
         "node": ">=22.0.0"
@@ -1457,6 +1458,15 @@
       "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==",
       "license": "MIT"
     },
+    "node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "node_modules/destr": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz",
@@ -1473,9 +1483,9 @@
       }
     },
     "node_modules/discord-api-types": {
-      "version": "0.38.47",
+      "version": "0.38.49",
-      "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.47.tgz",
+      "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.49.tgz",
-      "integrity": "sha512-XgXQodHQBAE6kfD7kMvVo30863iHX1LHSqNq6MGUTDwIFCCvHva13+rwxyxVXDqudyApMNAd32PGjgVETi5rjA==",
+      "integrity": "sha512-XnqcWmnFZFAE8ZM8SHAw9DIV8D3Or00rMQ8iQLotrEA2PmXhl+ykaf6L6q4l474hrSUH1JaYcv+iOMRWp2p6Tg==",
       "license": "MIT",
       "workspaces": [
         "scripts/actions/documentation"
@@ -2579,12 +2589,12 @@
       }
     },
     "node_modules/snowtransfer": {
-      "version": "0.17.7",
+      "version": "0.19.0",
-      "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.17.7.tgz",
+      "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.19.0.tgz",
-      "integrity": "sha512-scbOjYezo1Ycfk21atCEkeXIISTT7R7JTHCdiZ/7m7k4XbSb6o5q8Mu2fev5IqFpNyqIVjA0d/MZQ+eP/gtwfg==",
+      "integrity": "sha512-Vebj4FCdpUsEKAzqC1p50ol5o09CLnrffvBCpTv0FNFefvXPXwogxR8kr4zJApIRt7oWQVZZVkxAq4XnGlICjQ==",
       "license": "MIT",
       "dependencies": {
-        "discord-api-types": "^0.38.47"
+        "discord-api-types": "^0.38.49"
       },
       "engines": {
         "node": ">=22.0.0"

package.json

@@ -33,7 +33,8 @@
     "ansi-colors": "^4.1.3",
     "better-sqlite3": "^12.2.0",
     "chunk-text": "^2.0.1",
-    "cloudstorm": "^0.17.1",
+    "cloudstorm": "^0.19.0",
+    "denque": "^2.1.0",
     "discord-api-types": "^0.38.38",
     "domino": "^2.1.6",
     "enquirer": "^2.4.1",
@@ -46,7 +47,7 @@
     "mime-types": "^2.1.35",
     "prettier-bytes": "^1.0.4",
     "sharp": "^0.34.5",
-    "snowtransfer": "^0.17.5",
+    "snowtransfer": "^0.19.0",
     "try-to-catch": "^4.0.5",
     "uqr": "^0.1.2",
     "xxhash-wasm": "^1.0.2",

scripts/setup.js

@@ -359,7 +359,7 @@ function defineEchoHandler() {
 	console.log("✅ Emojis are ready...")
 
 	// set profile data on discord...
-	await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + avatarBuffer.toString("base64")})
+	await discord.snow.user.editSelf({avatar: "data:image/png;base64," + avatarBuffer.toString("base64")})
 	console.log("✅ Discord profile updated...")
 
 	// set profile data on homeserver...

src/d2m/actions/create-space.js

@@ -8,6 +8,8 @@ const {reg} = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
+/** @type {import("../../matrix/mreq")} */
+const mreq = sync.require("../../matrix/mreq")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
@@ -237,8 +239,12 @@ async function syncSpaceExpressions(data, checkBeforeSync) {
 			try {
 				existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey)
 			} catch (e) {
+				if (e instanceof mreq.MatrixServerError && e.httpStatus < 400) {
 					// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
 					existing = fn([], guild)
+				} else {
+					throw e
+				}
 			}
 			if (isDeepStrictEqual(existing, content)) return
 		}

src/d2m/discord-client.js

@@ -14,6 +14,8 @@ const {sync} = passthrough
 /** @type {import("./discord-packets")} */
 const discordPackets = sync.require("./discord-packets")
 
+const CONNECTION_DEBUG = false
+
 class DiscordClient {
 	/**
 	 * @param {string} discordToken
@@ -59,6 +61,7 @@ class DiscordClient {
 			})
 		}
 
+		if (CONNECTION_DEBUG) {
 			const addEventLogger = (eventName, logName) => {
 				this.cloud.on(eventName, (...args) => {
 					const d = new Date().toISOString().slice(0, 19)
@@ -69,6 +72,7 @@ class DiscordClient {
 			addEventLogger("disconnected", "Disconnected")
 			addEventLogger("ready", "Ready")
 		}
+	}
 }
 
 module.exports = DiscordClient

src/d2m/discord-packets.js

@@ -1,18 +1,21 @@
 // @ts-check
 
-// Discord library internals type beat
+const assert = require("assert")
-
+const {scheduler} = require("timers/promises")
-const DiscordTypes = require("discord-api-types/v10")
 const passthrough = require("../passthrough")
-const {sync, db} = passthrough
+const {sync} = passthrough
 
-const utils = {
+/** @type {import("../matrix/homeserver-status")} */
-	/**
+const homeserverStatus = sync.require("../matrix/homeserver-status")
+
+let checkedHomeserver = false
+
+/**
  * @param {import("./discord-client")} client
  * @param {import("cloudstorm").IGatewayMessage} message
  * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
  */
-	async onPacket(client, message, listen) {
+async function onPacket(client, message, listen) {
 	// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
 	/** @type {typeof import("./event-dispatcher")} */
 	const eventDispatcher = sync.require("./event-dispatcher")
@@ -48,6 +51,31 @@ const utils = {
 
 		if (listen === "full") {
 			try {
+				/*
+					Info about guilds is populated one guild at a time.
+					For m->d bridging to work, the guild needs to be populated, so we need to have GUILD_CREATE for the guild.
+					If we ping the homeserver, it will send us any pending events, so we need to wait for all GUILD_CREATES before we ping.
+					We must attempt a ping because we don't want to try sending missed d->m messages to an offline homeserver.
+					This delay can be removed if ONE of the following is done:
+						1. m->d can queue incoming events until their guild exists in memory
+						2. d->m missed messages can have their errors handled and added to queue, rather than pinging first
+				*/
+				let isMainCharacter = false
+				if (!checkedHomeserver) {
+					checkedHomeserver = true
+					isMainCharacter = true
+					console.log("Warming up guilds~")
+				}
+				await scheduler.wait(5000)
+				if (isMainCharacter) {
+					checkedHomeserver = true
+					process.stdout.write("Connecting to homeserver... ")
+					await homeserverStatus.homeserverStatus.waitForOnline(true)
+					console.log("ok.\nReplaying past events. Welcome to Out Of Your Element.")
+				} else {
+					await homeserverStatus.homeserverStatus.waitForOnline(false)
+				}
+
 				await eventDispatcher.checkMissedExpressions(message.d)
 				await eventDispatcher.checkMissedMessages(client, message.d)
 				await eventDispatcher.checkMissedPins(client, message.d)
@@ -152,6 +180,28 @@ const utils = {
 
 	// Event dispatcher for OOYE bridge operations
 	if (listen === "full" && message.t) {
+		const alwaysRealTimeEvents = ["PRESENCE_UPDATE"]
+		if (alwaysRealTimeEvents.includes(message.t) || homeserverStatus.homeserverStatus.isRealTime()) {
+			dispatchPacketToBridge(client, message)
+		} else {
+			homeserverStatus.homeserverStatus.queuePacket(message)
+		}
+	}
+}
+
+/**
+ * @param {import("./discord-client")} client
+ * @param {import("cloudstorm").IGatewayMessage} message
+ */
+async function dispatchPacketToBridge(client, message) {
+	// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
+	/** @type {typeof import("./event-dispatcher")} */
+	const eventDispatcher = sync.require("./event-dispatcher")
+	/** @type {import("../discord/register-interactions")} */
+	const interactions = sync.require("../discord/register-interactions")
+
+	assert(message.t) // checked above
+
 	try {
 		if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
 			await eventDispatcher.onSomeReactionsRemoved(client, message.d)
@@ -166,8 +216,7 @@ const utils = {
 		// Let OOYE try to handle errors too
 		await eventDispatcher.onError(client, e, message)
 	}
-		}
-	}
 }
 
-module.exports = utils
+module.exports.onPacket = onPacket
+module.exports.dispatchPacketToBridge = dispatchPacketToBridge

src/d2m/event-dispatcher.js

@@ -60,7 +60,7 @@ module.exports = {
 
 		matrixEventDispatcher.printError(gatewayMessage.t, "Discord", e, gatewayMessage)
 
-		const channelID = gatewayMessage.d["channel_id"]
+		const channelID = gatewayMessage.d?.["channel_id"]
 		if (!channelID) return
 		const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
 		if (!roomID) return

src/discord/register-interactions.js

@@ -16,7 +16,7 @@ const ping = sync.require("./interactions/ping.js")
 // User must have EVERY permission in default_member_permissions to be able to use the command
 
 function registerInteractions() {
-	discord.snow.interaction.bulkOverwriteApplicationCommands(id, [{
+	discord.snow.interaction.editApplicationCommands(id, [{
 		name: "Matrix info",
 		contexts: [DiscordTypes.InteractionContextType.Guild],
 		type: DiscordTypes.ApplicationCommandType.Message,

src/m2d/actions/emoji-sheet.js

@@ -23,7 +23,7 @@ async function getAndConvertEmoji(mxc) {
 	const res = await api.getMedia(mxc, {signal: abortController.signal})
 	if (res.status !== 200) {
 		const root = await res.json()
-		throw new mreq.MatrixServerError(root, {mxc})
+		throw new mreq.MatrixServerError(root, res.status, {mxc})
 	}
 	const readable = stream.Readable.fromWeb(res.body)
 	return emojiSheetConverter.convertImageStream(readable, () => {

src/m2d/actions/sticker.js

@@ -22,7 +22,7 @@ async function getAndResizeSticker(mxc) {
 	const res = await api.getMedia(mxc)
 	if (res.status !== 200) {
 		const root = await res.json()
-		throw new mreq.MatrixServerError(root, {mxc})
+		throw new mreq.MatrixServerError(root, res.status, {mxc})
 	}
 
 	const streamIn = Readable.fromWeb(res.body)

src/m2d/actions/update-pins.js

@@ -16,9 +16,9 @@ async function updatePins(pins, prev) {
 			.select("reference_channel_id", "message_id").where({event_id}).and("ORDER BY part ASC").get()
 		if (!row) continue
 		if (added) {
-			discord.snow.channel.addChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message pinned on Matrix")
+			discord.snow.channel.createChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message pinned on Matrix")
 		} else {
-			discord.snow.channel.removeChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message unpinned on Matrix")
+			discord.snow.channel.deleteChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message unpinned on Matrix")
 		}
 	}
 }

src/m2d/event-dispatcher.js

@@ -24,6 +24,8 @@ const vote = sync.require("./actions/vote")
 const matrixCommandHandler = sync.require("../matrix/matrix-command-handler")
 /** @type {import("../matrix/utils")} */
 const utils = sync.require("../matrix/utils")
+/** @type {import("../matrix/mreq")}) */
+const mreq = sync.require("../matrix/mreq")
 /** @type {import("../matrix/api")}) */
 const api = sync.require("../matrix/api")
 /** @type {import("../d2m/actions/create-room")} */
@@ -32,6 +34,8 @@ const createRoom = sync.require("../d2m/actions/create-room")
 const roomUpgrade = require("../matrix/room-upgrade")
 /** @type {import("../d2m/actions/retrigger")} */
 const retrigger = sync.require("../d2m/actions/retrigger")
+/** @type {import("../matrix/homeserver-status")} */
+const homeserverStatus = sync.require("../matrix/homeserver-status")
 const {reg} = require("../matrix/read-registration")
 
 let lastReportedEvent = 0
@@ -166,7 +170,12 @@ async function sendError(roomID, source, type, e, payload) {
 				key: "🔁"
 			}
 		})
-	} catch (e) {}
+	} catch (e) {
+		if (e instanceof mreq.MatrixServerError && [502, 503].includes(e.httpStatus)) {
+			// Matrix homeserver is down (reverse proxy indicated failure; synapse doesn't generate 502/503 when posting an event to a room)
+			homeserverStatus.homeserverStatus.setErrorWithPacket(payload)
+		}
+	}
 }
 
 function guard(type, fn) {

src/matrix/api.js

@@ -452,7 +452,8 @@ async function ping() {
 		headers: {
 			Authorization: `Bearer ${reg.as_token}`
 		},
-		body: "{}"
+		body: "{}",
+		signal: AbortSignal.timeout(15e3)
 	})
 	const root = await res.json()
 	return {

src/matrix/homeserver-status.js

@@ -0,0 +1,134 @@
+// @ts-check
+
+const assert = require("assert")
+const Denque = require("denque")
+const StateMachine = require("snowtransfer").StateMachine
+
+const passthrough = require("../passthrough")
+const {sync} = passthrough
+/** @type {import("../d2m/discord-packets")} */
+const discordPackets = sync.require("../d2m/discord-packets")
+/** @type {import("../matrix/api")} */
+const api = sync.require("../matrix/api")
+
+const DEBUG_HOMESERVER_STATUS = false
+
+function debugHomeserverStatus(message) {
+	if (DEBUG_HOMESERVER_STATUS) {
+		console.log(message)
+	}
+}
+
+const homeserverStatus = new class HomeserverStatus {
+	constructor() {
+		/** @private */
+		this.queue = new Denque()
+
+		/** @private */
+		this.pingInterval = undefined
+
+		/** @private */
+		this.sm = new StateMachine("online")
+			.defineState("online")
+
+			.defineState("checking", {
+				onEnter: [async () => {
+					const pingResult = await api.ping().catch(e => ({ok: false}))
+					if (pingResult.ok) {
+						this.sm.doTransition("check ok")
+					} else {
+						this.sm.doTransition("check fail")
+					}
+				}],
+				onLeave: [],
+				transitions: new Map()
+			})
+
+			.defineState("offline", {
+				onEnter: [() => {
+					this.pingInterval = setInterval(async () => {
+						const pingResult = await api.ping().catch(e => ({ok: false, status: "net", root: e.message}))
+						if (pingResult.ok) {
+							this.sm.doTransition("ping ok")
+						}
+					}, 15e3)
+				}],
+				onLeave: [() => {
+					clearInterval(this.pingInterval)
+				}],
+				transitions: new Map()
+			})
+
+			.defineState("recovering", {
+				onEnter: [async () => { // Drain queue.
+					while (!this.queue.isEmpty()) {
+						const packet = this.queue.peekFront() // same position as .shift()
+						debugHomeserverStatus(`homeserver status: ${new Date().toISOString()} dq packet ${packet.t} ${packet.d?.content}`)
+						await discordPackets.dispatchPacketToBridge(passthrough.discord, packet)
+						if (this.sm.currentStateName !== "recovering") return // got kicked out due to another error
+						this.queue.shift()
+					}
+					this.sm.doTransition("recovered")
+				}],
+				onLeave: [],
+				transitions: new Map()
+			})
+
+			.defineUniversalTransition("error", "offline")
+			.defineTransition("offline", "ping ok", "recovering")
+			.defineTransition("recovering", "recovered", "online")
+			.defineTransition("online", "check", "checking")
+			.defineTransition("checking", "check ok", "recovering")
+			.defineTransition("checking", "check fail", "offline")
+
+		this.sm.on("enter", st => debugHomeserverStatus(`homeserver status: ${st}`))
+		this.sm.setMaxListeners(101)
+		this.sm.freeze()
+	}
+
+	isRealTime() {
+		return this.sm.currentStateName === "online"
+	}
+
+	/** @param {boolean} forceCheck */
+	waitForOnline(forceCheck) {
+		const onlinePromise = new Promise(resolve => {
+			// Already online? Start check or just done
+			if (this.sm.currentStateName === "online") {
+				if (forceCheck) {
+					this.sm.doTransition("check")
+				} else {
+					return resolve(null)
+				}
+			}
+
+			// Checking or not online. Wait for online.
+			const onlineListener = stateName => {
+				if (stateName === "online") {
+					this.sm.removeListener("enter", onlineListener)
+					resolve(null)
+				}
+			}
+			this.sm.on("enter", onlineListener)
+		})
+		return onlinePromise
+	}
+
+	/**
+	 * When offline or recovering, call this for incoming packets to queue them to be sent in order later.
+	 */
+	queuePacket(packet) {
+		assert(["offline", "recovering"].includes(this.sm.currentStateName))
+		this.queue.push(packet)
+	}
+
+	setErrorWithPacket(packet) {
+		const wasRecovering = this.sm.currentStateName === "recovering"
+		this.sm.doTransition("error")
+		if (!wasRecovering) { // if was recovering then packet is already in the right place in queue
+			this.queuePacket(packet)
+		}
+	}
+}
+
+module.exports.homeserverStatus = homeserverStatus

src/matrix/mreq.js

@@ -9,9 +9,12 @@ const {reg} = require("./read-registration.js")
 const baseUrl = `${reg.ooye.server_origin}/_matrix`
 
 class MatrixServerError extends Error {
-	constructor(data, opts) {
+	/** @param {number} httpStatus} */
+	constructor(data, httpStatus, opts) {
 		super(data.error || data.errcode)
 		this.data = data
+		/** @type {number} */
+		this.httpStatus = httpStatus
 		/** @type {string} */
 		this.errcode = data.errcode
 		this.opts = opts
@@ -44,11 +47,11 @@ async function _convertBody(body) {
 async function makeMatrixServerError(res, opts = {}) {
 	delete opts.headers?.["Authorization"]
 	if (res.headers.get("content-type") === "application/json") {
-		return new MatrixServerError(await res.json(), opts)
+		return new MatrixServerError(await res.json(), res.status, opts)
 	} else if (res.headers.get("content-type")?.startsWith("text/")) {
-		return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, opts)
+		return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, res.status, opts)
 	} else {
-		return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, opts)
+		return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, res.status, opts)
 	}
 }
 
@@ -78,12 +81,12 @@ async function mreq(method, url, bodyIn, extra = {}) {
 		var root = JSON.parse(text)
 	} catch (e) {
 		delete opts.headers?.["Authorization"]
-		throw new MatrixServerError(text, {baseUrl, url, ...opts})
+		throw new MatrixServerError(text, res.status, {baseUrl, url, ...opts})
 	}
 
 	if (!res.ok || root.errcode) {
 		delete opts.headers?.["Authorization"]
-		throw new MatrixServerError(root, {baseUrl, url, ...opts})
+		throw new MatrixServerError(root, res.status, {baseUrl, url, ...opts})
 	}
 	return root
 }

start.js

@@ -31,8 +31,9 @@ sync.require("./src/m2d/event-dispatcher")
 
 ;(async () => {
 	await migrate.migrate(db)
+	process.stdout.write("Connecting to Discord... ")
 	await discord.cloud.connect()
-	console.log("Discord gateway started")
+	console.log("ok.")
 	sync.require("./src/web/server")
 	await power.applyPower()