Rework event dispatchers

The mutated event was being used as the error payload instead of the
original event, so it forgot that it was an edit.

.gitignore

@@ -1,7 +1,16 @@
-node_modules
+# Secrets
 config.js
 registration.yaml
+ooye.db*
+events.db*
+
+# Automatically generated
+node_modules
 coverage
-db/ooye.db*
 test/res/*
 !test/res/lottie*
+icon.svg
+*~
+.#*
+\#*#
+launch.json

addbot.js

@@ -1,15 +1,18 @@
+#!/usr/bin/env node
 // @ts-check
 
-const config = require("./config")
+const {reg} = require("./src/matrix/read-registration")
+const token = reg.ooye.discord_token
+const id = Buffer.from(token.split(".")[0], "base64").toString()
 
 function addbot() {
-	const token = config.discordToken
-	const id = Buffer.from(token.split(".")[0], "base64")
 	return `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=${id}&scope=bot&permissions=1610883072 `
 }
 
+/* c8 ignore next 3 */
 if (process.argv.find(a => a.endsWith("addbot") || a.endsWith("addbot.js"))) {
 	console.log(addbot())
 }
 
+module.exports.id = id
 module.exports.addbot = addbot

addbot.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 echo "Open this link to add the bot to a Discord server:"
-echo "https://discord.com/oauth2/authorize?client_id=$(grep discordToken config.js | sed -E 's!.*: ["'\'']([A-Za-z0-9+=/_-]*).*!\1!g' | base64 -d)&scope=bot&permissions=1610883072"
+echo "https://discord.com/oauth2/authorize?client_id=$(grep discord_token registration.yaml | sed -E 's!.*: ["'\'']([A-Za-z0-9+=/_-]*).*!\1!g' | base64 -d)&scope=bot&permissions=1610883072"

config.example.js

@@ -1,3 +0,0 @@
-module.exports = {
-	discordToken: "yes"
-}

d2m/actions/create-room.test.js

@@ -1,89 +0,0 @@
-// @ts-check
-
-const {channelToKState, _convertNameAndTopic} = require("./create-room")
-const {kstateStripConditionals} = require("../../matrix/kstate")
-const {test} = require("supertape")
-const testData = require("../../test/data")
-
-const passthrough = require("../../passthrough")
-const {db} = passthrough
-
-test("channel2room: discoverable privacy room", async t => {
-	db.prepare("UPDATE guild_space SET privacy_level = 2").run()
-	t.deepEqual(
-		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
-		Object.assign({}, testData.room.general, {
-			"m.room.guest_access/": {guest_access: "forbidden"},
-			"m.room.join_rules/": {join_rule: "public"},
-			"m.room.history_visibility/": {history_visibility: "world_readable"}
-		})
-	)
-})
-
-test("channel2room: linkable privacy room", async t => {
-	db.prepare("UPDATE guild_space SET privacy_level = 1").run()
-	t.deepEqual(
-		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
-		Object.assign({}, testData.room.general, {
-			"m.room.guest_access/": {guest_access: "forbidden"},
-			"m.room.join_rules/": {join_rule: "public"}
-		})
-	)
-})
-
-test("channel2room: invite-only privacy room", async t => {
-	db.prepare("UPDATE guild_space SET privacy_level = 0").run()
-	t.deepEqual(
-		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
-		testData.room.general
-	)
-})
-
-test("convertNameAndTopic: custom name and topic", t => {
-	t.deepEqual(
-		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, "hauntings"),
-		["hauntings", "#the-twilight-zone | Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
-	)
-})
-
-test("convertNameAndTopic: custom name, no topic", t => {
-	t.deepEqual(
-		_convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, "hauntings"),
-		["hauntings", "#the-twilight-zone\n\nChannel ID: 123\nGuild ID: 456"]
-	)
-})
-
-test("convertNameAndTopic: original name and topic", t => {
-	t.deepEqual(
-		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, null),
-		["the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
-	)
-})
-
-test("convertNameAndTopic: original name, no topic", t => {
-	t.deepEqual(
-		_convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, null),
-		["the-twilight-zone", "Channel ID: 123\nGuild ID: 456"]
-	)
-})
-
-test("convertNameAndTopic: public thread icon", t => {
-	t.deepEqual(
-		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 11}, {id: "456"}, null),
-		["[⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
-	)
-})
-
-test("convertNameAndTopic: private thread icon", t => {
-	t.deepEqual(
-		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 12}, {id: "456"}, null),
-		["[🔒⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
-	)
-})
-
-test("convertNameAndTopic: voice channel icon", t => {
-	t.deepEqual(
-		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 2}, {id: "456"}, null),
-		["[🔊] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
-	)
-})

d2m/converters/message-to-event.embeds.test.js

@@ -1,143 +0,0 @@
-const {test} = require("supertape")
-const {messageToEvent} = require("./message-to-event")
-const data = require("../../test/data")
-const Ty = require("../../types")
-
-test("message2event embeds: nothing but a field", async t => {
-	const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {})
-	t.deepEqual(events, [{
-		$type: "m.room.message",
-		"m.mentions": {},
-		msgtype: "m.notice",
-		body: "| ### Amanda 🎵#2192 :online:"
-			+ "\n| willow tree, branch 0"
-			+ "\n| **❯ Uptime:**\n| 3m 55s\n| **❯ Memory:**\n| 64.45MB",
-		format: "org.matrix.custom.html",
-		formatted_body: '<blockquote><p><strong>Amanda 🎵#2192 <img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ\" title=\":online:\" alt=\":online:\">'
-			+ '<br>willow tree, branch 0</strong>'
-			+ '<br><strong>❯ Uptime:</strong><br>3m 55s'
-			+ '<br><strong>❯ Memory:</strong><br>64.45MB</p></blockquote>'
-	}])
-})
-
-test("message2event embeds: reply with just an embed", async t => {
-	const events = await messageToEvent(data.message_with_embeds.reply_with_only_embed, data.guild.general, {})
-	t.deepEqual(events, [{
-		$type: "m.room.message",
-		msgtype: "m.notice",
-		"m.mentions": {},
-		body: "| ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
-			+ "\n| \n| ## https://twitter.com/i/status/1707484191963648161"
-			+ "\n| \n| does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
-			+ "\n| \n| ### Retweets"
-			+ "\n| 119"
-			+ "\n| \n| ### Likes"
-			+ "\n| 5581"
-			+ "\n| — Twitter",
-		format: "org.matrix.custom.html",
-		formatted_body: '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong></p>'
-			+ '<p><strong><a href="https://twitter.com/i/status/1707484191963648161">https://twitter.com/i/status/1707484191963648161</a></strong>'
-			+ '</p><p>does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
-			+ '</p><p><strong>Retweets</strong><br>119</p><p><strong>Likes</strong><br>5581</p>— Twitter</blockquote>'
-	}])
-})
-
-test("message2event embeds: image embed and attachment", async t => {
-	const events = await messageToEvent(data.message_with_embeds.image_embed_and_attachment, data.guild.general, {}, {
-		api: {
-			async getJoinedMembers(roomID) {
-				return {joined: []}
-			}
-		}
-	})
-	t.deepEqual(events, [{
-		$type: "m.room.message",
-		msgtype: "m.text",
-		body: "https://tootsuite.net/Warp-Gate2.gif\ntanget: @ monster spawner",
-		format: "org.matrix.custom.html",
-		formatted_body: '<a href="https://tootsuite.net/Warp-Gate2.gif">https://tootsuite.net/Warp-Gate2.gif</a><br>tanget: @ monster spawner',
-		"m.mentions": {}
-	}, {
-		$type: "m.room.message",
-		msgtype: "m.image",
-		url: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR",
-		body: "Screenshot_20231001_034036.jpg",
-		external_url: "https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg?ex=651a1faa&is=6518ce2a&hm=eb5ca80a3fa7add8765bf404aea2028a28a2341e4a62435986bcdcf058da82f3&",
-		filename: "Screenshot_20231001_034036.jpg",
-		info: {
-			h: 1170,
-			w: 1080,
-			size: 51981,
-			mimetype: "image/jpeg"
-		},
-		"m.mentions": {}
-	}])
-})
-
-test("message2event embeds: blockquote in embed", async t => {
-	let called = 0
-	const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general, {}, {
-		api: {
-			async getStateEvent(roomID, type, key) {
-				called++
-				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
-				t.equal(type, "m.room.power_levels")
-				t.equal(key, "")
-				return {
-					users: {
-						"@_ooye_bot:cadence.moe": 100
-					}
-				}
-			},
-			async getJoinedMembers(roomID) {
-				called++
-				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
-				return {
-					joined: {
-						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
-						"@user:example.invalid": {display_name: null, avatar_url: null}
-					}
-				}
-			}
-		}
-	})
-	t.deepEqual(events, [{
-		$type: "m.room.message",
-		msgtype: "m.text",
-		body: ":emoji: **4 |** #wonderland",
-		format: "org.matrix.custom.html",
-		formatted_body: `<img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO\" title=\":emoji:\" alt=\":emoji:\"> <strong>4 |</strong> <a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe?via=cadence.moe&via=example.invalid\">#wonderland</a>`,
-		"m.mentions": {}
-	}, {
-		$type: "m.room.message",
-		msgtype: "m.notice",
-		body: "| ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n| \n| reply draft\n| > The following is a message composed via consensus of the Stinker Council.\n| > \n| > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n| > \n| > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n| > \n| > There will be no further communication.\n| \n| [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)",
-		format: "org.matrix.custom.html",
-		formatted_body: "<blockquote><p><strong><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">⏺️ minimus</a></strong></p><p>reply draft<br><blockquote>The following is a message composed via consensus of the Stinker Council.<br><br>For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.<br><br>Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.<br><br>There will be no further communication.</blockquote></p><p><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">Go to Message</a></p></blockquote>",
-		"m.mentions": {}
-	}])
-	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
-})
-
-test("message2event embeds: crazy html is all escaped", async t => {
-	const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general)
-	t.deepEqual(events, [{
-		$type: "m.room.message",
-		msgtype: "m.notice",
-		body: "| ## ⏺️ <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
-			+ "\n| \n| ## <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
-			+ "\n| \n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
-			+ "\n| \n| ### <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
-			+ "\n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
-			+ "\n| — <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
-		format: "org.matrix.custom.html",
-		formatted_body: `<blockquote>`
-			+ `<p><strong><a href="https://a.co/&amp;amp;&lt;script&gt;">⏺️ &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</a></strong></p>`
-			+ `<p><strong><a href=\"https://a.co/&amp;amp;&lt;script&gt;">&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href="https://a.co/&amp;amp">https://a.co/&amp;amp</a>;)</a></strong></p>`
-			+ `<p>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
-			+ `<p><strong>&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href=\"https://a.co/&amp;amp\">https://a.co/&amp;amp</a>;)</strong>`
-			+ `<br>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
-			+ `— &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</blockquote>`,
-		"m.mentions": {}
-	}])
-})

discord/discord-command-handler.js

@@ -1,273 +0,0 @@
-// @ts-check
-
-const assert = require("assert").strict
-const util = require("util")
-const DiscordTypes = require("discord-api-types/v10")
-const reg = require("../matrix/read-registration")
-const {addbot} = require("../addbot")
-
-const {discord, sync, db, select} = require("../passthrough")
-/** @type {import("../matrix/api")}) */
-const api = sync.require("../matrix/api")
-/** @type {import("../matrix/file")} */
-const file = sync.require("../matrix/file")
-/** @type {import("../d2m/actions/create-space")} */
-const createSpace = sync.require("../d2m/actions/create-space")
-/** @type {import("./utils")} */
-const utils = sync.require("./utils")
-
-const PREFIX = "//"
-
-let buttons = []
-
-/**
- * @param {string} channelID where to add the button
- * @param {string} messageID where to add the button
- * @param {string} emoji emoji to add as a button
- * @param {string} userID only listen for responses from this user
- * @returns {Promise<import("discord-api-types/v10").GatewayMessageReactionAddDispatchData>}
- */
-async function addButton(channelID, messageID, emoji, userID) {
-	await discord.snow.channel.createReaction(channelID, messageID, emoji)
-	return new Promise(resolve => {
-		buttons.push({channelID, messageID, userID, resolve, created: Date.now()})
-	})
-}
-
-// Clear out old buttons every so often to free memory
-setInterval(() => {
-	const now = Date.now()
-	buttons = buttons.filter(b => now - b.created < 2*60*60*1000)
-}, 10*60*1000)
-
-/** @param {import("discord-api-types/v10").GatewayMessageReactionAddDispatchData} data */
-function onReactionAdd(data) {
-	const button = buttons.find(b => b.channelID === data.channel_id && b.messageID === data.message_id && b.userID === data.user_id)
-	if (button) {
-		buttons = buttons.filter(b => b !== button) // remove button data so it can't be clicked again
-		button.resolve(data)
-	}
-}
-
-/**
- * @callback CommandExecute
- * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
- * @param {DiscordTypes.APIGuildTextChannel} channel
- * @param {DiscordTypes.APIGuild} guild
- * @param {Partial<DiscordTypes.RESTPostAPIChannelMessageJSONBody>} [ctx]
- */
-
-/**
- * @typedef Command
- * @property {string[]} aliases
- * @property {(message: DiscordTypes.GatewayMessageCreateDispatchData, channel: DiscordTypes.APIGuildTextChannel, guild: DiscordTypes.APIGuild) => Promise<any>} execute
- */
-
-/** @param {CommandExecute} execute */
-function replyctx(execute) {
-	/** @type {CommandExecute} */
-	return function(message, channel, guild, ctx = {}) {
-		ctx.message_reference = {
-			message_id: message.id,
-			channel_id: channel.id,
-			guild_id: guild.id,
-			fail_if_not_exists: false
-		}
-		return execute(message, channel, guild, ctx)
-	}
-}
-
-/** @type {Command[]} */
-const commands = [{
-	aliases: ["icon", "avatar", "roomicon", "roomavatar", "channelicon", "channelavatar"],
-	execute: replyctx(
-		async (message, channel, guild, ctx) => {
-			// Guard
-			const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get()
-			if (!roomID) return discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: "This channel isn't bridged to the other side."
-			})
-
-			// Current avatar
-			const avatarEvent = await api.getStateEvent(roomID, "m.room.avatar", "")
-			const avatarURLParts = avatarEvent?.url.match(/^mxc:\/\/([^/]+)\/(\w+)$/)
-			let currentAvatarMessage =
-				( avatarURLParts ? `Current room-specific avatar: ${reg.ooye.server_origin}/_matrix/media/r0/download/${avatarURLParts[1]}/${avatarURLParts[2]}`
-				: "No avatar. Now's your time to strike. Use `//icon` again with a link or upload to set the room-specific avatar.")
-
-			// Next potential avatar
-			const nextAvatarURL = message.attachments.find(a => a.content_type?.startsWith("image/"))?.url || message.content.match(/https?:\/\/[^ ]+\.[^ ]+\.(?:png|jpg|jpeg|webp)\b/)?.[0]
-			let nextAvatarMessage =
-				( nextAvatarURL ? `\nYou want to set it to: ${nextAvatarURL}\nHit ✅ to make it happen.`
-				: "")
-
-			const sent = await discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: currentAvatarMessage + nextAvatarMessage
-			})
-
-			if (nextAvatarURL) {
-				addButton(channel.id, sent.id, "✅", message.author.id).then(async data => {
-					const mxcUrl = await file.uploadDiscordFileToMxc(nextAvatarURL)
-					await api.sendState(roomID, "m.room.avatar", "", {
-						url: mxcUrl
-					})
-					db.prepare("UPDATE channel_room SET custom_avatar = ? WHERE channel_id = ?").run(mxcUrl, channel.id)
-					await discord.snow.channel.createMessage(channel.id, {
-						...ctx,
-						content: "Your creation is unleashed. Any complaints will be redirected to Grelbo."
-					})
-				})
-			}
-		}
-	)
-}, {
-	aliases: ["invite"],
-	execute: replyctx(
-		async (message, channel, guild, ctx) => {
-			// Check guild is bridged
-			const spaceID = select("guild_space", "space_id", {guild_id: guild.id}).pluck().get()
-			const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get()
-			if (!spaceID || !roomID) return discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: "This server isn't bridged to Matrix, so you can't invite Matrix users."
-			})
-
-			// Check CREATE_INSTANT_INVITE permission
-			assert(message.member)
-			const guildPermissions = utils.getPermissions(message.member.roles, guild.roles)
-			if (!(guildPermissions & BigInt(1))) {
-				return discord.snow.channel.createMessage(channel.id, {
-					...ctx,
-					content: "You don't have permission to invite people to this Discord server."
-				})
-			}
-
-			// Guard against accidental mentions instead of the MXID
-			if (message.content.match(/<[@#:].*>/)) return discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: "You have to say the Matrix ID of the person you want to invite, but you mentioned a Discord user in your message.\nOne way to fix this is by writing `` ` `` backticks `` ` `` around the Matrix ID."
-			})
-
-			// Get named MXID
-			const mxid = message.content.match(/@([^:]+):([a-z0-9:-]+\.[a-z0-9.:-]+)/)?.[0]
-			if (!mxid) return discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: "You have to say the Matrix ID of the person you want to invite. Matrix IDs look like this: `@username:example.org`"
-			})
-
-			// Check for existing invite to the space
-			let spaceMember
-			try {
-				spaceMember = await api.getStateEvent(spaceID, "m.room.member", mxid)
-			} catch (e) {}
-			if (spaceMember && spaceMember.membership === "invite") {
-				return discord.snow.channel.createMessage(channel.id, {
-					...ctx,
-					content: `\`${mxid}\` already has an invite, which they haven't accepted yet.`
-				})
-			}
-
-			// Invite Matrix user if not in space
-			if (!spaceMember || spaceMember.membership !== "join") {
-				await api.inviteToRoom(spaceID, mxid)
-				return discord.snow.channel.createMessage(channel.id, {
-					...ctx,
-					content: `You invited \`${mxid}\` to the server.`
-				})
-			}
-
-			// The Matrix user *is* in the space, maybe we want to invite them to this channel?
-			let roomMember
-			try {
-				roomMember = await api.getStateEvent(roomID, "m.room.member", mxid)
-			} catch (e) {}
-			if (!roomMember || (roomMember.membership !== "join" && roomMember.membership !== "invite")) {
-				const sent = await discord.snow.channel.createMessage(channel.id, {
-					...ctx,
-					content: `\`${mxid}\` is already in this server. Would you like to additionally invite them to this specific channel?\nHit ✅ to make it happen.`
-				})
-				return addButton(channel.id, sent.id, "✅", message.author.id).then(async data => {
-					await api.inviteToRoom(roomID, mxid)
-					await discord.snow.channel.createMessage(channel.id, {
-						...ctx,
-						content: `You invited \`${mxid}\` to the channel.`
-					})
-				})
-			}
-
-			// The Matrix user *is* in the space and in the channel.
-			await discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: `\`${mxid}\` is already in this server and this channel.`
-			})
-		}
-	)
-}, {
-	aliases: ["addbot"],
-	execute: replyctx(
-		async (message, channel, guild, ctx) => {
-			return discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: addbot()
-			})
-		}
-	)
-}, {
-	aliases: ["privacy", "discoverable", "publish", "published"],
-	execute: replyctx(
-		async (message, channel, guild, ctx) => {
-			const current = select("guild_space", "privacy_level", {guild_id: guild.id}).pluck().get()
-			if (current == null) {
-				return discord.snow.channel.createMessage(channel.id, {
-					...ctx,
-					content: "This server isn't bridged to the other side."
-				})
-			}
-
-			const levels = ["invite", "link", "directory"]
-			const level = levels.findIndex(x => message.content.includes(x))
-			if (level === -1) {
-				return discord.snow.channel.createMessage(channel.id, {
-					...ctx,
-					content: "**Usage: `//privacy <level>`**. This will set who can join the space on Matrix-side. There are three levels:"
-						+ "\n`invite`: Can only join with a direct in-app invite from another Matrix user, or the //invite command."
-						+ "\n`link`: Matrix links can be created and shared like Discord's invite links. `invite` features also work."
-						+ "\n`directory`: Publishes to the Matrix in-app directory, like Server Discovery. Preview enabled. `invite` and `link` also work."
-						+ `\n**Current privacy level: \`${levels[current]}\`**`
-				})
-			}
-
-			assert(message.member)
-			const guildPermissions = utils.getPermissions(message.member.roles, guild.roles)
-			if (guild.owner_id !== message.author.id && !(guildPermissions & BigInt(0x28))) { // MANAGE_GUILD | ADMINISTRATOR
-				return discord.snow.channel.createMessage(channel.id, {
-					...ctx,
-					content: "You don't have permission to change the privacy level. You need Manage Server or Administrator."
-				})
-			}
-
-			db.prepare("UPDATE guild_space SET privacy_level = ? WHERE guild_id = ?").run(level, guild.id)
-			discord.snow.channel.createMessage(channel.id, {
-				...ctx,
-				content: `Privacy level updated to \`${levels[level]}\`. Changes will apply shortly.`
-			})
-			await createSpace.syncSpaceFully(guild.id)
-		}
-	)
-}]
-
-/** @type {CommandExecute} */
-async function execute(message, channel, guild) {
-	if (!message.content.startsWith(PREFIX)) return
-	const words = message.content.slice(PREFIX.length).split(" ")
-	const commandName = words[0]
-	const command = commands.find(c => c.aliases.includes(commandName))
-	if (!command) return
-
-	await command.execute(message, channel, guild)
-}
-
-module.exports.execute = execute
-module.exports.onReactionAdd = onReactionAdd

discord/utils.js

@@ -1,74 +0,0 @@
-// @ts-check
-
-const DiscordTypes = require("discord-api-types/v10")
-
-const EPOCH = 1420070400000
-
-/**
- * @param {string[]} userRoles
- * @param {DiscordTypes.APIGuild["roles"]} guildRoles
- * @param {string} [userID]
- * @param {DiscordTypes.APIGuildChannel["permission_overwrites"]} [channelOverwrites]
- */
-function getPermissions(userRoles, guildRoles, userID, channelOverwrites) {
-	let allowed = BigInt(0)
-	let everyoneID
-	// Guild allows
-	for (const role of guildRoles) {
-		if (role.name === "@everyone") {
-			allowed |= BigInt(role.permissions)
-			everyoneID = role.id
-		}
-		if (userRoles.includes(role.id)) {
-			allowed |= BigInt(role.permissions)
-		}
-	}
-
-	if (channelOverwrites) {
-		/** @type {((overwrite: Required<DiscordTypes.APIGuildChannel>["permission_overwrites"][0]) => any)[]} */
-		const actions = [
-			// Channel @everyone deny
-			overwrite => overwrite.id === everyoneID && (allowed &= ~BigInt(overwrite.deny)),
-			// Channel @everyone allow
-			overwrite => overwrite.id === everyoneID && (allowed |= BigInt(overwrite.allow)),
-			// Role deny
-			overwrite => userRoles.includes(overwrite.id) && (allowed &= ~BigInt(overwrite.deny)),
-			// Role allow
-			overwrite => userRoles.includes(overwrite.id) && (allowed |= BigInt(overwrite.allow)),
-			// User deny
-			overwrite => overwrite.id === userID && (allowed &= ~BigInt(overwrite.deny)),
-			// User allow
-			overwrite => overwrite.id === userID && (allowed |= BigInt(overwrite.allow))
-		]
-		for (let i = 0; i < actions.length; i++) {
-			for (const overwrite of channelOverwrites) {
-				actions[i](overwrite)
-			}
-		}
-	}
-	return allowed
-}
-
-/**
- * Command interaction responses have a webhook_id for some reason, but still have real author info of a real bot user in the server.
- * @param {DiscordTypes.APIMessage} message
- */
-function isWebhookMessage(message) {
-	const isInteractionResponse = message.type === 20
-	return message.webhook_id && !isInteractionResponse
-}
-
-/** @param {string} snowflake */
-function snowflakeToTimestampExact(snowflake) {
-	return Number(BigInt(snowflake) >> 22n) + EPOCH
-}
-
-/** @param {number} timestamp */
-function timestampToSnowflakeInexact(timestamp) {
-	return String((timestamp - EPOCH) * 2**22)
-}
-
-module.exports.getPermissions = getPermissions
-module.exports.isWebhookMessage = isWebhookMessage
-module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact
-module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact

discord/utils.test.js

@@ -1,84 +0,0 @@
-const {test} = require("supertape")
-const data = require("../test/data")
-const utils = require("./utils")
-
-test("is webhook message: identifies bot interaction response as not a message", t => {
-	t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction), false)
-})
-
-test("is webhook message: identifies webhook interaction response as not a message", t => {
-	t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction_without_bot_user), false)
-})
-
-test("is webhook message: identifies webhook message as a message", t => {
-	t.equal(utils.isWebhookMessage(data.special_message.bridge_echo_webhook), true)
-})
-
-test("discord utils: converts snowflake to timestamp", t => {
-	t.equal(utils.snowflakeToTimestampExact("86913608335773696"), 1440792219004)
-})
-
-test("discord utils: converts timestamp to snowflake", t => {
-	t.match(utils.timestampToSnowflakeInexact(1440792219004), /^869136083357.....$/)
-})
-
-test("getPermissions: channel overwrite to allow role works", t => {
-	const guildRoles = [
-		{
-			version: 1695412489043,
-			unicode_emoji: null,
-			tags: {},
-			position: 0,
-			permissions: "559623605571137",
-			name: "@everyone",
-			mentionable: false,
-			managed: false,
-			id: "1154868424724463687",
-			icon: null,
-			hoist: false,
-			flags: 0,
-			color: 0
-		},
-		{
-			version: 1695412604262,
-			unicode_emoji: null,
-			tags: { bot_id: "466378653216014359" },
-			position: 1,
-			permissions: "536995904",
-			name: "PluralKit",
-			mentionable: false,
-			managed: true,
-			id: "1154868908336099444",
-			icon: null,
-			hoist: false,
-			flags: 0,
-			color: 0
-		},
-		{
-			version: 1698778936921,
-			unicode_emoji: null,
-			tags: {},
-			position: 1,
-			permissions: "536870912",
-			name: "web hookers",
-			mentionable: false,
-			managed: false,
-			id: "1168988246680801360",
-			icon: null,
-			hoist: false,
-			flags: 0,
-			color: 0
-		}
-	]
-	const userRoles = [ "1168988246680801360" ]
-	const userID = "684280192553844747"
-	const overwrites = [
-		{ type: 0, id: "1154868908336099444", deny: "0", allow: "1024" },
-		{ type: 0, id: "1154868424724463687", deny: "1024", allow: "0" },
-		{ type: 0, id: "1168988246680801360", deny: "0", allow: "1024" },
-		{ type: 1, id: "353373325575323648", deny: "0", allow: "1024" }
-	]
-	const permissions = utils.getPermissions(userRoles, guildRoles, userID, overwrites)
-	const want = BigInt(1 << 10 | 1 << 16)
-	t.equal((permissions & want), want)
-})

docs/developer-orientation.md

@@ -0,0 +1,130 @@
+# Development setup
+
+* Install development dependencies with `npm install --save-dev` so you can run the tests.
+* Most files you change, such as actions, converters, and web, will automatically be reloaded.
+* If developing on a different computer to the one running the homeserver, use SSH port forwarding so that Synapse can connect on its `localhost:6693` to reach the running bridge on your computer. Example: `ssh -T -v -R 6693:localhost:6693 me@matrix.cadence.moe`
+* I recommend developing in Visual Studio Code so that the JSDoc x TypeScript annotation comments just work automatically. I don't know which other editors or language servers support annotations and type inference.
+
+# Efficiency details
+
+Using WeatherStack as a thin layer between the bridge application and the Discord API lets us control exactly what data is cached in memory. Only necessary information is cached. For example, member data, user data, message content, and past edits are never stored in memory. This keeps the memory usage low and also prevents it ballooning in size over the bridge's runtime.
+
+The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs and URLs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 300,000 messages (representing a year of conversations) requires 47.3 MB of storage space in the SQLite database.
+
+Only necessary data and columns are queried from the database. We only contact the homeserver API if the database doesn't contain what we need.
+
+File uploads (like avatars from bridged members) are checked locally and deduplicated. Only brand new files are uploaded to the homeserver. This saves loads of space in the homeserver's media repo, especially for Synapse.
+
+Switching to [WAL mode](https://www.sqlite.org/wal.html) could improve your database access speed even more. Run `node scripts/wal.js` if you want to switch to WAL mode. (This will also enable `synchronous = NORMAL`.)
+
+
+# Repository structure
+
+    .
+    * Runtime configuration, like tokens and user info:
+    ├── registration.yaml
+    * You are here! :)
+    ├── readme.md
+    * The bridge's SQLite database is stored here:
+    ├── ooye.db*
+    * Source code
+    └── src
+        * Database schema:
+        ├── db
+        │   ├── orm.js, orm-defs.d.ts
+        │   * Migrations change the database schema when you update to a newer version of OOYE:
+        │   ├── migrate.js
+        │   └── migrations
+        │       └── *.sql, *.js
+        * Discord-to-Matrix bridging:
+        ├── d2m
+        │   * Execute actions through the whole flow, like sending a Discord message to Matrix:
+        │   ├── actions
+        │   │   └── *.js
+        │   * Convert data from one form to another without depending on bridge state. Called by actions:
+        │   ├── converters
+        │   │   └── *.js
+        │   * Making Discord work:
+        │   ├── discord-*.js
+        │   * Listening to events from Discord and dispatching them to the correct `action`:
+        │   └── event-dispatcher.js
+        * Discord bot commands and menus:
+        ├── discord
+        │   ├── interactions
+        │   │   └── *.js
+        │   └── discord-command-handler.js
+        * Matrix-to-Discord bridging:
+        ├── m2d
+        │   * Execute actions through the whole flow, like sending a Matrix message to Discord:
+        │   ├── actions
+        │   │   └── *.js
+        │   * Convert data from one form to another without depending on bridge state. Called by actions:
+        │   ├── converters
+        │   │   └── *.js
+        │   * Listening to events from Matrix and dispatching them to the correct `action`:
+        │   └── event-dispatcher.js
+        * We aren't using the matrix-js-sdk, so here are all the functions for the Matrix C-S and Appservice APIs:
+        ├── matrix
+        │   └── *.js
+        * Various files you can run once if you need them.
+        └── scripts
+            * First time running a new bridge? Run this file to set up prerequisites on the Matrix server:
+            ├── setup.js
+            * Hopefully you won't need the rest of these. Code quality varies wildly.
+            └── *.js
+
+# Read next
+
+If you haven't set up Out Of Your Element yet, you might find [Simplified homeserver setup](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/simplified-homeserver-setup.md) helpful.
+
+If you don't know what the Matrix event JSON generally looks like, turn on developer tools in your client (Element has pretty good ones). Right click a couple of messages and see what they look like on the inside.
+
+I recommend first reading [How to add a new event type](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/how-to-add-a-new-event-type.md) as this will fill you in on key information in how the codebase is organised, which data structures are important, and what level of abstraction we're working on.
+
+If you haven't seen the [Discord API documentation](https://discord.com/developers/docs/) before, have a quick look at one of the pages on there. Same with the [Matrix Client-Server APIs](https://spec.matrix.org/latest/client-server-api/). You don't need to know these inside out, they're primarily references, not stories. But it is useful to have an idea of what a couple of the API endpoints look like, the kind of data they tend to accept, and the kind of data they tend to return.
+
+Then you might like to peruse the other files in the docs folder. Most of these were written stream-of-thought style as I try to work through a problem and find the best way to implement it. You might enjoy getting inside my head and seeing me invent and evaluate ways to solve the problem.
+
+Whether you read those or not, I'm more than happy to help you 1-on-1 with coding your dream feature. Join the chatroom [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) or PM me [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) and ask away.
+
+# Dependency justification
+
+Total transitive production dependencies: 137
+
+### <font size="+2">🦕</font>
+
+* (31) better-sqlite3: SQLite is the best database, and this is the best library for it.
+* (27) @cloudrac3r/pug: Language for dynamic web pages. This is my fork. (I released code that hadn't made it to npm, and removed the heavy pug-filters feature.)
+* (16) stream-mime-type@1: This seems like the best option. Version 1 is used because version 2 is ESM-only.
+* (9) h3: Web server. OOYE needs this for the appservice listener, authmedia proxy, self-service, and more.
+* (11) sharp: Image resizing and compositing. OOYE needs this for the emoji sprite sheets.
+
+### <font size="-1">🪱</font>
+
+* (0) @chriscdn/promise-semaphore: It does what I want.
+* (1) @cloudrac3r/discord-markdown: This is my fork.
+* (0) @cloudrac3r/giframe: This is my fork.
+* (1) @cloudrac3r/html-template-tag: This is my fork.
+* (0) @cloudrac3r/in-your-element: This is my Matrix Appservice API library. It depends on h3 and zod, which are already pulled in by OOYE.
+* (0) @cloudrac3r/mixin-deep: This is my fork. (It fixes a bug in regular mixin-deep.)
+* (0) @cloudrac3r/pngjs: Lottie stickers are converted to bitmaps with the vendored Rlottie WASM build, then the bitmaps are converted to PNG with pngjs.
+* (0) @cloudrac3r/turndown: This HTML-to-Markdown converter looked the most suitable. I forked it to change the escaping logic to match the way Discord works.
+* (3) @stackoverflow/stacks: Stack Overflow design language and icons.
+* (0) ansi-colors: Helps with interactive prompting for the initial setup, and it's already pulled in by enquirer.
+* (1) chunk-text: It does what I want.
+* (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
+* (0) discord-api-types: Bitfields needed at runtime and types needed for development.
+* (0) domino: DOM implementation that's already pulled in by turndown.
+* (1) enquirer: Interactive prompting for the initial setup rather than forcing users to edit YAML non-interactively.
+* (0) entities: Looks fine. No dependencies.
+* (0) get-relative-path: Looks fine. No dependencies.
+* (1) heatsync: Module hot-reloader that I trust.
+* (1) js-yaml: Will be removed in the future after registration.yaml is converted to JSON.
+* (0) lru-cache: For holding unused nonce in memory and letting them be overwritten later if never used.
+* (0) minimist: It's already pulled in by better-sqlite3->prebuild-install.
+* (0) prettier-bytes: It does what I want and has no dependencies.
+* (0) snowtransfer: Discord API library with bring-your-own-caching that I trust.
+* (0) try-to-catch: Not strictly necessary, but it's already pulled in by supertape, so I may as well.
+* (0) uqr: QR code SVG generator. Used on the website to scan in an invite link.
+* (0) xxhash-wasm: Used where cryptographically secure hashing is not required.
+* (0) zod: Input validation for the web server. It's popular and easy to use.

docs/foreign-keys.md

@@ -0,0 +1,98 @@
+# Foreign keys in the Out Of Your Element database
+
+Historically, Out Of Your Element did not use foreign keys in the database, but since I found a need for them, I have decided to add them. Referential integrity is probably valuable as well.
+
+The need is that unlinking a channel and room using the web interface should clear up all related entries from `message_channel`, `event_message`, `reaction`, etc. Without foreign keys, this requires multiple DELETEs with tricky queries. With foreign keys and ON DELETE CASCADE, this just works.
+
+## Quirks
+
+* **REPLACE INTO** internally causes a DELETE followed by an INSERT, and the DELETE part **will trigger any ON DELETE CASCADE** foreign key conditions on the table, even when the primary key being replaced is the same.
+	* ```sql
+		CREATE TABLE discord_channel (channel_id TEXT NOT NULL, name TEXT NOT NULL, PRIMARY KEY (channel_id));
+		CREATE TABLE discord_message (message_id TEXT NOT NULL, channel_id TEXT NOT NULL, PRIMARY KEY (message_id),
+		  FOREIGN KEY (channel_id) REFERENCES discord_channel (channel_id) ON DELETE CASCADE);
+		INSERT INTO discord_channel (channel_id, name) VALUES ("c_1", "place");
+		INSERT INTO discord_message (message_id, channel_id) VALUES ("m_2", "c_1"); -- i love my message
+		REPLACE INTO discord_channel (channel_id, name) VALUES ("c_1", "new place"); -- replace into time
+		-- i love my message
+		SELECT * FROM discord_message; -- where is my message
+		```
+* In SQLite, `pragma foreign_keys = on` must be set **for each connection** after it's established. I've added this at the start of `migrate.js`, which is called by all database connections.
+  * Pragma? Pragma keys
+* Whenever a child row is inserted, SQLite will look up a row from the parent table to ensure referential integrity. This means **the parent table should be sufficiently keyed or indexed on columns referenced by foreign keys**, or SQLite won't let you do it, with a cryptic error message later on during DML. Due to normal forms, foreign keys naturally tend to reference the parent table's primary key, which is indexed, so that's okay. But still keep this in mind, since many of OOYE's tables effectively have two primary keys, for the Discord and Matrix IDs. A composite primary key doesn't count, even when it's the first column. A unique index counts.
+
+## Where keys
+
+Here are some tables that could potentially have foreign keys added between them, and my thought process of whether foreign keys would be a good idea:
+
+* `guild_active` <--(PK guild_id FK)-- `channel_room` ✅
+	* Could be good for referential integrity.
+	* Linking to guild_space would be pretty scary in case the guild was being relinked to a different space - since rooms aren't tied to a space, this wouldn't actually disturb anything. So I pick guild_active instead.
+* `channel_room` <--(PK channel_id FK)-- `message_channel` ✅
+	* Seems useful as we want message records to be deleted when a channel is unlinked.
+* `message_channel` <--(PK message_id PK)-- `event_message` ✅
+	* Seems useful as we want event information to be deleted when a channel is unlinked.
+* `guild_active` <--(PK guild_id PK)-- `guild_space` ✅
+	* All bridged guilds should have a corresponding guild_active entry, so referential integrity would be useful here to make sure we haven't got any weird states.
+* `channel_room` <--(**C** room_id PK)-- `member_cache` ✅
+	* Seems useful as we want to clear the member cache when a channel is unlinked.
+	* There is no index on `channel_room.room_id` right now. It would be good to create this index. Will just make it UNIQUE in the table definition.
+* `message_channel` <--(PK message_id FK)-- `reaction` ✅
+	* Seems useful as we want to clear the reactions cache when a channel is unlinked.
+* `sim` <--(**C** mxid FK)-- `sim_member`
+	* OOYE inner joins on this.
+	* Sims are never deleted so if this was added it would only be used for enforcing referential integrity.
+	* The storage cost of the additional index on `sim` would not be worth the benefits.
+* `channel_room` <--(**C** room_id PK)-- `sim_member`
+	* If a room is being permanently unlinked, it may be useful to see a populated member list. If it's about to be relinked to another channel, we want to keep the sims in the room for more speed and to avoid spamming state events into the timeline.
+	* Either way, the sims could remain in the room even after it's been unlinked. So no referential integrity is desirable here.
+* `sim` <--(PK user_id PK)-- `sim_proxy`
+	* OOYE left joins on this. In normal operation, this relationship might not exist.
+* `channel_room` <--(PK channel_id PK)-- `webhook` ✅
+	* Seems useful. Webhooks should be deleted from Discord just before the channel is unlinked. That should be mirrored in the database too.
+
+## Occurrences of REPLACE INTO/DELETE FROM
+
+* `edit-message.js` — `REPLACE INTO message_channel`
+	* Scary! Changed to INSERT OR IGNORE
+* `send-message.js` — `REPLACE INTO message_channel`
+	* Changed to INSERT OR IGNORE
+* `add-reaction.js` — `REPLACE INTO reaction`
+* `channel-webhook.js` — `REPLACE INTO webhook`
+* `send-event.js` — `REPLACE INTO message_channel`
+	* Seems incorrect? Maybe?? Originally added in fcbb045. Changed to INSERT
+* `event-to-message.js` — `REPLACE INTO member_cache`
+* `oauth.js` — `REPLACE INTO guild_active`
+	* Very scary!! Changed to INSERT .. ON CONFLICT DO UPDATE
+* `create-room.js` — `DELETE FROM channel_room`
+	* Please cascade
+* `delete-message.js`
+	* Removed redundant DELETEs
+* `edit-message.js` — `DELETE FROM event_message`
+* `register-pk-user.js` — `DELETE FROM sim`
+	* It's a failsafe during creation
+* `register-user.js` — `DELETE FROM sim`
+	* It's a failsafe during creation
+* `remove-reaction.js` — `DELETE FROM reaction`
+* `event-dispatcher.js` — `DELETE FROM member_cache`
+* `redact.js` — `DELETE FROM event_message`
+	* Removed this redundant DELETE
+* `send-event.js` — `DELETE FROM event_message`
+	* Removed this redundant DELETE
+
+## How keys
+
+SQLite does not have a complete ALTER TABLE command, so I have to DROP and CREATE. According to [the docs](https://www.sqlite.org/lang_altertable.html), the correct strategy is:
+
+1. (Not applicable) *If foreign key constraints are enabled, disable them using PRAGMA foreign_keys=OFF.*
+2. Start a transaction.
+3. (Not applicable) *Remember the format of all indexes, triggers, and views associated with table X. This information will be needed in step 8 below. One way to do this is to run a query like the following: SELECT type, sql FROM sqlite_schema WHERE tbl_name='X'.*
+4. Use CREATE TABLE to construct a new table "new_X" that is in the desired revised format of table X. Make sure that the name "new_X" does not collide with any existing table name, of course.
+5. Transfer content from X into new_X using a statement like: INSERT INTO new_X SELECT ... FROM X.
+6. Drop the old table X: DROP TABLE X.
+7. Change the name of new_X to X using: ALTER TABLE new_X RENAME TO X.
+8. (Not applicable) *Use CREATE INDEX, CREATE TRIGGER, and CREATE VIEW to reconstruct indexes, triggers, and views associated with table X. Perhaps use the old format of the triggers, indexes, and views saved from step 3 above as a guide, making changes as appropriate for the alteration.*
+9. (Not applicable) *If any views refer to table X in a way that is affected by the schema change, then drop those views using DROP VIEW and recreate them with whatever changes are necessary to accommodate the schema change using CREATE VIEW.*
+10. If foreign key constraints were originally enabled then run PRAGMA foreign_key_check to verify that the schema change did not break any foreign key constraints.
+11. Commit the transaction started in step 2.
+12. (Not applicable) *If foreign keys constraints were originally enabled, reenable them now.*

docs/get-started.md

@@ -0,0 +1,108 @@
+# Setup
+
+If you get stuck, you're welcome to message [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) or [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) to ask for help setting up OOYE!
+
+You'll need:
+
+* Administrative access to a homeserver
+* Discord bot
+* Domain name for the bridge's website - [more info](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/why-does-the-bridge-have-a-website.md)
+* Reverse proxy for that domain - an interactive process will help you set this up in step 5!
+
+Follow these steps:
+
+1. [Get Node.js version 20 or later](https://nodejs.org/en/download/prebuilt-installer). If you're on Linux, you may prefer to install through system's package manager, though Debian and Ubuntu have hopelessly out of date packages.
+
+1. Switch to a normal user account. (i.e. do not run any of the following commands as root or sudo.)
+
+1. Clone this repo and checkout a specific tag. (Development happens on main. Stable versions are tagged.)
+	* The latest release tag is ![](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=flat-square&label=%20&color=black).
+
+1. Install dependencies: `npm install`
+
+1. Run `npm run setup` to check your setup and set the bot's initial state. You only need to run this once ever. This command will guide you precisely through the following steps:
+
+	* First, you'll be asked for information like your homeserver URL.
+
+	* Then you'll be prompted to set up a reverse proxy pointing from your domain to the bridge's web server. Sample configurations can be found at the end of this guide. It will check that the reverse proxy works before you continue.
+
+	* Then you'll need to provide information about your Discord bot, and you'll be asked to change some of its settings.
+
+	* Finally, a registration.yaml file will be generated, which you need to give to your homeserver. You'll be told how to do this. It will check that it's done properly.
+
+1. Start the bridge: `npm run start`
+
+## Update
+
+New versions are announced in [#updates](https://matrix.to/#/#ooye-updates:cadence.moe) and listed on [releases](https://gitdab.com/cadence/out-of-your-element/releases). Here's how to update:
+
+1. Fetch the repo and checkout the latest release tag.
+
+1. Install dependencies: `npm install`
+
+1. Restart the bridge: Stop the currently running process, and then start the new one with `npm run start`
+
+# Get Started
+
+Visit the website on the domain name you set up, and click the button to add the bot to your Discord server.
+
+* If you click the Easy Mode button, it will automatically create a Matrix room corresponding to each Discord channel. This happens next time a message is sent on Discord (so that your Matrix-side isn't immediately cluttered with lots of inactive rooms).
+
+* If you click the Self Service button, it won't create anything for you. You'll have to provide your own Matrix space and rooms. After you click, you'll be prompted through the process. Use this if you're migrating from another bridge!
+
+After that, to get into the rooms on your Matrix account, use the invite form on the website, or the `/invite [your mxid here]` command on Discord.
+
+I hope you enjoy Out Of Your Element!
+
+----
+<br><br><br><br><br>
+
+# Appendix
+
+## Example reverse proxy for nginx, dedicated domain name
+
+Replace `bridge.cadence.moe` with the hostname you're using.
+
+```nix
+server {
+	listen 80;
+	listen [::]:80;
+	server_name bridge.cadence.moe;
+
+	return 301 https://bridge.cadence.moe$request_uri;
+}
+
+server {
+	listen 443 ssl http2;
+	listen [::]:443 ssl http2;
+	server_name bridge.cadence.moe;
+
+	# ssl parameters here...
+	client_max_body_size 5M;
+
+	location / {
+		add_header Strict-Transport-Security "max-age=63072000; includeSubDomains" always;
+		proxy_pass http://127.0.0.1:6693;
+	}
+}
+```
+
+## Example reverse proxy for nginx, sharing a domain name
+
+Same as above, but change the following:
+
+- `location / {` -> `location /ooye/ {` (any sub-path you want; you MUST use a trailing slash or it won't work)
+- `proxy_pass http://127.0.0.1:6693;` -> `proxy_pass http://127.0.0.1:6693/;` (you MUST use a trailing slash on this too or it won't work)
+
+## Example reverse proxy for Caddy, dedicated domain name
+
+```nix
+bridge.cadence.moe {
+	log {
+		output file /var/log/caddy/access.log
+		format console
+	}
+	encode gzip
+	reverse_proxy 127.0.0.1:6693
+}
+```

docs/self-service-room-creation-rules.md

@@ -0,0 +1,77 @@
+# Self-service room creation rules
+
+Before version 3 of Out Of Your Element, new Matrix rooms would be created on-demand when a Discord channel is spoken in for the first time. This has worked pretty well.
+
+This is done through functions like ensureRoom and ensureSpace in actions:
+
+```js
+async function sendMessage(message, channel, guild, row) {
+	const roomID = await createRoom.ensureRoom(message.channel_id)
+	...
+}
+
+/**
+ * Ensures the room exists. If it doesn't, creates the room with an accurate initial state.
+ * @param {string} channelID
+ * @returns {Promise<string>} Matrix room ID
+ */
+function ensureRoom(channelID) {
+	return _syncRoom(channelID, /* shouldActuallySync */ false) /* calls ensureSpace */
+}
+
+/**
+ * Ensures the space exists. If it doesn't, creates the space with an accurate initial state.
+ * @param {DiscordTypes.APIGuild} guild
+ * @returns {Promise<string>} Matrix space ID
+ */
+function ensureSpace(guild) {
+	return _syncSpace(guild, /* shouldActuallySync */ false)
+}
+```
+
+With the introduction of self-service mode, we still want to retain this as a possible mode of operation, since some people prefer to have OOYE handle this administrative work. However, other people prefer to manage the links between channels and rooms themselves, and have control over what new rooms get linked up to.
+
+Visibly, this is managed through the web interface. The web interface lets moderators enable/disable auto-creation of new rooms, as well as set which channels and rooms are linked together.
+
+There is a small complication. Not only are Matrix rooms created automatically, their Matrix spaces are also created automatically during room sync: ensureRoom calls ensureSpace. If a user opts in to self-service mode by clicking the specific button in the web portal, we must ensure the _space is not created automatically either,_ because the Matrix user will provide a space to link to.
+
+To solve this, we need a way to suppress specific guilds from having auto-created spaces. The natural way to represent this is a column on guild_space, but that doesn't work, because each guild_space row requires a guild and space to be linked, and we _don't want_ them to be linked.
+
+So, internally, OOYE keeps track of this through a new table:
+
+```sql
+CREATE TABLE "guild_active" (
+	"guild_id"	TEXT NOT NULL, -- only guilds that are bridged are present in this table
+	"autocreate"	INTEGER NOT NULL, -- 0 or 1
+	PRIMARY KEY("guild_id")
+) WITHOUT ROWID;
+```
+
+There is one more complication. When adding a Discord bot through web oauth with a redirect_uri, Discord adds the bot to the server normally, _then_ redirects back to OOYE, and only then does OOYE know which guild the bot was just added to. So, for a short time between the bot being added and the user being redirected, OOYE might receive Discord events in the server before it has the chance to create the guild_active database row.
+
+So to prevent this, self-service behaviour needs to be an implicit default, and users must firmly choose one system or another to begin using OOYE. It is important for me to design this in a way that doesn't force users to do any extra work or make a choice they don't understand to keep the pre-v3 behaviour.
+
+So there will be 3 states of whether a guild is self-service or not. At first, it could be absent from the table, in which case events for it will be dropped. Or it could be in the table with autocomplete = 0, in which case only rooms that already exist in channel_room will have messages bridged. Or it could have autocomplete = 1, in which case Matrix rooms will be created as needed, as per the pre-v3 behaviour.
+
+| Auto-create | Meaning                  |
+| --          | ------------             |
+| 😶‍🌫️         | Unbridged - waiting      |
+| ❌         | Bridged - self-service   |
+| ✅         | Bridged - auto-create    |
+
+Pressing buttons on web or using the /invite command on a guild will insert a row into guild_active, allowing it to be bridged.
+
+So here's all the technical changes needed to support self-service in v3:
+
+- New guild_active table showing whether, and how, a guild is bridged.
+- When /invite command is used, INSERT OR IGNORE INTO state 1 and ensureRoom + ensureSpace.
+- When bot is added through "easy mode" web button, REPLACE INTO state 1 and ensureSpace.
+- When bot is added through "self-service" web button, REPLACE INTO state 0.
+- Event dispatcher will only ensureRoom if the guild_active state is 1.
+- createRoom will only create other dependencies if the guild is autocreate.
+
+## Enough with your theory. How do rooms actually get bridged now?
+
+After clicking the easy mode button on web and adding the bot to a server, it will create new Matrix rooms on-demand when any invite features are used (web or command) OR just when any message is sent on Discord.
+
+Alternatively, pressing the self-service mode button and adding the bot to a server will prompt the web user to link it with a space. After doing so, they'll be on the standard guild management page where they can invite to the space and manually link rooms. Nothing will be autocreated.

docs/why-does-the-bridge-have-a-website.md

@@ -0,0 +1,55 @@
+# Why does the bridge have a website?
+
+## It's essential for making images work
+
+Matrix has a feature called [Authenticated Media](https://matrix.org/blog/2024/06/26/sunsetting-unauthenticated-media/), where uploaded media (like user avatars and uploaded files) is restricted to Matrix users only. This means Discord users wouldn't be able to see important parts of the conversation.
+
+To keep things working for Discord users, OOYE's web server can act as a proxy files that were uploaded on Matrix-side. This will automatically take effect when needed, so Discord users shouldn't notice any issues.
+
+## Why now?
+
+I knew a web interface had a lot of potential, but I was reluctant to add one because it would make the initial setup more complicated. However, when authenticated media forced my hand, I saw an opportunity to introduce new useful features. Hopefully you'll agree that it's worth it!
+
+# What else does it do?
+
+## Makes it easy to invite the bot
+
+The home page of the website has buttons to add the bridge bot to a Discord server. If you are primarly a Matrix user and you want somebody else (who may be less technical) to add the bot to their own Discord server, this should make it a lot more intuitive for them.
+
+## Makes it easy to invite yourself or others
+
+After your hypothetical less-technical friend adds the bot to their Discord server, you need to generate an invite for your Matrix account on Matrix-side. Without the website, you might need to guide them through running a /invite command with your user ID. With the website, they don't have to do anything extra. You can use your phone to scan the QR code on their screen, which lets you invite your user ID in your own time.
+
+You can also set the person's permissions when you invite them, so you can easily bootstrap the Matrix side with your trusted ones as moderators.
+
+## To link channels and rooms
+
+Without a website, to link Discord channels to existing Matrix rooms, you'd need to run a /link command with the internal IDs of each room. This is tedious and error-prone, especially if you want to set up a lot of channels. With the web interface, you can see a list of all the available rooms and click them to link them together.
+
+## To change settings
+
+Important settings, like whether the Matrix rooms should be private or publicly accessible, can be configured by clicking buttons rather than memorising commands. Changes take effect immediately.
+
+# Permissions
+
+## Bot invites
+
+Anybody who can access the home page can use the buttons to add your bot - but even without the website, they can already do this by manually constructing a URL. If you want to make it so _only you_ can add your bot to servers, you need to edit [your Discord application](https://discord.com/developers/applications), go to the Bot section, and turn off the switch that says Public Bot.
+
+## Server settings
+
+If you have either the Administrator or Manage Server permissions in a Discord server, you can use the website to manage that server, including linking channels and changing settings.
+
+# Initial setup
+
+The website is built in to OOYE and is always running as part of the bridge. For authenticated media proxy to work, you'll need to make the web server accessible on the public internet over HTTPS, presumably using a reverse proxy.
+
+When you use `npm run setup` as part of OOYE's initial setup, it will guide you through this process, and it will do a thorough self-test to make sure it's configured correctly. If you get stuck or want a configuration template, check the notes below.
+
+## Reverse proxy
+
+When OOYE is running, the web server runs on port 6693. (To use a different port or a UNIX socket, edit registration.yaml's `socket` setting and restart.)
+
+It doesn't have to have its own dedicated domain name, you can also use a sub-path on an existing domain, like the domain of your Matrix homeserver. You are likely already using a reverse proxy to run your homeserver, so this should just be a configuration change.
+
+[See here for sample configurations!](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md#appendix)

m2d/converters/emoji.js

@@ -1,57 +0,0 @@
-// @ts-check
-
-const assert = require("assert").strict
-const Ty = require("../../types")
-
-const passthrough = require("../../passthrough")
-const {sync, select} = passthrough
-
-/**
- * @param {string} input
- * @param {string | null | undefined} shortcode
- * @returns {string?}
- */
-function encodeEmoji(input, shortcode) {
-	let discordPreferredEncoding
-	if (input.startsWith("mxc://")) {
-		// Custom emoji
-		let row = select("emoji", ["emoji_id", "name"], {mxc_url: input}).get()
-		if (!row && shortcode) {
-			// Use the name to try to find a known emoji with the same name.
-			const name = shortcode.replace(/^:|:$/g, "")
-			row = select("emoji", ["emoji_id", "name"], {name: name}).get()
-		}
-		if (!row) {
-			// We don't have this emoji and there's no realistic way to just-in-time upload a new emoji somewhere.
-			// Sucks!
-			return null
-		}
-		// Cool, we got an exact or a candidate emoji.
-		discordPreferredEncoding = encodeURIComponent(`${row.name}:${row.emoji_id}`)
-	} else {
-		// Default emoji
-		// https://github.com/discord/discord-api-docs/issues/2723#issuecomment-807022205 ????????????
-		const encoded = encodeURIComponent(input)
-		const encodedTrimmed = encoded.replace(/%EF%B8%8F/g, "")
-
-		const forceTrimmedList = [
-			"%F0%9F%91%8D", // 👍
-			"%F0%9F%91%8E", // 👎️
-			"%E2%AD%90", // ⭐
-			"%F0%9F%90%88", // 🐈
-			"%E2%9D%93", // ❓
-			"%F0%9F%8F%86", // 🏆️
-			"%F0%9F%93%9A", // 📚️
-		]
-
-		discordPreferredEncoding =
-			( forceTrimmedList.includes(encodedTrimmed) ? encodedTrimmed
-			: encodedTrimmed !== encoded && [...input].length === 2 ? encoded
-			: encodedTrimmed)
-
-		console.log("add reaction from matrix:", input, encoded, encodedTrimmed, "chosen:", discordPreferredEncoding)
-	}
-	return discordPreferredEncoding
-}
-
-module.exports.encodeEmoji = encodeEmoji

m2d/event-dispatcher.js

@@ -1,152 +0,0 @@
-// @ts-check
-
-/*
- * Grab Matrix events we care about, check them, and bridge them.
- */
-
-const util = require("util")
-const Ty = require("../types")
-const {discord, db, sync, as} = require("../passthrough")
-
-/** @type {import("./actions/send-event")} */
-const sendEvent = sync.require("./actions/send-event")
-/** @type {import("./actions/add-reaction")} */
-const addReaction = sync.require("./actions/add-reaction")
-/** @type {import("./actions/redact")} */
-const redact = sync.require("./actions/redact")
-/** @type {import("../matrix/matrix-command-handler")} */
-const matrixCommandHandler = sync.require("../matrix/matrix-command-handler")
-/** @type {import("./converters/utils")} */
-const utils = sync.require("./converters/utils")
-/** @type {import("../matrix/api")}) */
-const api = sync.require("../matrix/api")
-/** @type {import("../matrix/read-registration")}) */
-const reg = sync.require("../matrix/read-registration")
-
-let lastReportedEvent = 0
-
-function guard(type, fn) {
-	return async function(event, ...args) {
-		try {
-			return await fn(event, ...args)
-		} catch (e) {
-			console.error("hit event-dispatcher's error handler with this exception:")
-			console.error(e) // TODO: also log errors into a file or into the database, maybe use a library for this? or just wing it?
-			console.error(`while handling this ${type} gateway event:`)
-			console.dir(event, {depth: null})
-
-			if (Date.now() - lastReportedEvent < 5000) return
-			lastReportedEvent = Date.now()
-
-			let stackLines = e.stack.split("\n")
-			api.sendEvent(event.room_id, "m.room.message", {
-				msgtype: "m.text",
-				body: "\u26a0 Matrix event not delivered to Discord. See formatted content for full details.",
-				format: "org.matrix.custom.html",
-				formatted_body: "\u26a0 <strong>Matrix event not delivered to Discord</strong>"
-					+ `<br>Event type: ${type}`
-					+ `<br>${e.toString()}`
-					+ `<br><details><summary>Error trace</summary>`
-					+ `<pre>${stackLines.join("\n")}</pre></details>`
-					+ `<details><summary>Original payload</summary>`
-					+ `<pre>${util.inspect(event, false, 4, false)}</pre></details>`,
-				"moe.cadence.ooye.error": {
-					source: "matrix",
-					payload: event
-				},
-				"m.mentions": {
-					user_ids: ["@cadence:cadence.moe"]
-				}
-			})
-		}
-	}
-}
-
-async function retry(roomID, eventID) {
-	const event = await api.getEvent(roomID, eventID)
-	const error = event.content["moe.cadence.ooye.error"]
-	if (event.sender !== `@${reg.sender_localpart}:${reg.ooye.server_name}` || !error) return
-	if (error.source === "matrix") {
-		as.emit("type:" + error.payload.type, error.payload)
-	} else if (error.source === "discord") {
-		discord.cloud.emit("event", error.payload)
-	}
-}
-
-sync.addTemporaryListener(as, "type:m.room.message", guard("m.room.message",
-/**
- * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File} event it is a m.room.message because that's what this listener is filtering for
- */
-async event => {
-	if (utils.eventSenderIsFromDiscord(event.sender)) return
-	const messageResponses = await sendEvent.sendEvent(event)
-	if (event.type === "m.room.message" && event.content.msgtype === "m.text") {
-		// @ts-ignore
-		await matrixCommandHandler.execute(event)
-	}
-}))
-
-sync.addTemporaryListener(as, "type:m.sticker", guard("m.sticker",
-/**
- * @param {Ty.Event.Outer_M_Sticker} event it is a m.sticker because that's what this listener is filtering for
- */
-async event => {
-	if (utils.eventSenderIsFromDiscord(event.sender)) return
-	const messageResponses = await sendEvent.sendEvent(event)
-}))
-
-sync.addTemporaryListener(as, "type:m.reaction", guard("m.reaction",
-/**
- * @param {Ty.Event.Outer<Ty.Event.M_Reaction>} event it is a m.reaction because that's what this listener is filtering for
- */
-async event => {
-	if (utils.eventSenderIsFromDiscord(event.sender)) return
-	if (event.content["m.relates_to"].key === "🔁") {
-		// Try to bridge a failed event again?
-		await retry(event.room_id, event.content["m.relates_to"].event_id)
-	} else {
-		matrixCommandHandler.onReactionAdd(event)
-		await addReaction.addReaction(event)
-	}
-}))
-
-sync.addTemporaryListener(as, "type:m.room.redaction", guard("m.room.redaction",
-/**
- * @param {Ty.Event.Outer_M_Room_Redaction} event it is a m.room.redaction because that's what this listener is filtering for
- */
-async event => {
-	if (utils.eventSenderIsFromDiscord(event.sender)) return
-	await redact.handle(event)
-}))
-
-sync.addTemporaryListener(as, "type:m.room.avatar", guard("m.room.avatar",
-/**
- * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Avatar>} event
- */
-async event => {
-	if (event.state_key !== "") return
-	if (utils.eventSenderIsFromDiscord(event.sender)) return
-	const url = event.content.url || null
-	db.prepare("UPDATE channel_room SET custom_avatar = ? WHERE room_id = ?").run(url, event.room_id)
-}))
-
-sync.addTemporaryListener(as, "type:m.room.name", guard("m.room.name",
-/**
- * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Name>} event
- */
-async event => {
-	if (event.state_key !== "") return
-	if (utils.eventSenderIsFromDiscord(event.sender)) return
-	const name = event.content.name || null
-	db.prepare("UPDATE channel_room SET nick = ? WHERE room_id = ?").run(name, event.room_id)
-}))
-
-sync.addTemporaryListener(as, "type:m.room.member", guard("m.room.member",
-/**
- * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Member>} event
- */
-async event => {
-	if (event.state_key[0] !== "@") return
-	if (utils.eventSenderIsFromDiscord(event.state_key)) return
-	db.prepare("REPLACE INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(event.room_id, event.state_key, event.content.displayname || null, event.content.avatar_url || null)
-}))

matrix/appservice.js

@@ -1,8 +0,0 @@
-const reg = require("../matrix/read-registration")
-const AppService = require("matrix-appservice").AppService
-const as = new AppService({
-	homeserverToken: reg.hs_token
-})
-as.listen(+(new URL(reg.url).port))
-
-module.exports = as

matrix/mreq.js

@@ -1,85 +0,0 @@
-// @ts-check
-
-const fetch = require("node-fetch").default
-const mixin = require("mixin-deep")
-const stream = require("stream")
-const getStream = require("get-stream")
-
-const passthrough = require("../passthrough")
-const { sync } = passthrough
-/** @type {import("./read-registration")} */
-const reg = sync.require("./read-registration.js")
-
-const baseUrl = `${reg.ooye.server_origin}/_matrix`
-
-class MatrixServerError extends Error {
-	constructor(data, opts) {
-		super(data.error || data.errcode)
-		this.data = data
-		/** @type {string} */
-		this.errcode = data.errcode
-		this.opts = opts
-	}
-}
-
-/**
- * @param {string} method
- * @param {string} url
- * @param {any} [body]
- * @param {any} [extra]
- */
-async function mreq(method, url, body, extra = {}) {
-	if (body == undefined || Object.is(body.constructor, Object)) {
-		body = JSON.stringify(body)
-	} else if (body instanceof stream.Readable && reg.ooye.content_length_workaround) {
-		body = await getStream.buffer(body)
-	}
-
-	const opts = mixin({
-		method,
-		body,
-		headers: {
-			Authorization: `Bearer ${reg.as_token}`
-		}
-	}, extra)
-
-	// console.log(baseUrl + url, opts)
-	const res = await fetch(baseUrl + url, opts)
-	const root = await res.json()
-
-	if (!res.ok || root.errcode) {
-		if (root.error?.includes("Content-Length")) {
-			console.error(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:`
-				+ `\n  * Run an nginx reverse proxy to Synapse, and point registration.yaml's`
-				+ `\n    \`server_origin\` to nginx`
-				+ `\n  * Set \`content_length_workaround: true\` in registration.yaml (this will`
-				+ `\n    halve the speed of bridging d->m files)`)
-			throw new Error("Synapse is not accepting stream uploads, see the message above.")
-		}
-		delete opts.headers.Authorization
-		throw new MatrixServerError(root, {baseUrl, url, ...opts})
-	}
-	return root
-}
-
-/**
- * JavaScript doesn't have Racket-like parameters with dynamic scoping, so
- * do NOT do anything else at the same time as this.
- * @template T
- * @param {string} token
- * @param {(...arg: any[]) => Promise<T>} callback
- * @returns {Promise<T>}
- */
-async function withAccessToken(token, callback) {
-	const prevToken = reg.as_token
-	reg.as_token = token
-	try {
-		return await callback()
-	} finally {
-		reg.as_token = prevToken
-	}
-}
-
-module.exports.MatrixServerError = MatrixServerError
-module.exports.mreq = mreq
-module.exports.withAccessToken = withAccessToken

matrix/read-registration.js

@@ -1,14 +0,0 @@
-// @ts-check
-
-const fs = require("fs")
-const assert = require("assert").strict
-const yaml = require("js-yaml")
-
-/** @ts-ignore @type {import("../types").AppServiceRegistrationConfig} */
-const reg = yaml.load(fs.readFileSync("registration.yaml", "utf8"))
-reg["ooye"].invite = (reg.ooye.invite || []).filter(mxid => mxid.endsWith(`:${reg.ooye.server_name}`)) // one day I will understand why typescript disagrees with dot notation on this line
-assert(reg.ooye.max_file_size)
-assert(reg.ooye.namespace_prefix)
-assert(reg.ooye.server_name)
-
-module.exports = reg

matrix/read-registration.test.js

@@ -1,10 +0,0 @@
-const {test} = require("supertape")
-const reg = require("./read-registration")
-
-test("reg: has necessary parameters", t => {
-	const propertiesToCheck = ["sender_localpart", "id", "as_token", "ooye"]
-	t.deepEqual(
-		propertiesToCheck.filter(p => p in reg),
-		propertiesToCheck
-	)
-})

package.json

@@ -1,6 +1,6 @@
 {
   "name": "out-of-your-element",
-  "version": "1.1.1",
+  "version": "3.0.0",
   "description": "A bridge between Matrix and Discord",
   "main": "index.js",
   "repository": {
@@ -14,46 +14,57 @@
   ],
   "author": "Cadence, PapiOphidian",
   "license": "AGPL-3.0-or-later",
+  "engines": {
+    "node": ">=20"
+  },
   "dependencies": {
     "@chriscdn/promise-semaphore": "^2.0.1",
-    "better-sqlite3": "^9.0.0",
+    "@cloudrac3r/discord-markdown": "^2.6.5",
+    "@cloudrac3r/giframe": "^0.4.3",
+    "@cloudrac3r/html-template-tag": "^5.0.1",
+    "@cloudrac3r/in-your-element": "^1.1.0",
+    "@cloudrac3r/mixin-deep": "^3.0.1",
+    "@cloudrac3r/pngjs": "^7.0.3",
+    "@cloudrac3r/pug": "^4.0.4",
+    "@cloudrac3r/turndown": "^7.1.4",
+    "@stackoverflow/stacks": "^2.5.4",
+    "@stackoverflow/stacks-icons": "^6.0.2",
+    "ansi-colors": "^4.1.3",
+    "better-sqlite3": "^11.1.2",
     "chunk-text": "^2.0.1",
-    "cloudstorm": "^0.10.8",
-    "deep-equal": "^2.2.3",
-    "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#2881b447954fcea10510f212fa4c1dbbdc0a57a3",
-    "entities": "^4.5.0",
-    "get-stream": "^6.0.1",
-    "giframe": "github:cloudrac3r/giframe#v0.4.1",
-    "heatsync": "^2.4.1",
-    "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0",
-    "js-yaml": "^4.1.0",
-    "matrix-appservice": "^2.0.0",
+    "cloudstorm": "^0.12.0",
+    "discord-api-types": "^0.37.119",
+    "domino": "^2.1.6",
+    "enquirer": "^2.4.1",
+    "entities": "^5.0.0",
+    "get-relative-path": "^1.0.2",
+    "h3": "^1.15.1",
+    "heatsync": "^2.7.2",
+    "htmx.org": "^2.0.4",
+    "lru-cache": "^11.0.2",
     "minimist": "^1.2.8",
-    "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
-    "node-fetch": "^2.6.7",
-    "pngjs": "github:cloudrac3r/pngjs#v7.0.2",
     "prettier-bytes": "^1.0.4",
-    "sharp": "^0.32.6",
-    "snowtransfer": "^0.10.5",
+    "sharp": "^0.33.4",
+    "snowtransfer": "^0.13.1",
     "stream-mime-type": "^1.0.2",
     "try-to-catch": "^3.0.1",
-    "turndown": "^7.1.2",
-    "xxhash-wasm": "^1.0.2"
+    "uqr": "^0.1.2",
+    "xxhash-wasm": "^1.0.2",
+    "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@types/node": "^18.16.0",
-    "@types/node-fetch": "^2.6.3",
-    "c8": "^8.0.1",
-    "colorette": "^1.4.0",
+    "@cloudrac3r/tap-dot": "^2.0.3",
+    "@types/node": "^20.17.19",
+    "c8": "^10.1.2",
     "cross-env": "^7.0.3",
-    "discord-api-types": "^0.37.60",
-    "supertape": "^10.3.0",
-    "tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d"
+    "supertape": "^10.4.0"
   },
   "scripts": {
+    "start": "node --enable-source-maps start.js",
+    "setup": "node --enable-source-maps scripts/setup.js",
     "addbot": "node addbot.js",
-    "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap test/test.js | tap-dot",
+    "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js | tap-dot",
     "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot",
-    "cover": "c8 --skip-full -x db/migrations -x matrix/file.js -x matrix/api.js -x matrix/mreq.js -x d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow"
+    "cover": "c8 -o test/coverage --skip-full -x db/migrations -x src/matrix/file.js -x src/matrix/api.js -x src/d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow"
   }
 }

readme.md

@@ -6,15 +6,13 @@ Modern Matrix-to-Discord appservice bridge, created by [@cadence:cadence.moe](ht
 
 [![Releases](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=plastic&color=green)](https://gitdab.com/cadence/out-of-your-element/releases) [![Discuss on Matrix](https://img.shields.io/badge/discuss-%23out--of--your--element-white?style=plastic)](https://matrix.to/#/#out-of-your-element:cadence.moe)
 
-## Docs
-
-This readme has the most important info. The rest is [in the docs folder.](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs)
+![](https://cadence.moe/i/f42a3f)
 
 ## Why a new bridge?
 
 * Modern: Supports new Discord features like replies, threads and stickers, and new Matrix features like edits, spaces and space membership.
-* Efficient: Special attention has been given to memory usage, database indexes, disk footprint, runtime algorithms, and queries to the homeserver.
-* Reliable: Any errors on either side are notified on Matrix and can be retried.
+* Efficient: Special attention has been given to memory usage, database indexes, disk footprint, runtime algorithms, and queries to the homeserver. [Efficiency details.](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/developer-orientation.md)
+* Reliable: Any errors on either side are notified on Matrix and can be retried. Messages sent during bridge downtime will still be bridged after it comes back up.
 * Tested: A test suite and code coverage make sure all the logic and special cases work.
 * Simple development: No build step (it's JavaScript, not TypeScript), minimal/lightweight dependencies, and abstraction only where necessary so that less background knowledge is required. No need to learn about Intents or library functions.
 * No locking algorithm: Other bridges use a locking algorithm which is a source of frequent bugs. This bridge avoids the need for one.
@@ -22,27 +20,17 @@ This readme has the most important info. The rest is [in the docs folder.](https
 
 ## What works?
 
-Most features you'd expect in both directions, plus a little extra spice:
+Most features you'd expect in both directions: messages, edits, deletions, formatting (including spoilers), reactions, custom emojis, custom emoji reactions, mentions, channel mentions, replies, threads, stickers (all formats: PNG, APNG, GIF, Lottie), attachments, spoiler attachments (compatible with most clients), embeds, URL previews, presence, discord.com hyperlinks, and more.
 
-* Messages
-* Edits
-* Deletions
-* Text formatting, including spoilers
-* Reactions
-* Mentions
-* Replies
-* Threads
-* Stickers (all formats: PNG, APNG, GIF, and Lottie)
-* Attachments
-* Spoiler attachments
-* Embeds
-* Guild-Space details syncing
-* Channel-Room details syncing
-* Custom emoji list syncing
-* Custom emojis in messages
-* Custom room names/avatars can be applied on Matrix-side
-* Larger files from Discord are linked instead of reuploaded to Matrix
+Metadata is also synced: people's names, avatars, usernames; channel names, icons, topics; spaces containing rooms; custom emoji lists. Syncing Matrix rooms, room icons, and topics is optional: you can keep them different from the Discord ones if you prefer.
+
+I've also added some interesting features that I haven't seen in any other bridge:
+
+* Members using the PluralKit bot each get their own persistent accounts
+* Replies from PluralKit members are restyled into native Matrix replies
 * Simulated user accounts are named @the_persons_username rather than @112233445566778899
+* Matrix custom emojis from private rooms are still visible on Discord as a sprite sheet
+* To save space, larger files from Discord are linked instead of reuploaded to Matrix (links don't expire)
 
 For more information about features, [see the user guide.](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/user-guide.md)
 
@@ -51,136 +39,6 @@ For more information about features, [see the user guide.](https://gitdab.com/ca
 * This bridge is not designed for puppetting.
 * Direct Messaging is not supported until I figure out a good way of doing it.
 
-## Efficiency details
+## Get started!
 
-Using WeatherStack as a thin layer between the bridge application and the Discord API lets us control exactly what data is cached in memory. Only necessary information is cached. For example, member data, user data, message content, and past edits are never stored in memory. This keeps the memory usage low and also prevents it ballooning in size over the bridge's runtime.
-
-The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 100,000 messages require 16.1 MB of storage space in the SQLite database.
-
-Only necessary data and columns are queried from the database. We only contact the homeserver API if the database doesn't contain what we need.
-
-File uploads (like avatars from bridged members) are checked locally and deduplicated. Only brand new files are uploaded to the homeserver. This saves loads of space in the homeserver's media repo, especially for Synapse.
-
-Switching to [WAL mode](https://www.sqlite.org/wal.html) could improve your database access speed even more. Run `node scripts/wal.js` if you want to switch to WAL mode. This will also enable `synchronous = NORMAL`.
-
-# Setup
-
-If you get stuck, you're welcome to message [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) or [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) to ask for help setting up OOYE!
-
-You'll need:
-
-* Administrative access to a homeserver
-* Discord bot
-
-Follow these steps:
-
-1. [Get Node.js version 18 or later](https://nodejs.org/en/download/releases) (the version is required by the better-sqlite3 and matrix-appservice dependencies)
-
-1. Clone this repo and checkout a specific tag. (Development happens on main. Stable versions are tagged.)
-	* The latest release tag is ![](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=flat-square&label=%20&color=black).
-
-1. Install dependencies: `npm install --save-dev` (omit --save-dev if you will not run the automated tests)
-
-1. Copy `config.example.js` to `config.js` and fill in Discord token.
-
-1. Copy `registration.example.yaml` to `registration.yaml` and fill in bracketed values. You could generate each hex string with `dd if=/dev/urandom bs=32 count=1 2> /dev/null | basenc --base16 | dd conv=lcase 2> /dev/null`. Register the registration in Synapse's `homeserver.yaml` through the usual appservice installation process, then restart Synapse.
-
-1. Run `node scripts/seed.js` to check your setup and set the bot's initial state. You only need to run this once ever.
-1. Make sure the tests work by running `npm t`
-
-1. Start the bridge: `node start.js`
-
-1. Add the bot to a server - use any *one* of the following commands for an invite link:
-	* (in the REPL) `addbot`
-	* (in a chat) `//addbot`
-	* $ `node addbot.js`
-	* $ `npm run addbot`
-	* $ `./addbot.sh`
-
-Now any message on Discord will create the corresponding rooms on Matrix-side. After the rooms have been created, Matrix and Discord users can chat back and forth.
-
-To get into the rooms on your Matrix account, either add yourself to `invite` in `registration.yaml`, or use the `//invite [your mxid here]` command on Discord.
-
-# Development setup
-
-* Be sure to install dependencies with `--save-dev` so you can run the tests.
-* Any files you change will automatically be reloaded, except for `stdin.js` and `d2m/discord-*.js`.
-* If developing on a different computer to the one running the homeserver, use SSH port forwarding so that Synapse can connect on its `localhost:6693` to reach the running bridge on your computer. Example: `ssh -T -v -R 6693:localhost:6693 me@matrix.cadence.moe`
-* I recommend developing in Visual Studio Code so that the JSDoc x TypeScript annotation comments work. I don't know which other editors or language servers support annotations and type inference.
-
-## Repository structure
-
-    .
-    * Run this to start the bridge:
-    ├── start.js
-    * Runtime configuration, like tokens and user info:
-    ├── config.js
-    ├── registration.yaml
-    * The bridge's SQLite database is stored here:
-    ├── db
-    │   ├── *.sql, *.db
-    │   * Migrations change the database schema when you update to a newer version of OOYE:
-    │   └── migrations
-    │       └── *.sql, *.js
-    * Discord-to-Matrix bridging:
-    ├── d2m
-    │   * Execute actions through the whole flow, like sending a Discord message to Matrix:
-    │   ├── actions
-    │   │   └── *.js
-    │   * Convert data from one form to another without depending on bridge state. Called by actions:
-    │   ├── converters
-    │   │   └── *.js
-    │   * Making Discord work:
-    │   ├── discord-*.js
-    │   * Listening to events from Discord and dispatching them to the correct `action`:
-    │   └── event-dispatcher.js
-    ├── discord
-    │   └── discord-command-handler.js
-    * Matrix-to-Discord bridging:
-    ├── m2d
-    │   * Execute actions through the whole flow, like sending a Matrix message to Discord:
-    │   ├── actions
-    │   │   └── *.js
-    │   * Convert data from one form to another without depending on bridge state. Called by actions:
-    │   ├── converters
-    │   │   └── *.js
-    │   * Listening to events from Matrix and dispatching them to the correct `action`:
-    │   └── event-dispatcher.js
-    * We aren't using the matrix-js-sdk, so here are all the functions for the Matrix C-S and Appservice APIs:
-    ├── matrix
-    │   └── *.js
-    * Various files you can run once if you need them.
-    ├── scripts
-    │   * First time running a new bridge? Run this file to plant a seed, which will flourish into state for the bridge:
-    │   ├── seed.js
-    │   * Hopefully you won't need the rest of these. Code quality varies wildly.
-    │   └── *.js
-    * You are here! :)
-    └── readme.md
-
-## Dependency justification
-
-(deduped transitive dependency count) dependency name: explanation
-
-* (0) @chriscdn/promise-semaphore: It does what I want! I like it!
-* (42) better-sqlite3: SQLite3 is the best database, and this is the best library for it. Really! I love it.
-* (1) chunk-text: It does what I want.
-* (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
-* (8) snowtransfer: Discord API library with bring-your-own-caching that I trust.
-* (0) deep-equal: It's already pulled in by supertape.
-* (1) discord-markdown: This is my fork!
-* (0) get-stream: Only needed if content_length_workaround is true.
-* (0) giframe: This is my fork!
-* (1) heatsync: Module hot-reloader that I trust.
-* (0) entities: Looks fine. No dependencies.
-* (1) html-template-tag: This is my fork!
-* (1) js-yaml: It seems to do what I want, and it's already pulled in by matrix-appservice.
-* (70) matrix-appservice: I wish it didn't pull in express :(
-* (0) minimist: It's already pulled in by better-sqlite3->prebuild-install.
-* (0) mixin-deep: This is my fork! (It fixes a bug in regular mixin-deep.)
-* (3) node-fetch@2: I like it and it does what I want.
-* (0) pngjs: Lottie stickers are converted to bitmaps with the vendored Rlottie WASM build, then the bitmaps are converted to PNG with pngjs.
-* (0) prettier-bytes: It does what I want and has no dependencies.
-* (51) sharp: Jimp has fewer dependencies, but sharp is faster.
-* (0) try-to-catch: Not strictly necessary, but it does what I want and has no dependencies.
-* (1) turndown: I need an HTML-to-Markdown converter and this one looked suitable enough. It has some bugs that I've worked around, so I might switch away from it later.
+[Read the installation instructions →](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md)

registration.example.yaml

@@ -1,24 +0,0 @@
-id: de8c56117637cb5d9f4ac216f612dc2adb1de4c09ae8d13553f28c33a28147c7
-hs_token: [a unique 64 character hex string]
-as_token: [a unique 64 character hex string]
-url: http://localhost:6693
-sender_localpart: _ooye_bot
-protocols:
-  - discord
-namespaces:
-  users:
-    - exclusive: true
-      regex: '@_ooye_.*'
-  aliases:
-    - exclusive: true
-      regex: '#_ooye_.*'
-rate_limited: false
-ooye:
-  namespace_prefix: _ooye_
-  max_file_size: 5000000
-  server_name: [the part after the colon in your matrix id, like cadence.moe]
-  server_origin: [the full protocol and domain of your actual matrix server's location, with no trailing slash, like https://matrix.cadence.moe]
-  content_length_workaround: false
-  invite:
-    # uncomment this to auto-invite the named user to newly created spaces and mark them as admin (PL 100) everywhere
-    # - '@cadence:cadence.moe'

scripts/backfill.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+// @ts-check
+
+console.log("-=- This script is experimental. It WILL mess up the room history on Matrix. -=-")
+console.log()
+
+const {channel: channelID} = require("minimist")(process.argv.slice(2), {string: ["channel"]})
+if (!channelID) {
+	console.error("Usage: ./scripts/backfill.js --channel=<channel id here>")
+	process.exit(1)
+}
+
+const assert = require("assert/strict")
+const sqlite = require("better-sqlite3")
+const backfill = new sqlite("scripts/backfill.db")
+backfill.prepare("CREATE TABLE IF NOT EXISTS backfill (channel_id TEXT NOT NULL, message_id INTEGER NOT NULL, PRIMARY KEY (channel_id, message_id))").run()
+
+const HeatSync = require("heatsync")
+
+const {reg} = require("../src/matrix/read-registration")
+const passthrough = require("../src/passthrough")
+
+const sync = new HeatSync({watchFS: false})
+const db = new sqlite("ooye.db")
+Object.assign(passthrough, {sync, db})
+
+const DiscordClient = require("../src/d2m/discord-client")
+
+const discord = new DiscordClient(reg.ooye.discord_token, "half")
+passthrough.discord = discord
+
+const orm = sync.require("../src/db/orm")
+passthrough.from = orm.from
+passthrough.select = orm.select
+
+/** @type {import("../src/d2m/event-dispatcher")}*/
+const eventDispatcher = sync.require("../src/d2m/event-dispatcher")
+
+const roomID = passthrough.select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
+if (!roomID) {
+	console.error("Please choose a channel that's already bridged.")
+	process.exit(1)
+}
+
+;(async () => {
+	await discord.cloud.connect()
+	console.log("Connected, waiting for data about requested channel...")
+
+	discord.cloud.on("event", event)
+})()
+
+const preparedInsert = backfill.prepare("INSERT INTO backfill (channel_id, message_id) VALUES (?, ?)")
+
+async function event(event) {
+	if (event.t !== "GUILD_CREATE") return
+	const channel = event.d.channels.find(c => c.id === channelID)
+	if (!channel) return
+	const guild_id = event.d.id
+
+	let last = backfill.prepare("SELECT cast(max(message_id) as TEXT) FROM backfill WHERE channel_id = ?").pluck().get(channelID) || "0"
+	console.log(`OK, processing messages for #${channel.name}, continuing from ${last}`)
+
+	while (last) {
+		const messages = await discord.snow.channel.getChannelMessages(channelID, {limit: 50, after: String(last)})
+		messages.reverse() // More recent messages come first -> More recent messages come last
+		for (const message of messages) {
+			const simulatedGatewayDispatchData = {
+				guild_id,
+				backfill: true,
+				...message
+			}
+			await eventDispatcher.onMessageCreate(discord, simulatedGatewayDispatchData)
+			preparedInsert.run(channelID, message.id)
+		}
+		last = messages.at(-1)?.id
+	}
+
+	process.exit()
+}

scripts/capture-message-update-events.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 // @ts-check
 
 // ****
@@ -16,16 +17,16 @@ function fieldToPresenceValue(field) {
 const sqlite = require("better-sqlite3")
 const HeatSync = require("heatsync")
 
-const config = require("../config")
-const passthrough = require("../passthrough")
+const {reg} = require("../src/matrix/read-registration")
+const passthrough = require("../src/passthrough")
 
 const sync = new HeatSync({watchFS: false})
 
-Object.assign(passthrough, {config, sync})
+Object.assign(passthrough, {sync})
 
-const DiscordClient = require("../d2m/discord-client")
+const DiscordClient = require("../src/d2m/discord-client")
 
-const discord = new DiscordClient(config.discordToken, "no")
+const discord = new DiscordClient(reg.ooye.discord_token, "no")
 passthrough.discord = discord
 
 ;(async () => {
@@ -37,7 +38,7 @@ passthrough.discord = discord
 })()
 
 const events = new sqlite("scripts/events.db")
-const sql = "INSERT INTO \"update\" (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")"
+const sql = "INSERT INTO update_event (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")"
 console.log(sql)
 const prepared = events.prepare(sql)
 

scripts/check-migrate.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 // @ts-check
 
 // Trigger the database migration flow and exit after committing.
@@ -5,11 +6,10 @@
 
 const sqlite = require("better-sqlite3")
 
-const config = require("../config")
-const passthrough = require("../passthrough")
-const db = new sqlite("db/ooye.db")
-const migrate = require("../db/migrate")
+const passthrough = require("../src/passthrough")
+const db = new sqlite("ooye.db")
+const migrate = require("../src/db/migrate")
 
-Object.assign(passthrough, {config, db })
+Object.assign(passthrough, {db})
 
 migrate.migrate(db)

scripts/emoji-surrogates-statistics.js

@@ -0,0 +1,77 @@
+// @ts-check
+
+const fs = require("fs")
+const {join} = require("path")
+const s = fs.readFileSync(join(__dirname, "..", "src", "m2d", "converters", "emojis.txt"), "utf8").split("\n").map(x => encodeURIComponent(x))
+const searchPattern = "%EF%B8%8F"
+
+/**
+ * adapted from es.map.group-by.js in core-js
+ * @template K,V
+ * @param {V[]} items
+ * @param {(item: V) => K} fn
+ * @returns {Map<K, V[]>}
+ */
+function groupBy(items, fn) {
+	var map = new Map();
+	for (const value of items) {
+		var key = fn(value);
+		if (!map.has(key)) map.set(key, [value]);
+		else map.get(key).push(value);
+	}
+	return map;
+}
+
+/**
+ * @param {number[]} items
+ * @param {number} width
+ */
+function xhistogram(items, width) {
+	const chars = " ▏▎▍▌▋▊▉"
+	const max = items.reduce((a, c) => c > a ? c : a, 0)
+	return items.map(v => {
+		const p = v / max * (width-1)
+		return (
+			Array(Math.floor(p)).fill("█").join("") /* whole part */
+			+ chars[Math.ceil((p % 1) * (chars.length-1))] /* decimal part */
+		).padEnd(width)
+	})
+}
+
+/**
+ * @param {number[]} items
+ * @param {[number, number]} xrange
+ */
+function yhistogram(items, xrange, printHeader = false) {
+	const chars = "░▁_▂▃▄▅▆▇█"
+	const ones = "₀₁₂₃₄₅₆₇₈₉"
+	const tens = "0123456789"
+	const xy = []
+	let max = 0
+	/** value (x) -> frequency (y) */
+	const grouped = groupBy(items, x => x)
+	for (let i = xrange[0]; i <= xrange[1]; i++) {
+		if (printHeader) {
+			if (i === -1) process.stdout.write("-")
+			else if (i.toString().at(-1) === "0") process.stdout.write(tens[i/10])
+			else process.stdout.write(ones[i%10])
+		}
+		const y = grouped.get(i)?.length ?? 0
+		if (y > max) max = y
+		xy.push(y)
+	}
+	if (printHeader) console.log()
+	return xy.map(y => chars[Math.ceil(y / max * (chars.length-1))]).join("")
+}
+
+const grouped = groupBy(s, x => x.length)
+const sortedGroups = [...grouped.entries()].sort((a, b) => b[0] - a[0])
+let length = 0
+const lengthHistogram = xhistogram(sortedGroups.map(v => v[1].length), 10)
+for (let i = 0; i < sortedGroups.length; i++) {
+	const [k, v] = sortedGroups[i]
+	const l = lengthHistogram[i]
+	const h = yhistogram(v.map(x => x.indexOf(searchPattern)), [-1, k - searchPattern.length], i === 0)
+	if (i === 0) length = h.length + 1
+	console.log(`${h.padEnd(length, i % 2 === 0 ? "⸱" : " ")}length ${k.toString().padEnd(3)} ${l} ${v.length}`)
+}

scripts/migrate-from-old-bridge.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 // @ts-check
 
 const assert = require("assert").strict
@@ -6,19 +7,17 @@ const Semaphore = require("@chriscdn/promise-semaphore")
 const sqlite = require("better-sqlite3")
 const HeatSync = require("heatsync")
 
-const config = require("../config")
-const passthrough = require("../passthrough")
+const passthrough = require("../src/passthrough")
 
 const sync = new HeatSync({watchFS: false})
 
-/** @type {import("../matrix/read-registration")} */
-const reg = sync.require("../matrix/read-registration")
+const {reg} = require("../src/matrix/read-registration")
 assert(reg.old_bridge)
 const oldAT = reg.old_bridge.as_token
 const newAT = reg.as_token
 
 const oldDB = new sqlite(reg.old_bridge.database)
-const db = new sqlite("db/ooye.db")
+const db = new sqlite("ooye.db")
 
 db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration (
 	discord_channel	TEXT NOT NULL,
@@ -26,19 +25,19 @@ db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration (
 	PRIMARY KEY("discord_channel")
 ) WITHOUT ROWID;`)
 
-Object.assign(passthrough, {config, sync, db})
+Object.assign(passthrough, {sync, db})
 
-const DiscordClient = require("../d2m/discord-client")
-const discord = new DiscordClient(config.discordToken, "half")
+const DiscordClient = require("../src/d2m/discord-client")
+const discord = new DiscordClient(reg.ooye.discord_token, "half")
 passthrough.discord = discord
 
-/** @type {import("../d2m/actions/create-space")} */
+/** @type {import("../src/d2m/actions/create-space")} */
 const createSpace = sync.require("../d2m/actions/create-space")
-/** @type {import("../d2m/actions/create-room")} */
+/** @type {import("../src/d2m/actions/create-room")} */
 const createRoom = sync.require("../d2m/actions/create-room")
-/** @type {import("../matrix/mreq")} */
+/** @type {import("../src/matrix/mreq")} */
 const mreq = sync.require("../matrix/mreq")
-/** @type {import("../matrix/api")} */
+/** @type {import("../src/matrix/api")} */
 const api = sync.require("../matrix/api")
 
 const sema = new Semaphore()
@@ -116,7 +115,7 @@ async function migrateGuild(guild) {
 			// (By the way, thread_parent is always null here because thread rooms would never be migrated because they are not in the old bridge.)
 			db.transaction(() => {
 				db.prepare("DELETE FROM channel_room WHERE channel_id = ?").run(channel.id)
-				db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, custom_avatar) VALUES (?, ?, ?, ?, ?)").run(channel.id, row.matrix_id, channel.name, preMigrationRow.nick, preMigrationRow.custom_avatar)
+				db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, custom_avatar, guild_id) VALUES (?, ?, ?, ?, ?, ?)").run(channel.id, row.matrix_id, channel.name, preMigrationRow.nick, preMigrationRow.custom_avatar, guild.id)
 				console.log(`-- -- Added to database (transferred properties from previous OOYE room)`)
 			})()
 		} else {

scripts/register.js

@@ -1,22 +0,0 @@
-// @ts-check
-
-const { AppServiceRegistration } = require("matrix-appservice");
-
-let id = AppServiceRegistration.generateToken()
-try {
-	const reg = require("../matrix/read-registration")
-	if (reg.id) id = reg.id
-} catch (e) {}
-
-// creating registration files
-const newReg = new AppServiceRegistration(null);
-newReg.setAppServiceUrl("http://localhost:6693");
-newReg.setId(id);
-newReg.setHomeserverToken(AppServiceRegistration.generateToken());
-newReg.setAppServiceToken(AppServiceRegistration.generateToken());
-newReg.setSenderLocalpart("_ooye_bot");
-newReg.addRegexPattern("users", "@_ooye_.*", true);
-newReg.addRegexPattern("aliases", "#_ooye_.*", true);
-newReg.setProtocols(["discord"]); // For 3PID lookups
-newReg.setRateLimited(false);
-newReg.outputAsYaml("registration.yaml");

scripts/remove-old-bridged-users.js

@@ -0,0 +1,39 @@
+// @ts-check
+
+const HeatSync = require("heatsync")
+const sync = new HeatSync({watchFS: false})
+
+const sqlite = require("better-sqlite3")
+const db = new sqlite("db/ooye.db")
+
+const passthrough = require("../src/passthrough")
+Object.assign(passthrough, {db, sync})
+
+const api = require("../src/matrix/api")
+const mreq = require("../src/matrix/mreq")
+
+const rooms = db.prepare("select room_id from channel_room").pluck().all()
+
+;(async () => {
+	// Step 5: Kick users starting with @_discord_
+	await mreq.withAccessToken("baby", async () => {
+		for (const roomID of rooms) {
+			try {
+				const members = await api.getJoinedMembers(roomID)
+				for (const mxid of Object.keys(members.joined)) {
+					if (mxid.startsWith("@_discord_") && !mxid.startsWith("@_discord_bot")) {
+						await api.leaveRoom(roomID, mxid)
+					}
+				}
+				await api.setUserPower(roomID, "@_discord_bot:cadence.moe", 0)
+				await api.leaveRoom(roomID)
+			} catch (e) {
+				if (e.message.includes("Appservice not in room")) {
+					// ok
+				} else {
+					throw e
+				}
+			}
+		}
+	})
+})()

scripts/save-channel-names-to-db.js

@@ -1,19 +1,20 @@
+#!/usr/bin/env node
 // @ts-check
 
 const sqlite = require("better-sqlite3")
 const HeatSync = require("heatsync")
 
-const config = require("../config")
-const passthrough = require("../passthrough")
-const db = new sqlite("db/ooye.db")
+const {reg} = require("../src/matrix/read-registration")
+const passthrough = require("../src/passthrough")
+const db = new sqlite("ooye.db")
 
 const sync = new HeatSync({watchFS: false})
 
-Object.assign(passthrough, {config, sync, db})
+Object.assign(passthrough, {sync, db})
 
-const DiscordClient = require("../d2m/discord-client")
+const DiscordClient = require("../src/d2m/discord-client")
 
-const discord = new DiscordClient(config.discordToken, "no")
+const discord = new DiscordClient(reg.ooye.discord_token, "no")
 passthrough.discord = discord
 
 ;(async () => {

scripts/save-event-types-to-db.js

@@ -1,16 +1,17 @@
+#!/usr/bin/env node
 // @ts-check
 
 const sqlite = require("better-sqlite3")
 const HeatSync = require("heatsync")
 
-const passthrough = require("../passthrough")
-const db = new sqlite("db/ooye.db")
+const passthrough = require("../src/passthrough")
+const db = new sqlite("ooye.db")
 
 const sync = new HeatSync({watchFS: false})
 
 Object.assign(passthrough, {sync, db})
 
-const api = require("../matrix/api")
+const api = require("../src/matrix/api")
 
 /** @type {{event_id: string, room_id: string, event_type: string}[]} */ // @ts-ignore
 const rows = db.prepare("SELECT event_id, room_id, event_type FROM event_message INNER JOIN message_channel USING (message_id) INNER JOIN channel_room USING (channel_id)").all()

scripts/seed.js

@@ -1,136 +0,0 @@
-// @ts-check
-
-console.log("This could take up to 30 seconds. Please be patient.")
-
-const assert = require("assert").strict
-const fs = require("fs")
-const sqlite = require("better-sqlite3")
-const HeatSync = require("heatsync")
-
-const args = require("minimist")(process.argv.slice(2), {string: ["emoji-guild"]})
-
-const config = require("../config")
-const passthrough = require("../passthrough")
-const db = new sqlite("db/ooye.db")
-const migrate = require("../db/migrate")
-
-const sync = new HeatSync({watchFS: false})
-
-Object.assign(passthrough, { sync, config, db })
-
-const orm = sync.require("../db/orm")
-passthrough.from = orm.from
-passthrough.select = orm.select
-
-const DiscordClient = require("../d2m/discord-client")
-const discord = new DiscordClient(config.discordToken, "no")
-passthrough.discord = discord
-
-const api = require("../matrix/api")
-const file = require("../matrix/file")
-const reg = require("../matrix/read-registration")
-const utils = require("../m2d/converters/utils")
-
-function die(message) {
-	console.error(message)
-	process.exit(1)
-}
-
-async function uploadAutoEmoji(guild, name, filename) {
-	let emoji = guild.emojis.find(e => e.name === name)
-	if (!emoji) {
-		console.log(`   Uploading ${name}...`)
-		const data = fs.readFileSync(filename, null)
-		emoji = await discord.snow.guildAssets.createEmoji(guild.id, {name, image: "data:image/png;base64," + data.toString("base64")})
-	} else {
-		console.log(`   Reusing ${name}...`)
-	}
-	db.prepare("REPLACE INTO auto_emoji (name, emoji_id, guild_id) VALUES (?, ?, ?)").run(emoji.name, emoji.id, guild.id)
-	return emoji
-}
-
-;(async () => {
-	const mxid = `@${reg.sender_localpart}:${reg.ooye.server_name}`
-
-	// ensure registration is correctly set...
-	assert(reg.sender_localpart.startsWith(reg.ooye.namespace_prefix)) // appservice's localpart must be in the namespace it controls
-	assert(utils.eventSenderIsFromDiscord(mxid)) // appservice's mxid must be in the namespace it controls
-	assert(reg.ooye.server_origin.match(/^https?:\/\//)) // must start with http or https
-	assert.notEqual(reg.ooye.server_origin.slice(-1), "/") // must not end in slash
-	console.log("✅ Configuration looks good...")
-
-	// database ddl...
-	await migrate.migrate(db)
-
-	// add initial rows to database, like adding the bot to sim...
-	db.prepare("INSERT OR IGNORE INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run("0", reg.sender_localpart.slice(reg.ooye.namespace_prefix.length), reg.sender_localpart, mxid)
-
-	console.log("✅ Database is ready...")
-
-	// ensure appservice bot user is registered...
-	try {
-		await api.register(reg.sender_localpart)
-	} catch (e) {
-		if (e.errcode === "M_USER_IN_USE" || e.data?.error === "Internal server error") {
-			// "Internal server error" is the only OK error because older versions of Synapse say this if you try to register the same username twice.
-		} else {
-			throw e
-		}
-	}
-
-	// upload initial images...
-	const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png")
-
-	console.log("✅ Matrix appservice login works...")
-
-	// upload the L1 L2 emojis to some guild
-	const emojis = db.prepare("SELECT name FROM auto_emoji WHERE name = 'L1' OR name = 'L2'").pluck().all()
-	if (emojis.length !== 2) {
-		// If an argument was supplied, always use that one
-		let guild = null
-		if (args["emoji-guild"]) {
-			if (typeof args["emoji-guild"] === "string") {
-				guild = await discord.snow.guild.getGuild(args["emoji-guild"])
-			}
-			if (!guild) return die(`Error: You asked emojis to be uploaded to guild ID ${args["emoji-guild"]}, but the bot isn't in that guild.`)
-		}
-		// Otherwise, check if we have already registered an auto emoji guild
-		if (!guild) {
-			const guildID = passthrough.select("auto_emoji", "guild_id", {name: "_"}).pluck().get()
-			if (guildID) {
-				guild = await discord.snow.guild.getGuild(guildID, false)
-			}
-		}
-		// Otherwise, check if we should create a new guild
-		if (!guild) {
-			const guilds = await discord.snow.user.getGuilds({limit: 11, with_counts: false})
-			if (guilds.length < 10) {
-				console.log("   Creating a guild for emojis...")
-				guild = await discord.snow.guild.createGuild({name: "OOYE Emojis"})
-			}
-		}
-		// Otherwise, it's the user's problem
-		if (!guild) {
-			return die(`Error: The bot needs to upload some emojis. Please say where to upload them to. Run seed.js again with --emoji-guild=GUILD_ID`)
-		}
-		// Upload those emojis to the chosen location
-		db.prepare("REPLACE INTO auto_emoji (name, emoji_id, guild_id) VALUES ('_', '_', ?)").run(guild.id)
-		await uploadAutoEmoji(guild, "L1", "docs/img/L1.png")
-		await uploadAutoEmoji(guild, "L2", "docs/img/L2.png")
-	}
-	console.log("✅ Emojis are ready...")
-
-	// set profile data on discord...
-	const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer())
-	await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")})
-	await discord.snow.requestHandler.request(`/applications/@me`, {}, "patch", "json", {description: "Powered by **Out Of Your Element**\nhttps://gitdab.com/cadence/out-of-your-element"})
-	console.log("✅ Discord profile updated...")
-
-	// set profile data on homeserver...
-	await api.profileSetDisplayname(mxid, "Out Of Your Element")
-	await api.profileSetAvatarUrl(mxid, avatarUrl)
-	console.log("✅ Matrix profile updated...")
-
-	console.log("Good to go. I hope you enjoy Out Of Your Element.")
-	process.exit()
-})()

scripts/setup.js

@@ -0,0 +1,396 @@
+#!/usr/bin/env node
+// @ts-check
+
+const assert = require("assert").strict
+const fs = require("fs")
+const sqlite = require("better-sqlite3")
+const {scheduler} = require("timers/promises")
+const {isDeepStrictEqual} = require("util")
+const {createServer} = require("http")
+const {join} = require("path")
+
+const {prompt} = require("enquirer")
+const Input = require("enquirer/lib/prompts/input")
+const {magenta, bold, cyan} = require("ansi-colors")
+const HeatSync = require("heatsync")
+const {SnowTransfer} = require("snowtransfer")
+const DiscordTypes = require("discord-api-types/v10")
+const {createApp, defineEventHandler, toNodeListener} = require("h3")
+
+const args = require("minimist")(process.argv.slice(2), {string: ["emoji-guild"]})
+
+// Move database file if it's still in the old location
+if (fs.existsSync("db")) {
+	if (fs.existsSync("db/ooye.db")) {
+		fs.renameSync("db/ooye.db", "ooye.db")
+	}
+	const files = fs.readdirSync("db")
+	if (files.length) {
+		console.error("The db folder is deprecated and must be removed. Your ooye.db database file has already been moved to the root of the repo. You must manually move or delete the remaining files:")
+		for (const file of files) {
+			console.error(file)
+		}
+		process.exit(1)
+	}
+	fs.rmSync("db", {recursive: true})
+}
+
+const passthrough = require("../src/passthrough")
+const db = new sqlite("ooye.db")
+const migrate = require("../src/db/migrate")
+
+const sync = new HeatSync({watchFS: false})
+
+Object.assign(passthrough, {sync, db})
+
+const orm = sync.require("../src/db/orm")
+passthrough.from = orm.from
+passthrough.select = orm.select
+
+let registration = require("../src/matrix/read-registration")
+let {reg, getTemplateRegistration, writeRegistration, readRegistration, checkRegistration, registrationFilePath} = registration
+
+function die(message) {
+	console.error(message)
+	process.exit(1)
+}
+
+async function uploadAutoEmoji(snow, guild, name, filename) {
+	let emoji = guild.emojis.find(e => e.name === name)
+	if (!emoji) {
+		console.log(`   Uploading ${name}...`)
+		const data = fs.readFileSync(filename, null)
+		emoji = await snow.guildAssets.createEmoji(guild.id, {name, image: "data:image/png;base64," + data.toString("base64")})
+	} else {
+		console.log(`   Reusing ${name}...`)
+	}
+	db.prepare("REPLACE INTO auto_emoji (name, emoji_id, guild_id) VALUES (?, ?, ?)").run(emoji.name, emoji.id, guild.id)
+	return emoji
+}
+
+async function suggestWellKnown(serverUrlPrompt, url, otherwise) {
+	try {
+		var json = await fetch(`${url}/.well-known/matrix/client`).then(res => res.json())
+		let baseURL = json["m.homeserver"].base_url.replace(/\/$/, "")
+		if (baseURL && baseURL !== url) {
+			serverUrlPrompt.initial = baseURL
+			return `Did you mean: ${bold(baseURL)}? (Enter to accept)`
+		}
+	} catch (e) {}
+	return otherwise
+}
+
+async function validateHomeserverOrigin(serverUrlPrompt, url) {
+	if (!url.match(/^https?:\/\//)) return "Must be a URL"
+	if (url.match(/\/$/)) return "Must not end with a slash"
+	process.stdout.write(magenta(" checking, please wait..."))
+	try {
+		var res = await fetch(`${url}/_matrix/client/versions`)
+		if (res.status !== 200) {
+			return suggestWellKnown(serverUrlPrompt, url, `There is no Matrix server at that URL (${url}/_matrix/client/versions returned ${res.status})`)
+		}
+	} catch (e) {
+		return e.message
+	}
+	try {
+		var json = await res.json()
+		if (!Array.isArray(json?.versions) || !json.versions.includes("v1.11")) {
+			return `OOYE needs Matrix version v1.11, but ${url} doesn't support this`
+		}
+	} catch (e) {
+		return suggestWellKnown(serverUrlPrompt, url, `There is no Matrix server at that URL (${url}/_matrix/client/versions is not JSON)`)
+	}
+	return true
+}
+
+function defineEchoHandler() {
+	return defineEventHandler(event => {
+		return "Out Of Your Element is listening.\n" +
+			`Received a ${event.method} request on path ${event.path}\n`
+	})
+}
+
+;(async () => {
+	// create registration file with prompts...
+	if (!reg) {
+		console.log("What is the name of your homeserver? This is the part after : in your username.")
+		/** @type {{server_name: string}} */
+		const serverNameResponse = await prompt({
+			type: "input",
+			name: "server_name",
+			message: "Homeserver name",
+			validate: serverName => !!serverName.match(/[a-z][a-z.]+[a-z]/)
+		})
+
+		console.log("What is the URL of your homeserver?")
+		const serverOriginPrompt = new Input({
+			type: "input",
+			name: "server_origin",
+			message: "Homeserver URL",
+			initial: () => `https://${serverNameResponse.server_name}`,
+			validate: url => validateHomeserverOrigin(serverOriginPrompt, url)
+		})
+		/** @type {string} */ // @ts-ignore
+		const serverOrigin = await serverOriginPrompt.run()
+
+		const app = createApp()
+		app.use(defineEchoHandler())
+		const server = createServer(toNodeListener(app))
+		await server.listen(6693)
+
+		console.log("OOYE has its own web server. It needs to be accessible on the public internet.")
+		console.log("You need to enter a public URL where you will be able to host this web server.")
+		console.log("OOYE listens on localhost:6693, so you will probably have to set up a reverse proxy.")
+		console.log("Examples: https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md#appendix")
+		console.log("Now listening on port 6693. Feel free to send some test requests.")
+		/** @type {{bridge_origin: string}} */
+		const bridgeOriginResponse = await prompt({
+			type: "input",
+			name: "bridge_origin",
+			message: "URL to reach OOYE",
+			initial: () => `https://bridge.${serverNameResponse.server_name}`,
+			validate: async url => {
+				process.stdout.write(magenta(" checking, please wait..."))
+				try {
+					const res = await fetch(url)
+					if (res.status !== 200) return `Server returned status code ${res.status}`
+					const text = await res.text()
+					if (!text.startsWith("Out Of Your Element is listening.")) return `Server does not point to OOYE`
+					return true
+				} catch (e) {
+					return e.message
+				}
+			}
+		})
+		bridgeOriginResponse.bridge_origin = bridgeOriginResponse.bridge_origin.replace(/\/+$/, "") // remove trailing slash
+
+		await server.close()
+
+		console.log("What is your Discord bot token?")
+		console.log("Go to https://discord.com/developers, create or pick an app, go to the Bot section, and reset the token.")
+		/** @type {SnowTransfer} */ // @ts-ignore
+		let snow = null
+		/** @type {{id: string, flags: number, redirect_uris: string[]}} */ // @ts-ignore
+		let client = null
+		/** @type {{discord_token: string}} */
+		const discordTokenResponse = await prompt({
+			type: "input",
+			name: "discord_token",
+			message: "Bot token",
+			validate: async token => {
+				process.stdout.write(magenta(" checking, please wait..."))
+				try {
+					snow = new SnowTransfer(token)
+					client = await snow.requestHandler.request(`/applications/@me`, {}, "get", "json")
+					return true
+				} catch (e) {
+					return e.message
+				}
+			}
+		})
+
+		const mandatoryIntentFlags = DiscordTypes.ApplicationFlags.GatewayMessageContent | DiscordTypes.ApplicationFlags.GatewayMessageContentLimited
+		if (!(client.flags & mandatoryIntentFlags)) {
+			console.log(`On that same page, scroll down to Privileged Gateway Intents and enable all switches.`)
+			await prompt({
+				type: "invisible",
+				name: "intents",
+				message: "Press Enter when you've enabled them",
+				validate: async token => {
+					process.stdout.write(magenta("checking, please wait..."))
+					client = await snow.requestHandler.request(`/applications/@me`, {}, "get", "json")
+					if (client.flags & mandatoryIntentFlags) {
+						return true
+					} else {
+						return "Switches have not been enabled yet"
+					}
+				}
+			})
+		}
+
+		console.log("Would you like to require a password to add your bot to servers? This will discourage others from using your bridge.")
+		console.log("Important: To make it truly private, you MUST ALSO disable Public Bot in the Discord bot configuration page.")
+		/** @type {{web_password: string}} */
+		const passwordResponse = await prompt({
+			type: "text",
+			name: "web_password",
+			message: "Choose a simple password (optional)"
+		})
+
+		console.log("What is your Discord client secret?")
+		console.log(`You can find it in the application's OAuth2 section: https://discord.com/developers/applications/${client.id}/oauth2`)
+		/** @type {{discord_client_secret: string}} */
+		const clientSecretResponse = await prompt({
+			type: "input",
+			name: "discord_client_secret",
+			message: "Client secret"
+		})
+
+		const expectedUri = `${bridgeOriginResponse.bridge_origin}/oauth`
+		if (!client.redirect_uris.includes(expectedUri)) {
+			console.log(`On that same page, scroll down to Redirects and add this URI: ${cyan(expectedUri)}`)
+			await prompt({
+				type: "invisible",
+				name: "redirect_uri",
+				message: "Press Enter when you've added it",
+				validate: async token => {
+					process.stdout.write(magenta("checking, please wait..."))
+					client = await snow.requestHandler.request(`/applications/@me`, {}, "get", "json")
+					if (client.redirect_uris.includes(expectedUri)) {
+						return true
+					} else {
+						return "Redirect URI has not been added yet"
+					}
+				}
+			})
+		}
+
+		const template = getTemplateRegistration(serverNameResponse.server_name)
+		reg = {
+			...template,
+			url: bridgeOriginResponse.bridge_origin,
+			ooye: {
+				...template.ooye,
+				...bridgeOriginResponse,
+				server_origin: serverOrigin,
+				...discordTokenResponse,
+				...clientSecretResponse,
+				...passwordResponse
+			}
+		}
+		registration.reg = reg
+		checkRegistration(reg)
+		writeRegistration(reg)
+		console.log(`✅ Your responses have been saved as ${registrationFilePath}`)
+	} else {
+		try {
+			checkRegistration(reg)
+			console.log(`✅ Skipped questions - reusing data from ${registrationFilePath}`)
+		} catch (e) {
+			console.log(`❌ Failed to reuse data from ${registrationFilePath}`)
+			console.log("Consider deleting this file. You can re-run setup to safely make a new one.")
+			console.log("")
+			console.log(e.toString().replace(/^ *\n/gm, ""))
+			process.exit(1)
+		}
+	}
+	console.log(`  In ${cyan("Synapse")}, you need to reference that file in your homeserver.yaml and ${cyan("restart Synapse")}.`)
+	console.log("    https://element-hq.github.io/synapse/latest/application_services.html")
+	console.log(`  In ${cyan("Conduit")}, you need to send the file contents to the #admins room.`)
+	console.log("    https://docs.conduit.rs/appservices.html")
+	console.log()
+
+	// Done with user prompts, reg is now guaranteed to be valid
+	const api = require("../src/matrix/api")
+	const file = require("../src/matrix/file")
+	const DiscordClient = require("../src/d2m/discord-client")
+	const discord = new DiscordClient(reg.ooye.discord_token, "no")
+	passthrough.discord = discord
+
+	const {as} = require("../src/matrix/appservice")
+	as.router.use("/**", defineEchoHandler())
+
+	console.log("⏳ Waiting for you to register the file with your homeserver... (Ctrl+C to cancel)")
+	process.once("SIGINT", () => {
+		console.log("(Ctrl+C) Quit early. Please re-run setup later and allow it to complete.")
+		process.exit(1)
+	})
+
+	let itWorks = false
+	let lastError = null
+	do {
+		const result = await api.ping().catch(e => ({ok: false, status: "net", root: e.message}))
+		// If it didn't work, log details and retry after some time
+		itWorks = result.ok
+		if (!itWorks) {
+			// Log the full error data if the error is different to last time
+			if (!isDeepStrictEqual(lastError, result.root)) {
+				if (typeof result.root === "string") {
+					console.log(`\nCannot reach homeserver: ${result.root}`)
+				} else if (result.root.error) {
+					console.log(`\nHomeserver said: [${result.status}] ${result.root.error}`)
+				} else {
+					console.log(`\nHomeserver said: [${result.status}] ${JSON.stringify(result.root)}`)
+				}
+				lastError = result.root
+			} else {
+				process.stderr.write(".")
+			}
+			await scheduler.wait(5000)
+		}
+	} while (!itWorks)
+	console.log("")
+
+	as.close().catch(() => {})
+
+	const mxid = `@${reg.sender_localpart}:${reg.ooye.server_name}`
+
+	// database ddl...
+	await migrate.migrate(db)
+
+	// add initial rows to database, like adding the bot to sim...
+	const client = await discord.snow.user.getSelf()
+	db.prepare("INSERT INTO sim (user_id, username, sim_name, mxid) VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING").run(client.id, client.username, reg.sender_localpart.slice(reg.ooye.namespace_prefix.length), mxid)
+
+	console.log("✅ Database is ready...")
+
+	// ensure appservice bot user is registered...
+	await api.register(reg.sender_localpart)
+
+	// upload initial images...
+	const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png")
+
+	console.log("✅ Matrix appservice login works...")
+
+	// upload the L1 L2 emojis to some guild
+	const emojis = db.prepare("SELECT name FROM auto_emoji WHERE name = 'L1' OR name = 'L2'").pluck().all()
+	if (emojis.length !== 2) {
+		// If an argument was supplied, always use that one
+		let guild = null
+		if (args["emoji-guild"]) {
+			if (typeof args["emoji-guild"] === "string") {
+				guild = await discord.snow.guild.getGuild(args["emoji-guild"])
+			}
+			if (!guild) return die(`Error: You asked emojis to be uploaded to guild ID ${args["emoji-guild"]}, but the bot isn't in that guild.`)
+		}
+		// Otherwise, check if we have already registered an auto emoji guild
+		if (!guild) {
+			const guildID = passthrough.select("auto_emoji", "guild_id", {name: "_"}).pluck().get()
+			if (guildID) {
+				guild = await discord.snow.guild.getGuild(guildID, false)
+			}
+		}
+		// Otherwise, check if we should create a new guild
+		if (!guild) {
+			const guilds = await discord.snow.user.getGuilds({limit: 11, with_counts: false})
+			if (guilds.length < 10) {
+				console.log("   Creating a guild for emojis...")
+				guild = await discord.snow.guild.createGuild({name: "OOYE Emojis"})
+			}
+		}
+		// Otherwise, it's the user's problem
+		if (!guild) {
+			return die(`Error: The bot needs to upload some emojis. Please say where to upload them to. Run setup again with --emoji-guild=GUILD_ID`)
+		}
+		// Upload those emojis to the chosen location
+		db.prepare("REPLACE INTO auto_emoji (name, emoji_id, guild_id) VALUES ('_', '_', ?)").run(guild.id)
+		await uploadAutoEmoji(discord.snow, guild, "L1", join(__dirname, "../docs/img/L1.png"))
+		await uploadAutoEmoji(discord.snow, guild, "L2", join(__dirname, "../docs/img/L2.png"))
+	}
+	console.log("✅ Emojis are ready...")
+
+	// set profile data on discord...
+	const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer())
+	await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")})
+	await discord.snow.bot.updateApplicationInfo({description: "Powered by **Out Of Your Element**\nhttps://gitdab.com/cadence/out-of-your-element"})
+	console.log("✅ Discord profile updated...")
+
+	// set profile data on homeserver...
+	console.log("⏩ Updating Matrix profile... (If you've joined lots of rooms, this is slow. Please allow at least 30 seconds.)")
+	await api.profileSetDisplayname(mxid, "Out Of Your Element")
+	await api.profileSetAvatarUrl(mxid, avatarUrl)
+	console.log("✅ Matrix profile updated...")
+
+	console.log("Good to go. I hope you enjoy Out Of Your Element.")
+	process.exit()
+})()

scripts/start-server.js

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+// @ts-check
+
+const {createServer} = require("http")
+const EventEmitter = require("events")
+const {createApp, createRouter, toNodeListener} = require("h3")
+const sqlite = require("better-sqlite3")
+const migrate = require("../src/db/migrate")
+const HeatSync = require("heatsync")
+
+const {reg} = require("../src/matrix/read-registration")
+const passthrough = require("../src/passthrough")
+const db = new sqlite("ooye.db")
+
+const sync = new HeatSync()
+
+Object.assign(passthrough, {sync, db})
+
+const DiscordClient = require("../src/d2m/discord-client")
+
+const discord = new DiscordClient(reg.ooye.discord_token, "half")
+passthrough.discord = discord
+
+const app = createApp()
+const router = createRouter()
+app.use(router)
+const server = createServer(toNodeListener(app))
+server.listen(reg.socket || new URL(reg.url).port)
+const as = Object.assign(new EventEmitter(), {app, router, server}) // @ts-ignore
+passthrough.as = as
+
+const orm = sync.require("../src/db/orm")
+passthrough.from = orm.from
+passthrough.select = orm.select
+
+;(async () => {
+	await migrate.migrate(db)
+	await discord.cloud.connect()
+	console.log("Discord gateway started")
+	sync.require("../src/web/server")
+
+	require("../src/stdin")
+})()

scripts/wal.js

@@ -1,6 +1,7 @@
+#!/usr/bin/env node
 // @ts-check
 
 const sqlite = require("better-sqlite3")
-const db = new sqlite("db/ooye.db", {fileMustExist: true})
+const db = new sqlite("ooye.db", {fileMustExist: true})
 db.pragma("journal_mode = wal")
 db.close()

src/d2m/actions/create-room.js

@@ -2,18 +2,25 @@
 
 const assert = require("assert").strict
 const DiscordTypes = require("discord-api-types/v10")
-const reg = require("../../matrix/read-registration")
+const Ty = require("../../types")
+const {reg} = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select} = passthrough
+const {discord, sync, db, select, from} = passthrough
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
+/** @type {import("../../matrix/mreq")} */
+const mreq = sync.require("../../matrix/mreq")
 /** @type {import("../../matrix/kstate")} */
 const ks = sync.require("../../matrix/kstate")
-/** @type {import("./create-space")}) */
-const createSpace = sync.require("./create-space") // watch out for the require loop
+/** @type {import("../../discord/utils")} */
+const dUtils = sync.require("../../discord/utils")
+/** @type {import("../../m2d/converters/utils")} */
+const mUtils = sync.require("../../m2d/converters/utils")
+/** @type {import("./create-space")} */
+const createSpace = sync.require("./create-space")
 
 /**
  * There are 3 levels of room privacy:
@@ -23,7 +30,7 @@ const createSpace = sync.require("./create-space") // watch out for the require
  */
 const PRIVACY_ENUMS = {
 	PRESET: ["private_chat", "public_chat", "public_chat"],
-	VISIBILITY: ["private", "private", "public"],
+	VISIBILITY: ["private", "private", "private"],
 	SPACE_HISTORY_VISIBILITY: ["invited", "world_readable", "world_readable"], // copying from element client
 	ROOM_HISTORY_VISIBILITY: ["shared", "shared", "world_readable"], // any events sent after <value> are visible, but for world_readable anybody can read without even joining
 	GUEST_ACCESS: ["can_join", "forbidden", "forbidden"], // whether guests can join space if other conditions are met
@@ -37,33 +44,16 @@ const DEFAULT_PRIVACY_LEVEL = 0
 const inflightRoomCreate = new Map()
 
 /**
- * Async because it gets all room state from the homeserver.
- * @param {string} roomID
- */
-async function roomToKState(roomID) {
-	const root = await api.getAllState(roomID)
-	return ks.stateToKState(root)
-}
-
-/**
- * @param {string} roomID
- * @param {any} kstate
- */
-function applyKStateDiffToRoom(roomID, kstate) {
-	const events = ks.kstateToState(kstate)
-	return Promise.all(events.map(({type, state_key, content}) =>
-		api.sendState(roomID, type, state_key, content)
-	))
-}
-
-/**
- * @param {{id: string, name: string, topic?: string?, type: number}} channel
+ * @param {{id: string, name: string, topic?: string?, type: number, parent_id?: string?}} channel
  * @param {{id: string}} guild
  * @param {string | null | undefined} customName
  */
 function convertNameAndTopic(channel, guild, customName) {
+	// @ts-ignore
+	const parentChannel = discord.channels.get(channel.parent_id)
 	let channelPrefix =
-		( channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] "
+		( parentChannel?.type === DiscordTypes.ChannelType.GuildForum ? ""
+		: channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] "
 		: channel.type === DiscordTypes.ChannelType.PrivateThread ? "[🔒⛓️] "
 		: channel.type === DiscordTypes.ChannelType.GuildVoice ? "[🔊] "
 		: "")
@@ -84,26 +74,36 @@ function convertNameAndTopic(channel, guild, customName) {
  * Async because it may create the guild and/or upload the guild icon to mxc.
  * @param {DiscordTypes.APIGuildTextChannel | DiscordTypes.APIThreadChannel} channel
  * @param {DiscordTypes.APIGuild} guild
+ * @param {{api: {getStateEvent: typeof api.getStateEvent}}} di simple-as-nails dependency injection for the matrix API
  */
-async function channelToKState(channel, guild) {
-	const spaceID = await createSpace.ensureSpace(guild)
-	assert(typeof spaceID === "string")
-	const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get()
-	assert(typeof privacyLevel === "number")
+async function channelToKState(channel, guild, di) {
+	// @ts-ignore
+	const parentChannel = discord.channels.get(channel.parent_id)
 
-	const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get()
-	const customName = row?.nick
-	const customAvatar = row?.custom_avatar
+	/** Used for membership/permission checks. */
+	const guildSpaceID = await createSpace.ensureSpace(guild)
+	/** Used as the literal parent on Matrix, for categorisation. Will be the same as `guildSpaceID` unless it's a forum channel's thread, in which case a different space is used to group those threads. */
+	let parentSpaceID = guildSpaceID
+	if (parentChannel?.type === DiscordTypes.ChannelType.GuildForum) {
+		parentSpaceID = await ensureRoom(channel.parent_id)
+		assert(typeof parentSpaceID === "string")
+	}
+
+	const channelRow = select("channel_room", ["nick", "custom_avatar", "custom_topic"], {channel_id: channel.id}).get()
+	const customName = channelRow?.nick
+	const customAvatar = channelRow?.custom_avatar
+	const hasCustomTopic = channelRow?.custom_topic
 	const [convertedName, convertedTopic] = convertNameAndTopic(channel, guild, customName)
 
 	const avatarEventContent = {}
 	if (customAvatar) {
 		avatarEventContent.url = customAvatar
 	} else if (guild.icon) {
-		avatarEventContent.discord_path = file.guildIcon(guild)
-		avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
+		avatarEventContent.url = {$url: file.guildIcon(guild)}
 	}
 
+	const privacyLevel = select("guild_space", "privacy_level", {guild_id: guild.id}).pluck().get()
+	assert(privacyLevel != null) // already ensured the space exists
 	let history_visibility = PRIVACY_ENUMS.ROOM_HISTORY_VISIBILITY[privacyLevel]
 	if (channel["thread_metadata"]) history_visibility = "world_readable"
 
@@ -112,33 +112,49 @@ async function channelToKState(channel, guild) {
 		join_rule: "restricted",
 		allow: [{
 			type: "m.room_membership",
-			room_id: spaceID
+			room_id: guildSpaceID
 		}]
 	}
 	if (PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel] !== "restricted") {
 		join_rules = {join_rule: PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel]}
 	}
 
+	const everyonePermissions = dUtils.getPermissions([], guild.roles, undefined, channel.permission_overwrites)
+	const everyoneCanSend = dUtils.hasPermission(everyonePermissions, DiscordTypes.PermissionFlagsBits.SendMessages)
+	const everyoneCanMentionEveryone = dUtils.hasAllPermissions(everyonePermissions, ["MentionEveryone"])
+
+	const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all()
+	const globalAdminPower = globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {})
+
+	/** @type {Ty.Event.M_Power_Levels} */
+	const spacePowerEvent = await di.api.getStateEvent(guildSpaceID, "m.room.power_levels", "")
+	const spacePower = spacePowerEvent.users
+
+	/** @type {any} */
 	const channelKState = {
 		"m.room.name/": {name: convertedName},
 		"m.room.topic/": {topic: convertedTopic},
 		"m.room.avatar/": avatarEventContent,
 		"m.room.guest_access/": {guest_access: PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
 		"m.room.history_visibility/": {history_visibility},
-		[`m.space.parent/${spaceID}`]: {
+		[`m.space.parent/${parentSpaceID}`]: {
 			via: [reg.ooye.server_name],
 			canonical: true
 		},
 		/** @type {{join_rule: string, [x: string]: any}} */
 		"m.room.join_rules/": join_rules,
+		/** @type {Ty.Event.M_Power_Levels} */
 		"m.room.power_levels/": {
+			events_default: everyoneCanSend ? 0 : 50,
 			events: {
-				"m.room.avatar": 0
+				"m.reaction": 0
 			},
-			users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})
+			notifications: {
+				room: everyoneCanMentionEveryone ? 0 : 20
+			},
+			users: {...spacePower, ...globalAdminPower}
 		},
 		"chat.schildi.hide_ui/read_receipts": {
-			hidden: true
 		},
 		[`uk.half-shot.bridge/moe.cadence.ooye://discord/${guild.id}/${channel.id}`]: {
 			bridgebot: `@${reg.sender_localpart}:${reg.ooye.server_name}`,
@@ -149,7 +165,7 @@ async function channelToKState(channel, guild) {
 			network: {
 				id: guild.id,
 				displayname: guild.name,
-				avatar_url: await file.uploadDiscordFileToMxc(file.guildIcon(guild))
+				avatar_url: {$url: file.guildIcon(guild)}
 			},
 			channel: {
 				id: channel.id,
@@ -159,7 +175,9 @@ async function channelToKState(channel, guild) {
 		}
 	}
 
-	return {spaceID, privacyLevel, channelKState}
+	if (hasCustomTopic) delete channelKState["m.room.topic/"]
+
+	return {spaceID: parentSpaceID, privacyLevel, channelKState}
 }
 
 /**
@@ -175,6 +193,9 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 	let threadParent = null
 	if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id
 
+	let spaceCreationContent = {}
+	if (channel.type === DiscordTypes.ChannelType.GuildForum) spaceCreationContent = {creation_content: {type: "m.space"}}
+
 	// Name and topic can be done earlier in room creation rather than in initial_state
 	// https://spec.matrix.org/latest/client-server-api/#creation
 	const name = kstate["m.room.name/"].name
@@ -191,7 +212,8 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 			preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
 			visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
 			invite: [],
-			initial_state: ks.kstateToState(kstate)
+			initial_state: await ks.kstateToState(kstate),
+			...spaceCreationContent
 		})
 
 		db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent)
@@ -219,15 +241,16 @@ async function postApplyPowerLevels(kstate, callback) {
 	const powerLevelContent = kstate["m.room.power_levels/"]
 	const kstateWithoutPowerLevels = {...kstate}
 	delete kstateWithoutPowerLevels["m.room.power_levels/"]
+	delete kstateWithoutPowerLevels["chat.schildi.hide_ui/read_receipts"]
 
 	/** @type {string} */
 	const roomID = await callback(kstateWithoutPowerLevels)
 
 	// Now *really* apply the power level overrides on top of what Synapse *really* set
 	if (powerLevelContent) {
-		const newRoomKState = await roomToKState(roomID)
+		const newRoomKState = await ks.roomToKState(roomID)
 		const newRoomPowerLevelsDiff = ks.diffKState(newRoomKState, {"m.room.power_levels/": powerLevelContent})
-		await applyKStateDiffToRoom(roomID, newRoomPowerLevelsDiff)
+		await ks.applyKStateDiffToRoom(roomID, newRoomPowerLevelsDiff)
 	}
 
 	return roomID
@@ -244,6 +267,61 @@ function channelToGuild(channel) {
 	return guild
 }
 
+/**
+ * This function handles whether it's allowed to bridge messages in this channel, and if so, where to.
+ * This has to account for whether self-service is enabled for the guild or not.
+ * This also has to account for different channel types, like forum channels (which need the
+ * parent forum to already exist, and ignore the self-service setting), or thread channels (which
+ * need the parent channel to already exist, and ignore the self-service setting).
+ * @param {DiscordTypes.APIGuildTextChannel | DiscordTypes.APIThreadChannel} channel text channel or thread
+ * @param {string} guildID
+ * @returns obj if bridged; 1 if autocreatable; null/undefined if guild is not bridged; 0 if self-service and not autocreatable thread
+ */
+function existsOrAutocreatable(channel, guildID) {
+	// 1. If the channel is already linked somewhere, it's always okay to bridge to that destination, no matter what. Yippee!
+	const existing = select("channel_room", ["room_id", "thread_parent"], {channel_id: channel.id}).get()
+	if (existing) return existing
+
+	// 2. If the guild is an autocreate guild, it's always okay to bridge to that destination, and
+	// we'll need to create any dependent resources recursively.
+	const autocreate = select("guild_active", "autocreate", {guild_id: guildID}).pluck().get()
+	if (autocreate === 1) return autocreate
+
+	// 3. If the guild is not approved for bridging yet, we can't bridge there.
+	// They need to decide one way or another whether it's self-service before we can continue.
+	if (autocreate == null) return autocreate
+
+	// 4. If we got here, the guild is in self-service mode.
+	// New channels won't be able to create new rooms. But forum threads or channel threads could be fine.
+	if ([DiscordTypes.ChannelType.PublicThread, DiscordTypes.ChannelType.PrivateThread, DiscordTypes.ChannelType.AnnouncementThread].includes(channel.type)) {
+		// In self-service mode, threads rely on the parent resource already existing.
+		/** @type {DiscordTypes.APIGuildTextChannel} */ // @ts-ignore
+		const parent = discord.channels.get(channel.parent_id)
+		assert(parent)
+		const parentExisting = existsOrAutocreatable(parent, guildID)
+		if (parentExisting) return 1 // Autocreatable
+	}
+
+	// 5. If we got here, the guild is in self-service mode and the channel is truly not bridged.
+	return autocreate
+}
+
+/**
+ * @param {DiscordTypes.APIGuildTextChannel | DiscordTypes.APIThreadChannel} channel text channel or thread
+ * @param {string} guildID
+ * @returns obj if bridged; 1 if autocreatable. (throws if not autocreatable)
+ */
+function assertExistsOrAutocreatable(channel, guildID) {
+	const existing = existsOrAutocreatable(channel, guildID)
+	if (existing === 0) {
+		throw new Error(`Guild ${guildID} is self-service, so won't create a Matrix room for channel ${channel.id}`)
+	}
+	if (!existing) {
+		throw new Error(`Guild ${guildID} is not bridged, so won't create a Matrix room for channel ${channel.id}`)
+	}
+	return existing
+}
+
 /*
 	Ensure flow:
 	1. Get IDs
@@ -262,6 +340,7 @@ function channelToGuild(channel) {
 */
 
 /**
+ * Create room and/or sync room data. Please check that a channel_room entry exists or autocreate = 1 before calling this.
  * @param {string} channelID
  * @param {boolean} shouldActuallySync false if just need to ensure room exists (which is a quick database check), true if also want to sync room data when it does exist (slow)
  * @returns {Promise<string>} room ID
@@ -276,11 +355,11 @@ async function _syncRoom(channelID, shouldActuallySync) {
 		await inflightRoomCreate.get(channelID) // just waiting, and then doing a new db query afterwards, is the simplest way of doing it
 	}
 
-	const existing = select("channel_room", ["room_id", "thread_parent"], {channel_id: channelID}).get()
+	const existing = assertExistsOrAutocreatable(channel, guild.id)
 
-	if (!existing) {
+	if (existing === 1) {
 		const creation = (async () => {
-			const {spaceID, privacyLevel, channelKState} = await channelToKState(channel, guild)
+			const {spaceID, privacyLevel, channelKState} = await channelToKState(channel, guild, {api})
 			const roomID = await createRoom(channel, guild, spaceID, channelKState, privacyLevel)
 			inflightRoomCreate.delete(channelID) // OK to release inflight waiters now. they will read the correct `existing` row
 			return roomID
@@ -297,10 +376,10 @@ async function _syncRoom(channelID, shouldActuallySync) {
 
 	console.log(`[room sync] to matrix: ${channel.name}`)
 
-	const {spaceID, channelKState} = await channelToKState(channel, guild) // calling this in both branches because we don't want to calculate this if not syncing
+	const {spaceID, channelKState} = await channelToKState(channel, guild, {api}) // calling this in both branches because we don't want to calculate this if not syncing
 
 	// sync channel state to room
-	const roomKState = await roomToKState(roomID)
+	const roomKState = await ks.roomToKState(roomID)
 	if (+roomKState["m.room.create/"].room_version <= 8) {
 		// join_rule `restricted` is not available in room version < 8 and not working properly in version == 8
 		// read more: https://spec.matrix.org/v1.8/rooms/v9/
@@ -308,7 +387,7 @@ async function _syncRoom(channelID, shouldActuallySync) {
 		channelKState["m.room.join_rules/"] = {join_rule: "public"}
 	}
 	const roomDiff = ks.diffKState(roomKState, channelKState)
-	const roomApply = applyKStateDiffToRoom(roomID, roomDiff)
+	const roomApply = ks.applyKStateDiffToRoom(roomID, roomDiff)
 	db.prepare("UPDATE channel_room SET name = ? WHERE room_id = ?").run(channel.name, roomID)
 
 	// sync room as space member
@@ -318,35 +397,76 @@ async function _syncRoom(channelID, shouldActuallySync) {
 	return roomID
 }
 
-/** Ensures the room exists. If it doesn't, creates the room with an accurate initial state. */
+/** Ensures the room exists. If it doesn't, creates the room with an accurate initial state. Please check that a channel_room entry exists or guild autocreate = 1 before calling this. */
 function ensureRoom(channelID) {
 	return _syncRoom(channelID, false)
 }
 
-/** Actually syncs. Gets all room state from the homeserver in order to diff, and uploads the icon to mxc if it has changed. */
+/** Actually syncs. Gets all room state from the homeserver in order to diff, and uploads the icon to mxc if it has changed. Please check that a channel_room entry exists or guild autocreate = 1 before calling this. */
 function syncRoom(channelID) {
 	return _syncRoom(channelID, true)
 }
 
-async function _unbridgeRoom(channelID) {
+async function unbridgeChannel(channelID) {
 	/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
 	const channel = discord.channels.get(channelID)
 	assert.ok(channel)
-	return unbridgeDeletedChannel(channel.id, channel.guild_id)
+	assert.ok(channel.guild_id)
+	return unbridgeDeletedChannel(channel, channel.guild_id)
 }
 
-async function unbridgeDeletedChannel(channelID, guildID) {
-	const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
+/**
+ * @param {{id: string, topic?: string?}} channel channel-ish (just needs an id, topic is optional)
+ * @param {string} guildID
+ */
+async function unbridgeDeletedChannel(channel, guildID) {
+	const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get()
 	assert.ok(roomID)
-	const spaceID = select("guild_space", "space_id", {guild_id: guildID}).pluck().get()
-	assert.ok(spaceID)
+	const row = from("guild_space").join("guild_active", "guild_id").select("space_id", "autocreate").get()
+	assert.ok(row)
 
-	// remove room from being a space member
-	await api.sendState(roomID, "m.space.parent", spaceID, {})
-	await api.sendState(spaceID, "m.space.child", roomID, {})
+	let botInRoom = true
 
 	// remove declaration that the room is bridged
-	await api.sendState(roomID, "uk.half-shot.bridge", `moe.cadence.ooye://discord/${guildID}/${channelID}`, {})
+	try {
+		await api.sendState(roomID, "uk.half-shot.bridge", `moe.cadence.ooye://discord/${guildID}/${channel.id}`, {})
+	} catch (e) {
+		if (String(e).includes("not in room")) {
+			botInRoom = false
+		} else {
+			throw e
+		}
+	}
+
+	if (botInRoom && "topic" in channel) {
+		// previously the Matrix topic would say the channel ID. we should remove that
+		await api.sendState(roomID, "m.room.topic", "", {topic: channel.topic || ""})
+	}
+
+	// delete webhook on discord
+	const webhook = select("webhook", ["webhook_id", "webhook_token"], {channel_id: channel.id}).get()
+	if (webhook) {
+		await discord.snow.webhook.deleteWebhook(webhook.webhook_id, webhook.webhook_token)
+		db.prepare("DELETE FROM webhook WHERE channel_id = ?").run(channel.id)
+	}
+
+	// delete room from database
+	db.prepare("DELETE FROM member_cache WHERE room_id = ?").run(roomID)
+	db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channel.id) // cascades to most other tables, like messages
+
+	if (!botInRoom) return
+
+	// demote admins in room
+	/** @type {Ty.Event.M_Power_Levels} */
+	const powerLevelContent = await api.getStateEvent(roomID, "m.room.power_levels", "")
+	powerLevelContent.users ??= {}
+	const bot = `@${reg.sender_localpart}:${reg.ooye.server_name}`
+	for (const mxid of Object.keys(powerLevelContent.users)) {
+		if (powerLevelContent.users[mxid] >= 100 && mUtils.eventSenderIsFromDiscord(mxid) && mxid !== bot) {
+			delete powerLevelContent.users[mxid]
+			await api.sendState(roomID, "m.room.power_levels", "", powerLevelContent, mxid)
+		}
+	}
 
 	// send a notification in the room
 	await api.sendEvent(roomID, "m.room.message", {
@@ -354,12 +474,28 @@ async function unbridgeDeletedChannel(channelID, guildID) {
 		body: "⚠️ This room was removed from the bridge."
 	})
 
+	// if it is an easy mode room, clean up the room from the managed space and make it clear it's not being bridged
+	// (don't do this for self-service rooms, because they might continue to be used on Matrix or linked somewhere else later)
+	if (row.autocreate === 1) {
+		// remove room from being a space member
+		await api.sendState(roomID, "m.space.parent", row.space_id, {})
+		await api.sendState(row.space_id, "m.space.child", roomID, {})
+	}
+
+	// if it is a self-service room, remove sim members
+	// (the room can be used with less clutter and the member list makes sense if it's bridged somewhere else)
+	if (row.autocreate === 0) {
+		// remove sim members
+		const members = db.prepare("SELECT mxid FROM sim_member WHERE room_id = ? AND mxid <> ?").pluck().all(roomID, bot)
+		const preparedDelete = db.prepare("DELETE FROM sim_member WHERE room_id = ? AND mxid = ?")
+		for (const mxid of members) {
+			await api.leaveRoom(roomID, mxid)
+			preparedDelete.run(roomID, mxid)
+		}
+	}
+
 	// leave room
 	await api.leaveRoom(roomID)
-
-	// delete room from database
-	const {changes} = db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channelID)
-	assert.equal(changes, 1)
 }
 
 /**
@@ -370,7 +506,7 @@ async function unbridgeDeletedChannel(channelID, guildID) {
  * @returns {Promise<string[]>}
  */
 async function _syncSpaceMember(channel, spaceID, roomID) {
-	const spaceKState = await roomToKState(spaceID)
+	const spaceKState = await ks.roomToKState(spaceID)
 	let spaceEventContent = {}
 	if (
 		channel.type !== DiscordTypes.ChannelType.PrivateThread // private threads do not belong in the space (don't offer people something they can't join)
@@ -383,7 +519,7 @@ async function _syncSpaceMember(channel, spaceID, roomID) {
 	const spaceDiff = ks.diffKState(spaceKState, {
 		[`m.space.child/${roomID}`]: spaceEventContent
 	})
-	return applyKStateDiffToRoom(spaceID, spaceDiff)
+	return ks.applyKStateDiffToRoom(spaceID, spaceDiff)
 }
 
 async function createAllForGuild(guildID) {
@@ -406,9 +542,9 @@ module.exports.ensureRoom = ensureRoom
 module.exports.syncRoom = syncRoom
 module.exports.createAllForGuild = createAllForGuild
 module.exports.channelToKState = channelToKState
-module.exports.roomToKState = roomToKState
-module.exports.applyKStateDiffToRoom = applyKStateDiffToRoom
 module.exports.postApplyPowerLevels = postApplyPowerLevels
 module.exports._convertNameAndTopic = convertNameAndTopic
-module.exports._unbridgeRoom = _unbridgeRoom
+module.exports.unbridgeChannel = unbridgeChannel
 module.exports.unbridgeDeletedChannel = unbridgeDeletedChannel
+module.exports.existsOrAutocreatable = existsOrAutocreatable
+module.exports.assertExistsOrAutocreatable = assertExistsOrAutocreatable

src/d2m/actions/create-room.test.js

@@ -0,0 +1,246 @@
+// @ts-check
+
+const mixin = require("@cloudrac3r/mixin-deep")
+const {channelToKState, _convertNameAndTopic} = require("./create-room")
+const {kstateStripConditionals} = require("../../matrix/kstate")
+const {test} = require("supertape")
+const testData = require("../../../test/data")
+
+const passthrough = require("../../passthrough")
+const {db} = passthrough
+
+
+test("channel2room: discoverable privacy room", async t => {
+	let called = 0
+	async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room
+		called++
+		t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe")
+		t.equal(type, "m.room.power_levels")
+		t.equal(key, "")
+		return {users: {"@example:matrix.org": 50}}
+	}
+	db.prepare("UPDATE guild_space SET privacy_level = 2").run()
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)),
+		Object.assign({}, testData.room.general, {
+			"m.room.guest_access/": {guest_access: "forbidden"},
+			"m.room.join_rules/": {join_rule: "public"},
+			"m.room.history_visibility/": {history_visibility: "world_readable"},
+			"m.room.power_levels/": mixin({users: {"@example:matrix.org": 50}}, testData.room.general["m.room.power_levels/"])
+		})
+	)
+	t.equal(called, 1)
+})
+
+test("channel2room: linkable privacy room", async t => {
+	let called = 0
+	async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room
+		called++
+		t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe")
+		t.equal(type, "m.room.power_levels")
+		t.equal(key, "")
+		return {users: {"@example:matrix.org": 50}}
+	}
+	db.prepare("UPDATE guild_space SET privacy_level = 1").run()
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)),
+		Object.assign({}, testData.room.general, {
+			"m.room.guest_access/": {guest_access: "forbidden"},
+			"m.room.join_rules/": {join_rule: "public"},
+			"m.room.power_levels/": mixin({users: {"@example:matrix.org": 50}}, testData.room.general["m.room.power_levels/"])
+		})
+	)
+	t.equal(called, 1)
+})
+
+test("channel2room: invite-only privacy room", async t => {
+	let called = 0
+	async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room
+		called++
+		t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe")
+		t.equal(type, "m.room.power_levels")
+		t.equal(key, "")
+		return {users: {"@example:matrix.org": 50}}
+	}
+	db.prepare("UPDATE guild_space SET privacy_level = 0").run()
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)),
+		Object.assign({}, testData.room.general, {
+			"m.room.power_levels/": mixin({users: {"@example:matrix.org": 50}}, testData.room.general["m.room.power_levels/"])
+		})
+	)
+	t.equal(called, 1)
+})
+
+test("channel2room: room where limited people can mention everyone", async t => {
+	let called = 0
+	async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room
+		called++
+		t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe")
+		t.equal(type, "m.room.power_levels")
+		t.equal(key, "")
+		return {users: {"@example:matrix.org": 50}}
+	}
+	const limitedGuild = mixin({}, testData.guild.general)
+	limitedGuild.roles[0].permissions = (BigInt(limitedGuild.roles[0].permissions) - 131072n).toString()
+	const limitedRoom = mixin({}, testData.room.general, {"m.room.power_levels/": {
+		notifications: {room: 20},
+		users: {"@example:matrix.org": 50}
+	}})
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, limitedGuild, {api: {getStateEvent}}).then(x => x.channelKState)),
+		limitedRoom
+	)
+	t.equal(called, 1)
+})
+
+test("channel2room: matrix room that already has a custom topic set", async t => {
+	let called = 0
+	async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room
+		called++
+		t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe")
+		t.equal(type, "m.room.power_levels")
+		t.equal(key, "")
+		return {}
+	}
+	db.prepare("UPDATE channel_room SET custom_topic = 1 WHERE channel_id = ?").run(testData.channel.general.id)
+	const expected = mixin({}, testData.room.general, {"m.room.power_levels/": {notifications: {room: 20}}})
+	// @ts-ignore
+	delete expected["m.room.topic/"]
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)),
+		expected
+	)
+	t.equal(called, 1)
+})
+
+test("channel2room: read-only discord channel", async t => {
+	let called = 0
+	async function getStateEvent(roomID, type, key) { // getting power levels from space to apply to room
+		called++
+		t.equal(roomID, "!jjmvBegULiLucuWEHU:cadence.moe")
+		t.equal(type, "m.room.power_levels")
+		t.equal(key, "")
+		return {}
+	}
+	const expected = {
+		"chat.schildi.hide_ui/read_receipts": {},
+		"m.room.avatar/": {
+			url: {
+				$url: "/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024",
+			},
+		},
+		"m.room.guest_access/": {
+			guest_access: "can_join",
+		},
+		"m.room.history_visibility/": {
+			history_visibility: "shared",
+		},
+		"m.room.join_rules/": {
+			allow: [
+				{
+					room_id: "!jjmvBegULiLucuWEHU:cadence.moe",
+					type: "m.room_membership",
+				},
+			],
+			join_rule: "restricted",
+		},
+		"m.room.name/": {
+			name: "updates",
+		},
+		"m.room.topic/": {
+      	topic: "Updates and release announcements for Out Of Your Element.\n\nChannel ID: 1161864271370666075\nGuild ID: 112760669178241024"
+		},
+		"m.room.power_levels/": {
+			events_default: 50, // <-- it should be read-only!
+			events: {
+				"m.reaction": 0
+			},
+			notifications: {
+				room: 20,
+			},
+			users: {
+				"@test_auto_invite:example.org": 100,
+			},
+		},
+		"m.space.parent/!jjmvBegULiLucuWEHU:cadence.moe": {
+			canonical: true,
+			via: [
+				"cadence.moe",
+			],
+		},
+		"uk.half-shot.bridge/moe.cadence.ooye://discord/112760669178241024/1161864271370666075": {
+			bridgebot: "@_ooye_bot:cadence.moe",
+			channel: {
+				displayname: "updates",
+				external_url: "https://discord.com/channels/112760669178241024/1161864271370666075",
+				id: "1161864271370666075",
+			},
+			network: {
+				avatar_url: {
+					"$url": "/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024",
+				},
+				displayname: "Psychonauts 3",
+				id: "112760669178241024",
+			},
+			protocol: {
+				displayname: "Discord",
+				id: "discord",
+			}
+		}
+	}
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.updates, testData.guild.general, {api: {getStateEvent}}).then(x => x.channelKState)),
+		expected
+	)
+	t.equal(called, 1)
+})
+
+test("convertNameAndTopic: custom name and topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, "hauntings"),
+		["hauntings", "#the-twilight-zone | Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: custom name, no topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, "hauntings"),
+		["hauntings", "#the-twilight-zone\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: original name and topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, null),
+		["the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: original name, no topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, null),
+		["the-twilight-zone", "Channel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: public thread icon", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 11}, {id: "456"}, null),
+		["[⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: private thread icon", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 12}, {id: "456"}, null),
+		["[🔒⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: voice channel icon", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 2}, {id: "456"}, null),
+		["[🔊] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})

src/d2m/actions/create-space.js

@@ -1,9 +1,10 @@
 // @ts-check
 
 const assert = require("assert").strict
+const {isDeepStrictEqual} = require("util")
 const DiscordTypes = require("discord-api-types/v10")
-const deepEqual = require("deep-equal")
-const reg = require("../../matrix/read-registration")
+const Ty = require("../../types")
+const {reg} = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
@@ -30,6 +31,10 @@ async function createSpace(guild, kstate) {
 	const topic = kstate["m.room.topic/"]?.topic || undefined
 	assert(name)
 
+	const memberCount = guild["member_count"] ?? guild.approximate_member_count ?? 0
+	const enablePresenceByDefault = +(memberCount < 50) // scary! all active users in a presence-enabled guild will be pinging the server every <30 seconds to stay online
+	const globalAdmins = select("member_power", "mxid", {room_id: "*"}).pluck().all()
+
 	const roomID = await createRoom.postApplyPowerLevels(kstate, async kstate => {
 		return api.createRoom({
 			name,
@@ -39,15 +44,15 @@ async function createSpace(guild, kstate) {
 				events_default: 100, // space can only be managed by bridge
 				invite: 0 // any existing member can invite others
 			},
-			invite: reg.ooye.invite,
+			invite: globalAdmins,
 			topic,
 			creation_content: {
 				type: "m.space"
 			},
-			initial_state: ks.kstateToState(kstate)
+			initial_state: await ks.kstateToState(kstate)
 		})
 	})
-	db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, roomID)
+	db.prepare("INSERT INTO guild_space (guild_id, space_id, presence) VALUES (?, ?, ?)").run(guild.id, roomID, enablePresenceByDefault)
 	return roomID
 }
 
@@ -56,19 +61,18 @@ async function createSpace(guild, kstate) {
  * @param {number} privacyLevel
  */
 async function guildToKState(guild, privacyLevel) {
-	const avatarEventContent = {}
-	if (guild.icon) {
-		avatarEventContent.discord_path = file.guildIcon(guild)
-		avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
-	}
-
+	assert.equal(typeof privacyLevel, "number")
+	const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all()
 	const guildKState = {
 		"m.room.name/": {name: guild.name},
-		"m.room.avatar/": avatarEventContent,
+		"m.room.avatar/": {
+			$if: guild.icon,
+			url: {$url: file.guildIcon(guild)}
+		},
 		"m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
 		"m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]},
 		"m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]},
-		"m.room.power_levels/": {users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})}
+		"m.room.power_levels/": {users: globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {})} // used in guild initial creation postApplyPowerLevels
 	}
 
 	return guildKState
@@ -90,6 +94,9 @@ async function _syncSpace(guild, shouldActuallySync) {
 	const row = select("guild_space", ["space_id", "privacy_level"], {guild_id: guild.id}).get()
 
 	if (!row) {
+		const autocreate = select("guild_active", "autocreate", {guild_id: guild.id}).pluck().get()
+		assert.equal(autocreate, 1, `refusing to implicitly create a space for guild ${guild.id}. set the guild_active data first before calling ensureSpace/syncSpace.`)
+
 		const creation = (async () => {
 			const guildKState = await guildToKState(guild, createRoom.DEFAULT_PRIVACY_LEVEL) // New spaces will have to use the default privacy level; we obviously can't look up the existing entry
 			const spaceID = await createSpace(guild, guildKState)
@@ -111,9 +118,9 @@ async function _syncSpace(guild, shouldActuallySync) {
 	const guildKState = await guildToKState(guild, privacy_level) // calling this in both branches because we don't want to calculate this if not syncing
 
 	// sync guild state to space
-	const spaceKState = await createRoom.roomToKState(spaceID)
+	const spaceKState = await ks.roomToKState(spaceID)
 	const spaceDiff = ks.diffKState(spaceKState, guildKState)
-	await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff)
+	await ks.applyKStateDiffToRoom(spaceID, spaceDiff)
 
 	// guild icon was changed, so room avatars need to be updated as well as the space ones
 	// doing it this way rather than calling syncRoom for great efficiency gains
@@ -122,7 +129,8 @@ async function _syncSpace(guild, shouldActuallySync) {
 		// don't try to update rooms with custom avatars though
 		const roomsWithCustomAvatars = select("channel_room", "room_id", {}, "WHERE custom_avatar IS NOT NULL").pluck().all()
 
-		const childRooms = ks.kstateToState(spaceKState).filter(({type, state_key, content}) => {
+		const state = await ks.kstateToState(spaceKState)
+		const childRooms = state.filter(({type, state_key, content}) => {
 			return type === "m.space.child" && "via" in content && !roomsWithCustomAvatars.includes(state_key)
 		}).map(({state_key}) => state_key)
 
@@ -177,21 +185,19 @@ async function syncSpaceFully(guildID) {
 	const guildKState = await guildToKState(guild, privacy_level)
 
 	// sync guild state to space
-	const spaceKState = await createRoom.roomToKState(spaceID)
+	const spaceKState = await ks.roomToKState(spaceID)
 	const spaceDiff = ks.diffKState(spaceKState, guildKState)
-	await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff)
+	await ks.applyKStateDiffToRoom(spaceID, spaceDiff)
 
-	const childRooms = ks.kstateToState(spaceKState).filter(({type, content}) => {
-		return type === "m.space.child" && "via" in content
-	}).map(({state_key}) => state_key)
+	const childRooms = await api.getFullHierarchy(spaceID)
 
-	for (const roomID of childRooms) {
-		const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
+	for (const {room_id} of childRooms) {
+		const channelID = select("channel_room", "channel_id", {room_id}).pluck().get()
 		if (!channelID) continue
 		if (discord.channels.has(channelID)) {
 			await createRoom.syncRoom(channelID)
 		} else {
-			await createRoom.unbridgeDeletedChannel(channelID, guildID)
+			await createRoom.unbridgeDeletedChannel({id: channelID}, guildID)
 		}
 	}
 
@@ -226,13 +232,13 @@ async function syncSpaceExpressions(data, checkBeforeSync) {
 				// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
 				existing = fn([])
 			}
-			if (deepEqual(existing, content, {strict: true})) return
+			if (isDeepStrictEqual(existing, content)) return
 		}
-		api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content)
+		await api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content)
 	}
 
-	update(spaceID, "emojis", "moe.cadence.ooye.pack.emojis", expression.emojisToState)
-	update(spaceID, "stickers", "moe.cadence.ooye.pack.stickers", expression.stickersToState)
+	await update(spaceID, "emojis", "moe.cadence.ooye.pack.emojis", expression.emojisToState)
+	await update(spaceID, "stickers", "moe.cadence.ooye.pack.stickers", expression.stickersToState)
 }
 
 module.exports.createSpace = createSpace

src/d2m/actions/create-space.test.js

@@ -0,0 +1,38 @@
+// @ts-check
+
+const mixin = require("@cloudrac3r/mixin-deep")
+const {guildToKState, ensureSpace} = require("./create-space")
+const {kstateStripConditionals, kstateUploadMxc} = require("../../matrix/kstate")
+const {test} = require("supertape")
+const testData = require("../../../test/data")
+
+const passthrough = require("../../passthrough")
+const {db} = passthrough
+
+test("guild2space: can generate kstate for a guild, passing privacy level 0", async t => {
+	t.deepEqual(
+		await kstateUploadMxc(kstateStripConditionals(await guildToKState(testData.guild.general, 0))),
+		{
+			"m.room.avatar/": {
+				url: "mxc://cadence.moe/zKXGZhmImMHuGQZWJEFKJbsF"
+			},
+			"m.room.guest_access/": {
+				guest_access: "can_join"
+			},
+			"m.room.history_visibility/": {
+				history_visibility: "invited"
+			},
+			"m.room.join_rules/": {
+				join_rule: "invite"
+			},
+			"m.room.name/": {
+				name: "Psychonauts 3"
+			},
+			"m.room.power_levels/": {
+				users: {
+					"@test_auto_invite:example.org": 100
+				},
+			},
+		}
+	)
+})

src/d2m/actions/delete-message.js

@@ -16,7 +16,6 @@ async function deleteMessage(data) {
 
 	const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all()
 	db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id)
-	db.prepare("DELETE FROM event_message WHERE message_id = ?").run(data.id)
 	for (const eventID of eventsToRedact) {
 		// Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs
 		await api.redactEvent(row.room_id, eventID)
@@ -33,9 +32,8 @@ async function deleteMessageBulk(data) {
 	if (!roomID) return
 
 	const sids = JSON.stringify(data.ids)
-	const eventsToRedact = from("event_message").pluck("event_id").and("WHERE message_id IN (SELECT value FROM json_each(?)").all(sids)
-	db.prepare("DELETE FROM message_channel WHERE message_id IN (SELECT value FROM json_each(?)").run(sids)
-	db.prepare("DELETE FROM event_message WHERE message_id IN (SELECT value FROM json_each(?)").run(sids)
+	const eventsToRedact = from("event_message").pluck("event_id").and("WHERE message_id IN (SELECT value FROM json_each(?))").all(sids)
+	db.prepare("DELETE FROM message_channel WHERE message_id IN (SELECT value FROM json_each(?))").run(sids)
 	for (const eventID of eventsToRedact) {
 		// Awaiting will make it go slower, but since this could be a long-running operation either way, we want to leave rate limit capacity for other operations
 		await api.redactEvent(roomID, eventID)

src/d2m/actions/edit-message.js

@@ -53,7 +53,7 @@ async function editMessage(message, guild, row) {
 	const sendNewEventParts = new Set()
 	for (const promotion of promotions) {
 		if ("eventID" in promotion) {
-			db.prepare(`UPDATE event_message SET ${promotion.column} = 0 WHERE event_id = ?`).run(promotion.eventID)
+			db.prepare(`UPDATE event_message SET ${promotion.column} = ? WHERE event_id = ?`).run(promotion.value ?? 0, promotion.eventID)
 		} else if ("nextEvent" in promotion) {
 			sendNewEventParts.add(promotion.column)
 		}
@@ -61,7 +61,7 @@ async function editMessage(message, guild, row) {
 
 	// 4. Send all the things.
 	if (eventsToSend.length) {
-		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
+		db.prepare("INSERT OR IGNORE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
 	}
 	for (const content of eventsToSend) {
 		const eventType = content.$type

src/d2m/actions/expression.js

@@ -30,7 +30,7 @@ async function emojisToState(emojis) {
 			}
 			db.prepare("INSERT OR IGNORE INTO emoji (emoji_id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(emoji.id, emoji.name, +!!emoji.animated, url)
 		}).catch(e => {
-			if (e.data.errcode === "M_TOO_LARGE") { // Very unlikely to happen. Only possible for 3x-series emojis uploaded shortly after animated emojis were introduced, when there was no 256 KB size limit.
+			if (e.data?.errcode === "M_TOO_LARGE") { // Very unlikely to happen. Only possible for 3x-series emojis uploaded shortly after animated emojis were introduced, when there was no 256 KB size limit.
 				return
 			}
 			console.error(`Trying to handle emoji ${emoji.name} (${emoji.id}), but...`)
@@ -66,7 +66,7 @@ async function stickersToState(stickers) {
 			while (shortcodes.includes(shortcode)) shortcode = shortcode + "~"
 			shortcodes.push(shortcode)
 
-			result.images[shortcodes] = {
+			result.images[shortcode] = {
 				info: {
 					mimetype: file.stickerFormat.get(sticker.format_type)?.mime || "image/png"
 				},

src/d2m/actions/lottie.js

@@ -33,7 +33,7 @@ async function convert(stickerItem) {
 	if (res.status !== 200) throw new Error("Sticker data file not found.")
 	const text = await res.text()
 
-	// Convert to PNG (readable stream)
+	// Convert to PNG (stream.Readable)
 	const readablePng = await convertLottie.convert(text)
 
 	// Upload to MXC

src/d2m/actions/register-pk-user.js

@@ -1,12 +1,11 @@
 // @ts-check
 
 const assert = require("assert")
-const reg = require("../../matrix/read-registration")
+const {reg} = require("../../matrix/read-registration")
 const Ty = require("../../types")
-const fetch = require("node-fetch").default
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select} = passthrough
+const {sync, db, select} = passthrough
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
@@ -33,7 +32,7 @@ async function createSim(pkMessage) {
 	const mxid = `@${localpart}:${reg.ooye.server_name}`
 
 	// Save chosen name in the database forever
-	db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, localpart, mxid)
+	db.prepare("INSERT INTO sim (user_id, username, sim_name, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, simName, mxid)
 
 	// Register matrix user with that name
 	try {
@@ -131,7 +130,7 @@ async function syncUser(author, pkMessage, roomID) {
 	db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username)
 	// Sync the member state
 	const content = await memberToStateContent(pkMessage, author)
-	const currentHash = registerUser._hashProfileContent(content)
+	const currentHash = registerUser._hashProfileContent(content, 0)
 	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
 	// only do the actual sync if the hash has changed since we last looked
 	if (existingHash !== currentHash) {
@@ -142,8 +141,24 @@ async function syncUser(author, pkMessage, roomID) {
 }
 
 /** @returns {Promise<Ty.PkMessage>} */
-function fetchMessage(messageID) {
-	return fetch(`https://api.pluralkit.me/v2/messages/${messageID}`).then(res => res.json())
+async function fetchMessage(messageID) {
+	// Their backend is weird. Sometimes it says "message not found" (code 20006) on the first try, so we make multiple attempts.
+	let attempts = 0
+	do {
+		try {
+			var res = await fetch(`https://api.pluralkit.me/v2/messages/${messageID}`)
+			if (res.ok) return res.json()
+			var errorGetter = res.json
+		} catch (e) {
+			// Catch any network issues too.
+			errorGetter = e.toString
+		}
+
+		// I think the backend needs some time to update.
+		await new Promise(resolve => setTimeout(resolve, 1500))
+	} while (++attempts < 3)
+
+	throw new Error(`PK API returned an error after ${attempts} tries: ${JSON.stringify(await errorGetter())}`)
 }
 
 module.exports._memberToStateContent = memberToStateContent

src/d2m/actions/register-user.js

@@ -1,7 +1,9 @@
 // @ts-check
 
-const assert = require("assert")
-const reg = require("../../matrix/read-registration")
+const assert = require("assert").strict
+const {reg} = require("../../matrix/read-registration")
+const DiscordTypes = require("discord-api-types/v10")
+const mixin = require("@cloudrac3r/mixin-deep")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
@@ -9,6 +11,8 @@ const {discord, sync, db, select} = passthrough
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
+/** @type {import("../../discord/utils")} */
+const utils = sync.require("../../discord/utils")
 /** @type {import("../converters/user-to-mxid")} */
 const userToMxid = sync.require("../converters/user-to-mxid")
 /** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
@@ -18,7 +22,7 @@ require("xxhash-wasm")().then(h => hasher = h)
 
 /**
  * A sim is an account that is being simulated by the bridge to copy events from the other side.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @returns mxid
  */
 async function createSim(user) {
@@ -29,7 +33,7 @@ async function createSim(user) {
 
 	// Save chosen name in the database forever
 	// Making this database change right away so that in a concurrent registration, the 2nd registration will already have generated a different localpart because it can see this row when it generates
-	db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(user.id, simName, localpart, mxid)
+	db.prepare("INSERT INTO sim (user_id, username, sim_name, mxid) VALUES (?, ?, ?, ?)").run(user.id, user.username, simName, mxid)
 
 	// Register matrix user with that name
 	try {
@@ -46,7 +50,7 @@ async function createSim(user) {
 /**
  * Ensure a sim is registered for the user.
  * If there is already a sim, use that one. If there isn't one yet, register a new sim.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @returns {Promise<string>} mxid
  */
 async function ensureSim(user) {
@@ -62,7 +66,7 @@ async function ensureSim(user) {
 
 /**
  * Ensure a sim is registered for the user and is joined to the room.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @param {string} roomID
  * @returns {Promise<string>} mxid
  */
@@ -92,13 +96,13 @@ async function ensureSimJoined(user, roomID) {
 }
 
 /**
- * @param {import("discord-api-types/v10").APIUser} user
- * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user"> | undefined} member
  */
 async function memberToStateContent(user, member, guildID) {
 	let displayname = user.username
 	if (user.global_name) displayname = user.global_name
-	if (member.nick) displayname = member.nick
+	if (member?.nick) displayname = member.nick
 
 	const content = {
 		displayname,
@@ -113,7 +117,7 @@ async function memberToStateContent(user, member, guildID) {
 		}
 	}
 
-	if (member.avatar || user.avatar) {
+	if (member?.avatar || user.avatar) {
 		// const avatarPath = file.userAvatar(user) // the user avatar only
 		const avatarPath = file.memberAvatar(guildID, user, member) // the member avatar or the user avatar
 		content["moe.cadence.ooye.member"].avatar = avatarPath
@@ -123,8 +127,48 @@ async function memberToStateContent(user, member, guildID) {
 	return content
 }
 
-function _hashProfileContent(content) {
-	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
+/**
+ * https://gitdab.com/cadence/out-of-your-element/issues/9
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user"> | undefined} member
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @returns {number} 0 to 100
+ */
+function memberToPowerLevel(user, member, guild, channel) {
+	if (!member) return 0
+
+	const permissions = utils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites)
+	/*
+	 * PL 100 = Administrator = People who can brick the room. RATIONALE:
+	 * 	- Administrator.
+	 * 	- Manage Webhooks: People who remove the webhook can break the room.
+	 * 	- Manage Guild: People who can manage guild can add bots.
+	 * 	- Manage Channels: People who can manage the channel can delete it.
+	 * (Setting sim users to PL 100 is safe because even though we can't demote the sims we can use code to make the sims demote themselves.)
+	 */
+	if (guild.owner_id === user.id || utils.hasSomePermissions(permissions, ["Administrator", "ManageWebhooks", "ManageGuild", "ManageChannels"])) return 100
+	/*
+	 * PL 50 = Moderator = People who can manage people and messages in many ways. RATIONALE:
+	 * 	- Manage Messages: Can moderate by pinning or deleting the conversation.
+	 * 	- Manage Nicknames: Can moderate by removing inappropriate nicknames.
+	 * 	- Manage Threads: Can moderate by deleting conversations.
+	 * 	- Kick Members & Ban Members: Can moderate by removing disruptive people.
+	 * 	- Mute Members & Deafen Members: Can moderate by silencing disruptive people in ways they can't undo.
+	 * 	- Moderate Members.
+	 */
+	if (utils.hasSomePermissions(permissions, ["ManageMessages", "ManageNicknames", "ManageThreads", "KickMembers", "BanMembers", "MuteMembers", "DeafenMembers", "ModerateMembers"])) return 50
+	/* PL 20 = Mention Everyone for technical reasons. */
+	if (utils.hasSomePermissions(permissions, ["MentionEveryone"])) return 20
+	return 0
+}
+
+/**
+ * @param {any} content
+ * @param {number} powerLevel
+ */
+function _hashProfileContent(content, powerLevel) {
+	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}\u0000${powerLevel}`)
 	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
 	return signedHash
 }
@@ -133,48 +177,70 @@ function _hashProfileContent(content) {
  * Sync profile data for a sim user. This function follows the following process:
  * 1. Join the sim to the room if needed
  * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
- * 3. Compare against the previously known state content, which is helpfully stored in the database
- * 4. If the state content has changed, send it to Matrix and update it in the database for next time
- * @param {import("discord-api-types/v10").APIUser} user
- * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ * 3. Calculate the power level the user should get based on their Discord permissions
+ * 4. Compare against the previously known state content, which is helpfully stored in the database
+ * 5. If the state content or power level have changed, send them to Matrix and update them in the database for next time
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user"> | undefined} member
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {string} roomID
  * @returns {Promise<string>} mxid of the updated sim
  */
-async function syncUser(user, member, guildID, roomID) {
+async function syncUser(user, member, channel, guild, roomID) {
 	const mxid = await ensureSimJoined(user, roomID)
-	const content = await memberToStateContent(user, member, guildID)
-	const currentHash = _hashProfileContent(content)
+	const content = await memberToStateContent(user, member, guild.id)
+	const powerLevel = memberToPowerLevel(user, member, guild, channel)
+	const currentHash = _hashProfileContent(content, powerLevel)
 	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
 	// only do the actual sync if the hash has changed since we last looked
-	if (existingHash !== currentHash) {
+	const hashHasChanged = existingHash !== currentHash
+	// however, do not overwrite pre-existing data if we already have data and `member` is not accessible, because this would replace good data with bad data
+	const wouldOverwritePreExisting = existingHash && !member
+	if (hashHasChanged && !wouldOverwritePreExisting) {
+		// Update room member state
 		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+		// Update power levels (only if we can actually access the member roles)
+		const powerLevelsStateContent = await api.getStateEvent(roomID, "m.room.power_levels", "")
+		const oldPowerLevel = powerLevelsStateContent.users?.[mxid] || 0
+		mixin(powerLevelsStateContent, {users: {[mxid]: powerLevel}})
+		if (powerLevel === 0) delete powerLevelsStateContent.users[mxid] // keep the event compact
+		const sendPowerLevelAs = powerLevel < oldPowerLevel ? mxid : undefined // bridge bot won't not have permission to demote equal power users, so do this action as themselves
+		await api.sendState(roomID, "m.room.power_levels", "", powerLevelsStateContent, sendPowerLevelAs)
+		// Update cached hash
 		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
 	}
 	return mxid
 }
 
+/**
+ * @param {string} roomID
+ */
 async function syncAllUsersInRoom(roomID) {
 	const mxids = select("sim_member", "mxid", {room_id: roomID}).pluck().all()
 
 	const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
 	assert.ok(typeof channelID === "string")
 
-	/** @ts-ignore @type {import("discord-api-types/v10").APIGuildChannel} */
+	/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
 	const channel = discord.channels.get(channelID)
 	const guildID = channel.guild_id
 	assert.ok(typeof guildID === "string")
+	/** @ts-ignore @type {DiscordTypes.APIGuild} */
+	const guild = discord.guilds.get(guildID)
 
 	for (const mxid of mxids) {
 		const userID = select("sim", "user_id", {mxid}).pluck().get()
 		assert.ok(typeof userID === "string")
 
-		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIGuildMember>} */
+		/** @ts-ignore @type {Required<DiscordTypes.APIGuildMember>} */
 		const member = await discord.snow.guild.getGuildMember(guildID, userID)
-		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIUser>} user */
+		/** @ts-ignore @type {Required<DiscordTypes.APIUser>} user */
 		const user = member.user
 		assert.ok(user)
 
 		console.log(`[user sync] to matrix: ${user.username} in ${channel.name}`)
-		await syncUser(user, member, guildID, roomID)
+		await syncUser(user, member, channel, guild, roomID)
 	}
 }
 

src/d2m/actions/register-user.test.js

@@ -1,6 +1,6 @@
 const {_memberToStateContent} = require("./register-user")
 const {test} = require("supertape")
-const testData = require("../../test/data")
+const testData = require("../../../test/data")
 
 test("member2state: without member nick or avatar", async t => {
 	t.deepEqual(

src/d2m/actions/remove-reaction.js

@@ -23,16 +23,7 @@ async function removeSomeReactions(data) {
 	const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get()
 	if (!eventIDForMessage) return
 
-	/** @type {Ty.Event.Outer<Ty.Event.M_Reaction>[]} */
-	let reactions = []
-	/** @type {string | undefined} */
-	let nextBatch = undefined
-	do {
-		/** @type {Ty.Pagination<Ty.Event.Outer<Ty.Event.M_Reaction>>} */
-		const res = await api.getRelations(roomID, eventIDForMessage, {from: nextBatch}, "m.annotation")
-		reactions = reactions.concat(res.chunk)
-		nextBatch = res.next_batch
-	} while (nextBatch)
+	const reactions = await api.getFullRelations(roomID, eventIDForMessage, "m.annotation")
 
 	// Run the proper strategy and any strategy-specific database changes
 	const removals = await
@@ -62,7 +53,7 @@ async function removeReaction(data, reactions) {
  */
 async function removeEmojiReaction(data, reactions) {
 	const key = await emojiToKey.emojiToKey(data.emoji)
-	const discordPreferredEncoding = emoji.encodeEmoji(key, undefined)
+	const discordPreferredEncoding = await emoji.encodeEmoji(key, undefined)
 	db.prepare("DELETE FROM reaction WHERE message_id = ? AND encoded_emoji = ?").run(data.message_id, discordPreferredEncoding)
 
 	return converter.removeEmojiReaction(data, reactions, key)

src/d2m/actions/retrigger.js

@@ -0,0 +1,84 @@
+// @ts-check
+
+const {EventEmitter} = require("events")
+const passthrough = require("../../passthrough")
+const {select} = passthrough
+
+const DEBUG_RETRIGGER = false
+
+function debugRetrigger(message) {
+	if (DEBUG_RETRIGGER) {
+		console.log(message)
+	}
+}
+
+const paused = new Set()
+const emitter = new EventEmitter()
+
+/**
+ * Due to Eventual Consistency(TM) an update/delete may arrive before the original message arrives
+ * (or before the it has finished being bridged to an event).
+ * In this case, wait until the original message has finished bridging, then retrigger the passed function.
+ * @template {(...args: any[]) => Promise<any>} T
+ * @param {string} messageID
+ * @param {T} fn
+ * @param {Parameters<T>} rest
+ * @returns {boolean} false if the event was found and the function will be ignored, true if the event was not found and the function will be retriggered
+ */
+function eventNotFoundThenRetrigger(messageID, fn, ...rest) {
+	if (!paused.has(messageID)) {
+		const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get()
+		if (eventID) {
+			debugRetrigger(`[retrigger] OK mid <-> eid = ${messageID} <-> ${eventID}`)
+			return false // event was found so don't retrigger
+		}
+	}
+
+	debugRetrigger(`[retrigger] WAIT mid = ${messageID}`)
+	emitter.once(messageID, () => {
+		debugRetrigger(`[retrigger] TRIGGER mid = ${messageID}`)
+		fn(...rest)
+	})
+	// if the event never arrives, don't trigger the callback, just clean up
+	setTimeout(() => {
+		if (emitter.listeners(messageID).length) {
+			debugRetrigger(`[retrigger] EXPIRE mid = ${messageID}`)
+		}
+		emitter.removeAllListeners(messageID)
+	}, 60 * 1000) // 1 minute
+	return true // event was not found, then retrigger
+}
+
+/**
+ * Anything calling retrigger during the callback will be paused and retriggered after the callback resolves.
+ * @template T
+ * @param {string} messageID
+ * @param {Promise<T>} promise
+ * @returns {Promise<T>}
+ */
+async function pauseChanges(messageID, promise) {
+	try {
+		debugRetrigger(`[retrigger] PAUSE mid = ${messageID}`)
+		paused.add(messageID)
+		return await promise
+	} finally {
+		debugRetrigger(`[retrigger] RESUME mid = ${messageID}`)
+		paused.delete(messageID)
+		messageFinishedBridging(messageID)
+	}
+}
+
+/**
+ * Triggers any pending operations that were waiting on the corresponding event ID.
+ * @param {string} messageID
+ */
+function messageFinishedBridging(messageID) {
+	if (emitter.listeners(messageID).length) {
+		debugRetrigger(`[retrigger] EMIT mid = ${messageID}`)
+	}
+	emitter.emit(messageID)
+}
+
+module.exports.eventNotFoundThenRetrigger = eventNotFoundThenRetrigger
+module.exports.messageFinishedBridging = messageFinishedBridging
+module.exports.pauseChanges = pauseChanges

src/d2m/actions/send-message.js

@@ -1,6 +1,7 @@
 // @ts-check
 
-const assert = require("assert")
+const assert = require("assert").strict
+const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
 const { discord, sync, db } = passthrough
@@ -18,40 +19,34 @@ const createRoom = sync.require("../actions/create-room")
 const dUtils = sync.require("../../discord/utils")
 
 /**
- * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
- * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
  * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
-async function sendMessage(message, guild, row) {
+async function sendMessage(message, channel, guild, row) {
 	const roomID = await createRoom.ensureRoom(message.channel_id)
 
 	let senderMxid = null
 	if (!dUtils.isWebhookMessage(message)) {
-		if (message.member) { // available on a gateway message create event
-			senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID)
-		} else { // well, good enough...
-			senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
+		if (message.author.id === discord.application.id) {
+			// no need to sync the bot's own user
+		} else {
+			senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID)
 		}
 	} else if (row && row.speedbump_webhook_id === message.webhook_id) {
 		// Handle the PluralKit public instance
 		if (row.speedbump_id === "466378653216014359") {
-			const root = await registerPkUser.fetchMessage(message.id)
-			// Member is null if member was deleted. We just got this message, so member surely exists.
-			if (!root.member) {
-				const e = new Error("PK API did not return a member")
-				message["__pk_response__"] = root
-				console.error(root)
-				throw e
-			}
-			senderMxid = await registerPkUser.syncUser(message.author, root, roomID)
+			const pkMessage = await registerPkUser.fetchMessage(message.id)
+			senderMxid = await registerPkUser.syncUser(message.author, pkMessage, roomID)
 		}
 	}
 
 	const events = await messageToEvent.messageToEvent(message, guild, {}, {api})
 	const eventIDs = []
 	if (events.length) {
-		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
-		if (senderMxid) api.sendTyping(roomID, false, senderMxid)
+		db.prepare("INSERT OR IGNORE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
+		if (senderMxid) api.sendTyping(roomID, false, senderMxid).catch(() => {})
 	}
 	for (const event of events) {
 		const part = event === events[0] ? 0 : 1

src/d2m/actions/set-presence.js

@@ -0,0 +1,114 @@
+// @ts-check
+
+const passthrough = require("../../passthrough")
+const {sync, select} = passthrough
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+
+/*
+	We do this in two phases for optimisation reasons.
+	Discord sends us an event when the presence *changes.*
+	We need to keep the event data in memory because we need to *repeatedly* send it to Matrix using a long-lived loop.
+
+	There are two phases to get it from Discord to Matrix.
+	The first phase stores Discord presence data in memory.
+	The second phase loops over the memory and sends it on to Matrix.
+
+	Optimisations:
+	* Presence can be deactivated per-guild in OOYE settings. If the user doesn't share any presence-enabled-guilds with us, we don't need to do anything.
+	* Presence can be sent for users without sims. In this case, they will be discarded from memory when the next loop begins.
+	* Matrix ID is cached in memory on the Presence class. The alternative to this is querying it every time we receive a presence change event in a valid guild.
+	* Presence can be sent multiple times in a row for the same user for each guild we share. The loop timer prevents these "changes" from individually reaching the homeserver.
+*/
+
+// Synapse expires each user's presence after 30 seconds and makes them offline, so we have to loop every 28 seconds and update each user again.
+const presenceLoopInterval = 28e3
+
+// Cache the list of enabled guilds rather than accessing it like multiple times per second when any user changes presence
+const guildPresenceSetting = new class {
+	/** @private @type {Set<string>} */ guilds
+	constructor() {
+		this.update()
+	}
+	update() {
+		this.guilds = new Set(select("guild_space", "guild_id", {presence: 1}).pluck().all())
+	}
+	isEnabled(guildID) {
+		return this.guilds.has(guildID)
+	}
+}
+
+class Presence extends sync.reloadClassMethods(() => Presence) {
+	/** @type {string} */ userID
+	/** @type {{presence: "online" | "offline" | "unavailable", status_msg?: string}} */ data
+	/** @private @type {?string | undefined} */ mxid
+	/** @private @type {number} */ delay = Math.random()
+
+	constructor(userID) {
+		super()
+		this.userID = userID
+	}
+
+	/**
+	 * @param {string} status status field from Discord's PRESENCE_UPDATE event
+	 */
+	setData(status) {
+		const presence =
+			( status === "online" ? "online"
+			: status === "offline" ? "offline"
+			: "unavailable")
+		this.data = {presence}
+	}
+
+	sync(presences) {
+		const mxid = this.mxid ??= select("sim", "mxid", {user_id: this.userID}).pluck().get()
+		if (!mxid) return presences.delete(this.userID)
+		// I haven't tried, but I assume Synapse explodes if you try to update too many presences at the same time.
+		// This random delay will space them out over the whole 28 second cycle.
+		setTimeout(() => {
+			api.setPresence(this.data, mxid).catch(() => {})
+		}, this.delay * presenceLoopInterval).unref()
+	}
+}
+
+const presenceTracker = new class {
+	/** @private @type {Map<string, Presence>} userID -> Presence */ presences = sync.remember(() => new Map())
+
+	constructor() {
+		sync.addTemporaryInterval(() => this.syncPresences(), presenceLoopInterval)
+	}
+
+	/**
+	 * This function is called for each Discord presence packet.
+	 * @param {string} userID Discord user ID
+	 * @param {string} guildID Discord guild ID that this presence applies to (really, the same presence applies to every single guild, but is delivered separately by Discord for some reason)
+	 * @param {string} status status field from Discord's PRESENCE_UPDATE event
+	 */
+	incomingPresence(userID, guildID, status) {
+		// stop tracking offline presence objects - they will naturally expire and fall offline on the homeserver
+		if (status === "offline") return this.presences.delete(userID)
+		// check if we care about this guild
+		if (!guildPresenceSetting.isEnabled(guildID)) return
+		// start tracking presence for user (we'll check if they have a sim in the next sync loop)
+		this.getOrCreatePresence(userID).setData(status)
+	}
+
+	/** @private */
+	getOrCreatePresence(userID) {
+		return this.presences.get(userID) || (() => {
+			const presence = new Presence(userID)
+			this.presences.set(userID, presence)
+			return presence
+		})()
+	}
+
+	/** @private */
+	syncPresences() {
+		for (const presence of this.presences.values()) {
+			presence.sync(this.presences)
+		}
+	}
+}
+
+module.exports.presenceTracker = presenceTracker
+module.exports.guildPresenceSetting = guildPresenceSetting

src/d2m/actions/update-pins.js

@@ -6,6 +6,8 @@ const {discord, sync, db} = passthrough
 const pinsToList = sync.require("../converters/pins-to-list")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
+/** @type {import("../../matrix/kstate")} */
+const ks = sync.require("../../matrix/kstate")
 
 /**
  * @template {string | null | undefined} T
@@ -23,13 +25,20 @@ function convertTimestamp(timestamp) {
  * @param {number?} convertedTimestamp
  */
 async function updatePins(channelID, roomID, convertedTimestamp) {
-	const pins = await discord.snow.channel.getChannelPinnedMessages(channelID)
-	const eventIDs = pinsToList.pinsToList(pins)
-	if (pins.length === eventIDs.length || eventIDs.length) {
-		await api.sendState(roomID, "m.room.pinned_events", "", {
-			pinned: eventIDs
-		})
+	try {
+		var discordPins = await discord.snow.channel.getChannelPinnedMessages(channelID)
+	} catch (e) {
+		if (e.message === `{"message": "Missing Access", "code": 50001}`) {
+			return // Discord sends channel pins update events even for channels that the bot can't view/get pins in, just ignore it
+		}
+		throw e
 	}
+	const pinned = pinsToList.pinsToList(discordPins)
+
+	const kstate = await ks.roomToKState(roomID)
+	const diff = ks.diffKState(kstate, {"m.room.pinned_events/": {pinned}})
+	await ks.applyKStateDiffToRoom(roomID, diff)
+
 	db.prepare("UPDATE channel_room SET last_bridged_pin_timestamp = ? WHERE channel_id = ?").run(convertedTimestamp || 0, channelID)
 }
 

src/d2m/converters/edit-to-changes.js

@@ -3,31 +3,56 @@
 const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select, from} = passthrough
+const {sync, select, from} = passthrough
 /** @type {import("./message-to-event")} */
 const messageToEvent = sync.require("../converters/message-to-event")
-/** @type {import("../actions/register-user")} */
-const registerUser = sync.require("../actions/register-user")
-/** @type {import("../actions/create-room")} */
-const createRoom = sync.require("../actions/create-room")
+/** @type {import("../../m2d/converters/utils")} */
+const utils = sync.require("../../m2d/converters/utils")
+
+function eventCanBeEdited(ev) {
+	// Discord does not allow files, images, attachments, or videos to be edited.
+	if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
+		return false
+	}
+	// Discord does not allow stickers to be edited.
+	if (ev.old.event_type === "m.sticker") {
+		return false
+	}
+	// Anything else is fair game.
+	return true
+}
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
- * IMPORTANT: This may not have all the normal fields! The API documentation doesn't provide possible types, just says it's all optional!
- * Since I don't have a spec, I will have to capture some real traffic and add it as test cases... I hope they don't change anything later...
  * @param {import("discord-api-types/v10").APIGuild} guild
  * @param {import("../../matrix/api")} api simple-as-nails dependency injection for the matrix API
  */
 async function editToChanges(message, guild, api) {
+	// If it is a user edit, allow deleting old messages (e.g. they might have removed text from an image).
+	// If it is the system adding a generated embed to a message, don't delete old messages since the system only sends partial data.
+	// Since an update in August 2024, the system always provides the full data of message updates. I'll leave in the old code since it won't cause problems.
+
+	const isGeneratedEmbed = !("content" in message)
+
 	// Figure out what events we will be replacing
 
 	const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
 	assert(roomID)
-	/** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */
-	const senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
-
 	const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all()
 
+	/** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */
+	let senderMxid = null
+	if (message.author) {
+		senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
+	} else {
+		// Should be a system generated embed. We want the embed to be sent by the same user who sent the message, so that the messages get grouped in most clients.
+		const eventID = oldEventRows[0].event_id // a calling function should have already checked that there is at least one message to edit
+		const event = await api.getEvent(roomID, eventID)
+		if (utils.eventSenderIsFromDiscord(event.sender)) {
+			senderMxid = event.sender
+		}
+	}
+
 	// Figure out what we will be replacing them with
 
 	const newFallbackContent = await messageToEvent.messageToEvent(message, guild, {includeEditFallbackStar: true}, {api})
@@ -48,7 +73,8 @@ async function editToChanges(message, guild, api) {
 	let eventsToRedact = []
 	/** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */
 	let eventsToSend = []
-	//  4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. This is represented as nothing.
+	/**  4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. */
+	let unchangedEvents = []
 
 	function shift() {
 		newFallbackContent.shift()
@@ -81,48 +107,61 @@ async function editToChanges(message, guild, api) {
 		shift()
 	}
 	// Anything remaining in oldEventRows is present in the old version only and should be redacted.
-	eventsToRedact = oldEventRows
+	eventsToRedact = oldEventRows.map(e => ({old: e}))
 
-	// We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times.
-	/** @type {({column: string, eventID: string} | {column: string, nextEvent: true})[]} */
-	const promotions = []
-	for (const column of ["part", "reaction_part"]) {
-		// If no events with part = 0 exist (or will exist), we need to do some management.
-		if (!eventsToReplace.some(e => e.old[column] === 0)) {
-			if (eventsToReplace.length) {
-				// We can choose an existing event to promote. Bigger order is better.
-				const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text")
-				eventsToReplace.sort((a, b) => order(b) - order(a))
-				if (column === "part") {
-					promotions.push({column, eventID: eventsToReplace[0].old.event_id}) // part should be the first one
-				} else {
-					promotions.push({column, eventID: eventsToReplace[eventsToReplace.length - 1].old.event_id}) // reaction_part should be the last one
-				}
-			} else {
-				// No existing events to promote, but new events are being sent. Whatever gets sent will be the next part = 0.
-				promotions.push({column, nextEvent: true})
-			}
-		}
+	// If this is a generated embed update, only allow the embeds to be updated, since the system only sends data about events. Ignore changes to other things.
+	if (isGeneratedEmbed) {
+		unchangedEvents.push(...eventsToRedact.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToRedact to unchangedEvents.
+		eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice")
 	}
 
 	// Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed!
 	// (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.)
 	// So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed.
-	eventsToReplace = eventsToReplace.filter(ev => {
-		// Discord does not allow files, images, attachments, or videos to be edited.
-		if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
-			return false
+	unchangedEvents.push(...eventsToReplace.filter(ev => !eventCanBeEdited(ev))) // Move them from eventsToRedact to unchangedEvents.
+	eventsToReplace = eventsToReplace.filter(eventCanBeEdited)
+
+	// We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times.
+	// This would be disrupted if existing events that are (reaction_)part = 0 will be redacted.
+	// If that is the case, pick a different existing or newly sent event to be (reaction_)part = 0.
+	/** @type {({column: string, eventID: string, value?: number} | {column: string, nextEvent: true})[]} */
+	const promotions = []
+	for (const column of ["part", "reaction_part"]) {
+		const candidatesForParts = unchangedEvents.concat(eventsToReplace)
+		// If no events with part = 0 exist (or will exist), we need to do some management.
+		if (!candidatesForParts.some(e => e.old[column] === 0)) {
+			// Try to find an existing event to promote. Bigger order is better.
+			if (candidatesForParts.length) {
+				const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.old.event_subtype === "m.text")
+				candidatesForParts.sort((a, b) => order(b) - order(a))
+				if (column === "part") {
+					promotions.push({column, eventID: candidatesForParts[0].old.event_id}) // part should be the first one
+				} else if (eventsToSend.length) {
+					promotions.push({column, nextEvent: true}) // reaction_part should be the last one
+				} else {
+					promotions.push({column, eventID: candidatesForParts[candidatesForParts.length - 1].old.event_id}) // reaction_part should be the last one
+				}
+			}
+			// Or, if there are no existing events to promote and new events will be sent, whatever gets sent will be the next part = 0.
+			else {
+				promotions.push({column, nextEvent: true})
+			}
 		}
-		// Discord does not allow stickers to be edited.
-		if (ev.old.event_type === "m.sticker") {
-			return false
+	}
+
+	// If adding events, try to keep reactions attached to the bottom of the group (unless reactions have already been added)
+	if (eventsToSend.length && !promotions.length) {
+		const existingReaction = select("reaction", "message_id", {message_id: message.id}).pluck().get()
+		if (!existingReaction) {
+			const existingPartZero = unchangedEvents.concat(eventsToReplace).find(p => p.old.reaction_part === 0)
+			assert(existingPartZero) // will exist because a reaction_part=0 always exists and no events are being removed
+			promotions.push({column: "reaction_part", eventID: existingPartZero.old.event_id, value: 1}) // update the current reaction_part to 1
+			promotions.push({column: "reaction_part", nextEvent: true}) // the newly created event will have reaction_part = 0
 		}
-		// Anything else is fair game.
-		return true
-	})
+	}
 
 	// Removing unnecessary properties before returning
-	eventsToRedact = eventsToRedact.map(e => e.event_id)
+	eventsToRedact = eventsToRedact.map(e => e.old.event_id)
 	eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)}))
 
 	return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions}

src/d2m/converters/edit-to-changes.test.js

@@ -1,6 +1,6 @@
 const {test} = require("supertape")
 const {editToChanges} = require("./edit-to-changes")
-const data = require("../../test/data")
+const data = require("../../../test/data")
 const Ty = require("../../types")
 
 test("edit2changes: edit by webhook", async t => {
@@ -99,9 +99,9 @@ test("edit2changes: change file type", async t => {
 	t.deepEqual(eventsToRedact, ["$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ"])
 	t.deepEqual(eventsToSend, [{
 		$type: "m.room.message",
-		body: "📝 Uploaded file: https://cdn.discordapp.com/attachments/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt (20 MB)",
+		body: "📝 Uploaded file: https://bridge.example.org/download/discordcdn/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt (20 MB)",
 		format: "org.matrix.custom.html",
-		formatted_body: "📝 Uploaded file: <a href=\"https://cdn.discordapp.com/attachments/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt\">gaze_into_my_dark_mind.txt</a> (20 MB)",
+		formatted_body: "📝 Uploaded file: <a href=\"https://bridge.example.org/download/discordcdn/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt\">gaze_into_my_dark_mind.txt</a> (20 MB)",
 		"m.mentions": {},
 		msgtype: "m.text"
 	}])
@@ -109,7 +109,7 @@ test("edit2changes: change file type", async t => {
 	t.deepEqual(promotions, [{column: "part", nextEvent: true}, {column: "reaction_part", nextEvent: true}])
 })
 
-test("edit2changes: add caption back to that image", async t => {
+test("edit2changes: add caption back to that image (due to it having a reaction, the reaction_part will not be moved)", async t => {
 	const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.added_caption_to_image, data.guild.general, {})
 	t.deepEqual(eventsToRedact, [])
 	t.deepEqual(eventsToSend, [{
@@ -235,3 +235,93 @@ test("edit2changes: promotes the text event when multiple rows have part = 1 (sh
 		}
 	])
 })
+
+test("edit2changes: generated embed", async t => {
+	let called = 0
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.embed_generated_social_media_image, data.guild.general, {
+		async getEvent(roomID, eventID) {
+			called++
+			t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+			t.equal(eventID, "$mPSzglkCu-6cZHbYro0RW2u5mHvbH9aXDjO5FCzosc0")
+			return {sender: "@_ooye_cadence:cadence.moe"}
+		}
+	})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToReplace, [])
+	t.deepEqual(eventsToSend, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via hthrflwrs on cohost"
+			+ "\n| \n| ## This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO https://cohost.org/jkap/post/4794219-empty"
+			+ "\n| \n| 1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:"
+			+ "\n| \n|  * Both players draw eight cards"
+			+ "\n|  * Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand"
+			+ "\n|  * Both players present their best five-or-less-card pok...",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub>hthrflwrs on cohost</sub>`
+			+ `</p><p><strong><a href="https://cohost.org/jkap/post/4794219-empty">This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO</a></strong>`
+			+ `</p><p>1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:`
+			+ `<br><br><ul><li>Both players draw eight cards`
+			+ `</li><li>Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand`
+			+ `</li><li>Both players present their best five-or-less-card pok...</li></ul></p></blockquote>`,
+		"m.mentions": {}
+	}])
+	t.deepEqual(promotions, [{
+		"column": "reaction_part",
+		"eventID": "$mPSzglkCu-6cZHbYro0RW2u5mHvbH9aXDjO5FCzosc0",
+		"value": 1,
+	}, {
+		"column": "reaction_part",
+		"nextEvent": true,
+	}])
+	t.equal(senderMxid, "@_ooye_cadence:cadence.moe")
+	t.equal(called, 1)
+})
+
+test("edit2changes: generated embed on a reply", async t => {
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.embed_generated_on_reply, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$UTqiL3Zj3FC4qldxRLggN1fhygpKl8sZ7XGY5f9MNbF",
+		newContent: {
+			$type: "m.room.message",
+			// Unfortunately the edited message doesn't include the message_reference field. Fine. Whatever. It looks normal if you're using a good client.
+			body: "> a Discord user: [Replied-to message content wasn't provided by Discord]"
+				+ "\n\n* https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$aLVZyiC3HlOu-prCSIaXlQl68I8leUdnPFiCwkgn6qM",
+			format: "org.matrix.custom.html",
+			formatted_body: "<mx-reply><blockquote><a href=\"https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$aLVZyiC3HlOu-prCSIaXlQl68I8leUdnPFiCwkgn6qM\">In reply to</a> a Discord user<br>[Replied-to message content wasn't provided by Discord]</blockquote></mx-reply>* <a href=\"https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$aLVZyiC3HlOu-prCSIaXlQl68I8leUdnPFiCwkgn6qM\">https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$aLVZyiC3HlOu-prCSIaXlQl68I8leUdnPFiCwkgn6qM</a>",
+			"m.mentions": {},
+			"m.new_content": {
+				body: "https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$aLVZyiC3HlOu-prCSIaXlQl68I8leUdnPFiCwkgn6qM",
+				format: "org.matrix.custom.html",
+				formatted_body: "<a href=\"https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$aLVZyiC3HlOu-prCSIaXlQl68I8leUdnPFiCwkgn6qM\">https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$aLVZyiC3HlOu-prCSIaXlQl68I8leUdnPFiCwkgn6qM</a>",
+				"m.mentions": {},
+				msgtype: "m.text",
+			},
+			"m.relates_to": {
+				event_id: "$UTqiL3Zj3FC4qldxRLggN1fhygpKl8sZ7XGY5f9MNbF",
+				rel_type: "m.replace",
+			},
+			msgtype: "m.text",
+		},
+	}])
+	t.deepEqual(eventsToSend, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## Matrix - Decentralised and secure communication https://matrix.to/"
+			+ "\n| \n| You're invited to talk on Matrix. If you don't already have a client this link will help you pick one, and join the conversation. If you already have one, this link will help you join the conversation",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><strong><a href="https://matrix.to/">Matrix - Decentralised and secure communication</a></strong>`
+			+ `</p><p>You're invited to talk on Matrix. If you don't already have a client this link will help you pick one, and join the conversation. If you already have one, this link will help you join the conversation</p></blockquote>`,
+		"m.mentions": {}
+	}])
+	t.deepEqual(promotions, [{
+		"column": "reaction_part",
+		"eventID": "$UTqiL3Zj3FC4qldxRLggN1fhygpKl8sZ7XGY5f9MNbF",
+		"value": 1,
+	}, {
+		"column": "reaction_part",
+		"nextEvent": true,
+	}])
+	t.equal(senderMxid, "@_ooye_cadence:cadence.moe")
+})

src/d2m/converters/emoji-to-key.test.js

@@ -2,7 +2,7 @@
 
 const {test} = require("supertape")
 const {emojiToKey} = require("./emoji-to-key")
-const data = require("../../test/data")
+const data = require("../../../test/data")
 const Ty = require("../../types")
 
 test("emoji2key: unicode emoji works", async t => {

src/d2m/converters/lottie.js

@@ -2,7 +2,7 @@
 
 const assert = require("assert")
 const stream = require("stream")
-const {PNG} = require("pngjs")
+const {PNG} = require("@cloudrac3r/pngjs")
 
 const SIZE = 160 // Discord's display size on 1x displays is 160
 
@@ -21,7 +21,7 @@ const Rlottie = (async () => {
 
 /**
  * @param {string} text
- * @returns {Promise<import("stream").Readable>}
+ * @returns {Promise<stream.Readable>}
  */
 async function convert(text) {
 	const r = await Rlottie
@@ -41,6 +41,7 @@ async function convert(text) {
 	png.data = Buffer.from(rendered)
 	// png.pack() is a bad stream and will throw away any data it sends if it's not connected to a destination straight away.
 	// We use Duplex.from to convert it into a good stream.
+	// @ts-ignore
 	return stream.Duplex.from(png.pack())
 }
 

src/d2m/converters/message-to-event.embeds.test.js

@@ -0,0 +1,369 @@
+const {test} = require("supertape")
+const {messageToEvent} = require("./message-to-event")
+const data = require("../../../test/data")
+const {db} = require("../../passthrough")
+
+test("message2event embeds: nothing but a field", async t => {
+	const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		body: "> ↪️ @papiophidian: used `/stats`",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote>↪️ <a href=\"https://matrix.to/#/@_ooye_papiophidian:cadence.moe\">@papiophidian</a> used <code>/stats</code></blockquote>",
+		"m.mentions": {},
+		msgtype: "m.text",
+	}, {
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.notice",
+		body: "| ### Amanda 🎵#2192 :online:"
+			+ "\n| willow tree, branch 0"
+			+ "\n| **❯ Uptime:**\n| 3m 55s\n| **❯ Memory:**\n| 64.45MB",
+		format: "org.matrix.custom.html",
+		formatted_body: '<blockquote><p><strong>Amanda 🎵#2192 <img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ\" title=\":online:\" alt=\":online:\">'
+			+ '<br>willow tree, branch 0</strong>'
+			+ '<br><strong>❯ Uptime:</strong><br>3m 55s'
+			+ '<br><strong>❯ Memory:</strong><br>64.45MB</p></blockquote>'
+	}])
+})
+
+test("message2event embeds: reply with just an embed", async t => {
+	const events = await messageToEvent(data.message_with_embeds.reply_with_only_embed, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		"m.mentions": {},
+		body: "> In reply to an unbridged message:"
+      	+ "\n> PokemonGod: https://twitter.com/dynastic/status/1707484191963648161"
+			+ "\n\n| ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
+			+ "\n| \n| does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
+			+ "\n| \n| ### Retweets"
+			+ "\n| 119"
+			+ "\n| \n| ### Likes"
+			+ "\n| 5581"
+			+ "\n| — Twitter",
+		format: "org.matrix.custom.html",
+		formatted_body: '<blockquote>In reply to an unbridged message from PokemonGod:<br><a href=\"https://twitter.com/dynastic/status/1707484191963648161\">https://twitter.com/dynastic/status/1707484191963648161</a></blockquote>'
+			+ '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong>'
+			+ '</p><p>does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
+			+ '</p><p><strong>Retweets</strong><br>119</p><p><strong>Likes</strong><br>5581</p>— Twitter</blockquote>'
+	}])
+})
+
+test("message2event embeds: image embed and attachment", async t => {
+	const events = await messageToEvent(data.message_with_embeds.image_embed_and_attachment, data.guild.general, {}, {
+		api: {
+			async getJoinedMembers(roomID) {
+				return {joined: []}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://tootsuite.net/Warp-Gate2.gif\ntanget: @ monster spawner",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://tootsuite.net/Warp-Gate2.gif">https://tootsuite.net/Warp-Gate2.gif</a><br>tanget: @ monster spawner',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.image",
+		url: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR",
+		body: "Screenshot_20231001_034036.jpg",
+		external_url: "https://bridge.example.org/download/discordcdn/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg",
+		filename: "Screenshot_20231001_034036.jpg",
+		info: {
+			h: 1170,
+			w: 1080,
+			size: 51981,
+			mimetype: "image/jpeg"
+		},
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: blockquote in embed", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:example.invalid": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: ":emoji: **4 |** #wonderland",
+		format: "org.matrix.custom.html",
+		formatted_body: `<img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO\" title=\":emoji:\" alt=\":emoji:\"> <strong>4 |</strong> <a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe?via=cadence.moe&via=example.invalid\">#wonderland</a>`,
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n| \n| reply draft\n| > The following is a message composed via consensus of the Stinker Council.\n| > \n| > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n| > \n| > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n| > \n| > There will be no further communication.\n| \n| [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote><p><strong><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">⏺️ minimus</a></strong></p><p>reply draft<br><blockquote>The following is a message composed via consensus of the Stinker Council.<br><br>For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.<br><br>Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.<br><br>There will be no further communication.</blockquote></p><p><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">Go to Message</a></p></blockquote>",
+		"m.mentions": {}
+	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
+})
+
+test("message2event embeds: crazy html is all escaped", async t => {
+	const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## ⏺️ <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n| \n| ## <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n| \n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| \n| ### <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| — <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote>`
+			+ `<p><strong><a href="https://a.co/&amp;amp;&lt;script&gt;">⏺️ &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</a></strong></p>`
+			+ `<p><strong><a href=\"https://a.co/&amp;amp;&lt;script&gt;">&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href="https://a.co/&amp;amp">https://a.co/&amp;amp</a>;)</a></strong></p>`
+			+ `<p>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
+			+ `<p><strong>&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href=\"https://a.co/&amp;amp\">https://a.co/&amp;amp</a>;)</strong>`
+			+ `<br>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
+			+ `— &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: title without url", async t => {
+	const events = await messageToEvent(data.message_with_embeds.title_without_url, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		body: "> ↪️ @papiophidian: used `/stats`",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote>↪️ <a href=\"https://matrix.to/#/@_ooye_papiophidian:cadence.moe\">@papiophidian</a> used <code>/stats</code></blockquote>",
+		"m.mentions": {},
+		msgtype: "m.text",
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## Hi, I'm Amanda!\n| \n| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><strong>Hi, I'm Amanda!</strong></p><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: url without title", async t => {
+	const events = await messageToEvent(data.message_with_embeds.url_without_title, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		body: "> ↪️ @papiophidian: used `/stats`",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote>↪️ <a href=\"https://matrix.to/#/@_ooye_papiophidian:cadence.moe\">@papiophidian</a> used <code>/stats</code></blockquote>",
+		"m.mentions": {},
+		msgtype: "m.text",
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: author without url", async t => {
+	const events = await messageToEvent(data.message_with_embeds.author_without_url, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		body: "> ↪️ @papiophidian: used `/stats`",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote>↪️ <a href=\"https://matrix.to/#/@_ooye_papiophidian:cadence.moe\">@papiophidian</a> used <code>/stats</code></blockquote>",
+		"m.mentions": {},
+		msgtype: "m.text",
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## Amanda\n| \n| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><strong>Amanda</strong></p><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: author url without name", async t => {
+	const events = await messageToEvent(data.message_with_embeds.author_url_without_name, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		body: "> ↪️ @papiophidian: used `/stats`",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote>↪️ <a href=\"https://matrix.to/#/@_ooye_papiophidian:cadence.moe\">@papiophidian</a> used <code>/stats</code></blockquote>",
+		"m.mentions": {},
+		msgtype: "m.text",
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: vx image", async t => {
+	const events = await messageToEvent(data.message_with_embeds.vx_image, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://vxtwitter.com/TomorrowCorp/status/1760330671074287875 we got a release date!!!",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://vxtwitter.com/TomorrowCorp/status/1760330671074287875">https://vxtwitter.com/TomorrowCorp/status/1760330671074287875</a> we got a release date!!!',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via vxTwitter / fixvx https://github.com/dylanpdx/BetterTwitFix"
+			+ "\n| "
+			+ "\n| ## Twitter https://twitter.com/tomorrowcorp/status/1760330671074287875"
+			+ "\n| "
+			+ "\n| ## Tomorrow Corporation (@TomorrowCorp) https://vxtwitter.com/TomorrowCorp/status/1760330671074287875"
+			+ "\n| "
+			+ "\n| Mark your calendar with a wet black stain! World of Goo 2 releases on May 23, 2024 on Nintendo Switch, Epic Games Store (Win/Mac), and http://WorldOfGoo2.com (Win/Mac/Linux)."
+			+ "\n| "
+			+ "\n| https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms"
+			+ "\n| "
+			+ "\n| 💖 123 🔁 36"
+			+ "\n| "
+			+ "\n| 📸 https://pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://github.com/dylanpdx/BetterTwitFix">vxTwitter / fixvx</a></sub>`
+			+ `</p><p><strong><a href="https://twitter.com/tomorrowcorp/status/1760330671074287875">Twitter</a></strong>`
+			+ `</p><p><strong><a href="https://vxtwitter.com/TomorrowCorp/status/1760330671074287875">Tomorrow Corporation (@TomorrowCorp)</a></strong>`
+			+ `</p><p>Mark your calendar with a wet black stain! World of Goo 2 releases on May 23, 2024 on Nintendo Switch, Epic Games Store (Win/Mac), and <a href="http://WorldOfGoo2.com">http://WorldOfGoo2.com</a> (Win/Mac/Linux).`
+			+ `<br><br><a href="https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms">https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms</a>`
+			+ `<br><br>💖 123 🔁 36`
+			+ `</p><p>📸 https://pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: vx video", async t => {
+	const events = await messageToEvent(data.message_with_embeds.vx_video, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://vxtwitter.com/McDonalds/status/1759971752254341417",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://vxtwitter.com/McDonalds/status/1759971752254341417">https://vxtwitter.com/McDonalds/status/1759971752254341417</a>',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via vxTwitter / fixvx https://github.com/dylanpdx/BetterTwitFix"
+			+ "\n| \n| ## McDonald’s🤝@studiopierrot"
+			+ "\n| \n| 💖 89 🔁 21 https://twitter.com/McDonalds/status/1759971752254341417"
+			+ "\n| \n| ## McDonald's (@McDonalds) https://vxtwitter.com/McDonalds/status/1759971752254341417"
+			+ "\n| \n| 🎞️ https://video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4?tag=12",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://github.com/dylanpdx/BetterTwitFix">vxTwitter / fixvx</a></sub>`
+			+ `</p><p><strong><a href="https://twitter.com/McDonalds/status/1759971752254341417">McDonald’s🤝@studiopierrot\n\n💖 89 🔁 21</a></strong>`
+			+ `</p><p><strong><a href="https://vxtwitter.com/McDonalds/status/1759971752254341417">McDonald's (@McDonalds)</a></strong>`
+			+ `</p><p>🎞️ https://video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4?tag=12</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: youtube video", async t => {
+	const events = await messageToEvent(data.message_with_embeds.youtube_video, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E\n\n\nJutomi I'm gonna make these sounds in your walls tonight",
+		format: "org.matrix.custom.html",
+		formatted_body: `<a href="https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E">https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E</a><br><br><br>Jutomi I'm gonna make these sounds in your walls tonight`,
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via YouTube https://www.youtube.com"
+			+ "\n| \n| ## Happy O Funny https://www.youtube.com/channel/UCEpQ9aEb1NafpvWp5Aoizrg"
+      	+ "\n| \n| ## Shoebill stork clattering sounds like machine guun~!! (Japan Matsue... https://www.youtube.com/watch?v=kDMHHw8JqLE"
+			+ "\n| \n| twitter"
+			+ "\n| https://twitter.com/matsuevogelpark"
+			+ "\n| \n| The shoebill (Balaeniceps rex) also known as whalehead, whale-headed stork, or shoe-billed stork, is a very large stork-like bird. It derives its name from its enormous shoe-shaped bill"
+			+ "\n| some people also called them the living dinosaur~~"
+			+ "\n| \n| #shoebill #livingdinosaur #happyofunny #weirdcreature #weirdsoun..."
+			+ "\n| \n| 🎞️ https://www.youtube.com/embed/kDMHHw8JqLE",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://www.youtube.com">YouTube</a></sub></p>`
+			+ `<p><strong><a href="https://www.youtube.com/channel/UCEpQ9aEb1NafpvWp5Aoizrg">Happy O Funny</a></strong>`
+			+ `</p><p><strong><a href="https://www.youtube.com/watch?v=kDMHHw8JqLE">Shoebill stork clattering sounds like machine guun~!! (Japan Matsue...</a></strong>`
+			+ `</p><p>twitter<br><a href="https://twitter.com/matsuevogelpark">https://twitter.com/matsuevogelpark</a><br><br>The shoebill (Balaeniceps rex) also known as whalehead, whale-headed stork, or shoe-billed stork, is a very large stork-like bird. It derives its name from its enormous shoe-shaped bill<br>some people also called them the living dinosaur~~<br><br>#shoebill #livingdinosaur #happyofunny #weirdcreature #weirdsoun...`
+			+ `</p><p>🎞️ https://www.youtube.com/embed/kDMHHw8JqLE`
+			+ `</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: if discord creates an embed preview for a discord channel link, don't copy that embed", async t => {
+	const events = await messageToEvent(data.message_with_embeds.discord_server_included_punctuation_bad_discord, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "(test https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&via=matrix.org)",
+		format: "org.matrix.custom.html",
+		formatted_body: `(test <a href="https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&amp;via=matrix.org">https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&amp;via=matrix.org</a>)`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: nothing generated if embeds are disabled in settings", async t => {
+	db.prepare("UPDATE guild_space SET url_preview = 0 WHERE guild_id = ?").run(data.guild.general.id)
+	const events = await messageToEvent(data.message_with_embeds.youtube_video, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E\n\n\nJutomi I'm gonna make these sounds in your walls tonight",
+		format: "org.matrix.custom.html",
+		formatted_body: `<a href="https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E">https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E</a><br><br><br>Jutomi I'm gonna make these sounds in your walls tonight`,
+		"m.mentions": {}
+	}])
+})

src/d2m/converters/message-to-event.js

@@ -1,10 +1,10 @@
 // @ts-check
 
 const assert = require("assert").strict
-const markdown = require("discord-markdown")
+const markdown = require("@cloudrac3r/discord-markdown")
 const pb = require("prettier-bytes")
 const DiscordTypes = require("discord-api-types/v10")
-const {tag} = require("html-template-tag")
+const {tag} = require("@cloudrac3r/html-template-tag")
 
 const passthrough = require("../../passthrough")
 const {sync, db, discord, select, from} = passthrough
@@ -18,7 +18,7 @@ const lottie = sync.require("../actions/lottie")
 const mxUtils = sync.require("../../m2d/converters/utils")
 /** @type {import("../../discord/utils")} */
 const dUtils = sync.require("../../discord/utils")
-const reg = require("../../matrix/read-registration")
+const {reg} = require("../../matrix/read-registration")
 
 const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
 
@@ -32,7 +32,10 @@ function getDiscordParseCallbacks(message, guild, useHTML) {
 		/** @param {{id: string, type: "discordUser"}} node */
 		user: node => {
 			const mxid = select("sim", "mxid", {user_id: node.id}).pluck().get()
-			const username = message.mentions.find(ment => ment.id === node.id)?.username || node.id
+			const interaction = message.interaction_metadata || message.interaction
+			const username = message.mentions.find(ment => ment.id === node.id)?.username
+				|| (interaction?.user.id === node.id ? interaction.user.username : null)
+				|| node.id
 			if (mxid && useHTML) {
 				return `<a href="https://matrix.to/#/${mxid}">@${username}</a>`
 			} else {
@@ -58,7 +61,7 @@ function getDiscordParseCallbacks(message, guild, useHTML) {
 		emoji: node => {
 			if (useHTML) {
 				const mxc = select("emoji", "mxc_url", {emoji_id: node.id}).pluck().get()
-				assert(mxc) // All emojis should have been added ahead of time in the messageToEvent function.
+				assert(mxc, `Emoji consistency assertion failed for ${node.name}:${node.id}`) // All emojis should have been added ahead of time in the messageToEvent function.
 				return `<img data-mx-emoticon height="32" src="${mxc}" title=":${node.name}:" alt=":${node.name}:">`
 			} else {
 				return `:${node.name}:`
@@ -100,6 +103,7 @@ const embedTitleParser = markdown.markdownEngine.parserFor({
  * @param {DiscordTypes.APIAttachment} attachment
  */
 async function attachmentToEvent(mentions, attachment) {
+	const external_url = dUtils.getPublicUrlForCdn(attachment.url)
 	const emoji =
 		attachment.content_type?.startsWith("image/jp") ? "📸"
 		: attachment.content_type?.startsWith("image/") ? "🖼️"
@@ -113,9 +117,9 @@ async function attachmentToEvent(mentions, attachment) {
 			$type: "m.room.message",
 			"m.mentions": mentions,
 			msgtype: "m.text",
-			body: `${emoji} Uploaded SPOILER file: ${attachment.url} (${pb(attachment.size)})`,
+			body: `${emoji} Uploaded SPOILER file: ${external_url} (${pb(attachment.size)})`,
 			format: "org.matrix.custom.html",
-			formatted_body: `<blockquote>${emoji} Uploaded SPOILER file: <a href="${attachment.url}">${attachment.url}</a> (${pb(attachment.size)})</blockquote>`
+			formatted_body: `<blockquote>${emoji} Uploaded SPOILER file: <a href="${external_url}">${external_url}</a> (${pb(attachment.size)})</blockquote>`
 		}
 	}
 	// for large files, always link them instead of uploading so I don't use up all the space in the content repo
@@ -124,9 +128,9 @@ async function attachmentToEvent(mentions, attachment) {
 			$type: "m.room.message",
 			"m.mentions": mentions,
 			msgtype: "m.text",
-			body: `${emoji} Uploaded file: ${attachment.url} (${pb(attachment.size)})`,
+			body: `${emoji} Uploaded file: ${external_url} (${pb(attachment.size)})`,
 			format: "org.matrix.custom.html",
-			formatted_body: `${emoji} Uploaded file: <a href="${attachment.url}">${attachment.filename}</a> (${pb(attachment.size)})`
+			formatted_body: `${emoji} Uploaded file: <a href="${external_url}">${attachment.filename}</a> (${pb(attachment.size)})`
 		}
 	} else if (attachment.content_type?.startsWith("image/") && attachment.width && attachment.height) {
 		return {
@@ -134,7 +138,7 @@ async function attachmentToEvent(mentions, attachment) {
 			"m.mentions": mentions,
 			msgtype: "m.image",
 			url: await file.uploadDiscordFileToMxc(attachment.url),
-			external_url: attachment.url,
+			external_url,
 			body: attachment.description || attachment.filename,
 			filename: attachment.filename,
 			info: {
@@ -150,7 +154,7 @@ async function attachmentToEvent(mentions, attachment) {
 			"m.mentions": mentions,
 			msgtype: "m.video",
 			url: await file.uploadDiscordFileToMxc(attachment.url),
-			external_url: attachment.url,
+			external_url,
 			body: attachment.description || attachment.filename,
 			filename: attachment.filename,
 			info: {
@@ -166,13 +170,13 @@ async function attachmentToEvent(mentions, attachment) {
 			"m.mentions": mentions,
 			msgtype: "m.audio",
 			url: await file.uploadDiscordFileToMxc(attachment.url),
-			external_url: attachment.url,
+			external_url,
 			body: attachment.description || attachment.filename,
 			filename: attachment.filename,
 			info: {
 				mimetype: attachment.content_type,
 				size: attachment.size,
-				duration: attachment.duration_secs ? attachment.duration_secs * 1000 : undefined
+				duration: attachment.duration_secs ? Math.round(attachment.duration_secs * 1000) : undefined
 			}
 		}
 	} else {
@@ -181,7 +185,7 @@ async function attachmentToEvent(mentions, attachment) {
 			"m.mentions": mentions,
 			msgtype: "m.file",
 			url: await file.uploadDiscordFileToMxc(attachment.url),
-			external_url: attachment.url,
+			external_url,
 			body: attachment.description || attachment.filename,
 			filename: attachment.filename,
 			info: {
@@ -193,11 +197,13 @@ async function attachmentToEvent(mentions, attachment) {
 }
 
 /**
- * @param {import("discord-api-types/v10").APIMessage} message
- * @param {import("discord-api-types/v10").APIGuild} guild
- * @param {{includeReplyFallback?: boolean, includeEditFallbackStar?: boolean}} options default values:
+ * @param {DiscordTypes.APIMessage} message
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {{includeReplyFallback?: boolean, includeEditFallbackStar?: boolean, alwaysReturnFormattedBody?: boolean, scanTextForMentions?: boolean}} options default values:
  * - includeReplyFallback: true
  * - includeEditFallbackStar: false
+ * - alwaysReturnFormattedBody: false - formatted_body will be skipped if it is the same as body because the message is plaintext. if you want the formatted_body to be returned anyway, for example to merge it with another message, then set this to true.
+ * - scanTextForMentions: true - needs to be set to false when converting forwarded messages etc which may be from a different channel that can't be scanned.
  * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API
  */
 async function messageToEvent(message, guild, options = {}, di) {
@@ -229,6 +235,16 @@ async function messageToEvent(message, guild, options = {}, di) {
 		}]
 	}
 
+	const interaction = message.interaction_metadata || message.interaction
+	if (message.type === DiscordTypes.MessageType.ChatInputCommand && interaction && "name" in interaction) {
+		// Commands are sent by the responding bot. Need to attach the metadata of the person using the command at the top.
+		let content = message.content
+		if (content) content = `\n${content}`
+		else if ((message.flags || 0) & DiscordTypes.MessageFlags.Loading) content = " — interaction loading..."
+		content = `> ↪️ <@${interaction.user.id}> used \`/${interaction.name}\`${content}`
+		message = {...message, content} // editToChanges reuses the object so we can't mutate it. have to clone it
+	}
+
 	/**
 	   @type {{room?: boolean, user_ids?: string[]}}
 		We should consider the following scenarios for mentions:
@@ -246,9 +262,13 @@ async function messageToEvent(message, guild, options = {}, di) {
 				- So make sure we don't do anything in this case.
 	*/
 	const mentions = {}
+	/** @type {{event_id: string, room_id: string, source: number}?} */
 	let repliedToEventRow = null
+	let repliedToUnknownEvent = false
 	let repliedToEventSenderMxid = null
 
+	if (message.mention_everyone) mentions.room = true
+
 	function addMention(mxid) {
 		if (!mentions.user_ids) mentions.user_ids = []
 		if (!mentions.user_ids.includes(mxid)) mentions.user_ids.push(mxid)
@@ -260,6 +280,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 		const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
 		if (row) {
 			repliedToEventRow = row
+		} else if (message.referenced_message) {
+			repliedToUnknownEvent = true
 		}
 	} else if (dUtils.isWebhookMessage(message) && message.embeds[0]?.author?.name?.endsWith("↩️")) {
 		// It could be a PluralKit emulated reply, let's see if it has a message link
@@ -336,8 +358,13 @@ async function messageToEvent(message, guild, options = {}, di) {
 					result = `https://matrix.to/#/${roomID}/${eventID}?${via}`
 				} else {
 					const ts = dUtils.snowflakeToTimestampExact(messageID)
-					const {event_id} = await di.api.getEventForTimestamp(roomID, ts)
-					result = `https://matrix.to/#/${roomID}/${event_id}?${via}`
+					try {
+						const {event_id} = await di.api.getEventForTimestamp(roomID, ts)
+						result = `https://matrix.to/#/${roomID}/${event_id}?${via}`
+					} catch (e) {
+						// M_NOT_FOUND: Unable to find event from <ts> in direction Direction.FORWARDS
+						result = `[unknown event, timestamp resolution failed, in room: https://matrix.to/#/${roomID}?${via}]`
+					}
 				}
 			} else {
 				result = `${match[0]} [event is from another server]`
@@ -349,6 +376,13 @@ async function messageToEvent(message, guild, options = {}, di) {
 		return content
 	}
 
+	/**
+	 * Translate Discord attachment links into links that go via the bridge, so they last forever.
+	 */
+	function transformAttachmentLinks(content) {
+		return content.replace(/https:\/\/(cdn|media)\.discordapp\.(?:com|net)\/attachments\/([0-9]+)\/([0-9]+)\/([-A-Za-z0-9_.,]+)/g, url => dUtils.getPublicUrlForCdn(url))
+	}
+
 	/**
 	 * Translate links and emojis and mentions and stuff. Give back the text and HTML so they can be combined into bigger events.
 	 * @param {string} content Partial or complete Discord message content
@@ -357,11 +391,12 @@ async function messageToEvent(message, guild, options = {}, di) {
 	 * @param {any} customHtmlOutput
 	 */
 	async function transformContent(content, customOptions = {}, customParser = null, customHtmlOutput = null) {
+		content = transformAttachmentLinks(content)
 		content = await transformContentMessageLinks(content)
 
 		// Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter.
 		// So we scan the message ahead of time for all its emojis and ensure they are in the DB.
-		const emojiMatches = [...content.matchAll(/<(a?):([^:>]{2,64}):([0-9]+)>/g)]
+		const emojiMatches = [...content.matchAll(/<(a?):([^:>]{1,64}):([0-9]+)>/g)]
 		await Promise.all(emojiMatches.map(match => {
 			const id = match[3]
 			const name = match[2]
@@ -394,13 +429,18 @@ async function messageToEvent(message, guild, options = {}, di) {
 			discordOnly: true,
 			escapeHTML: false,
 			...customOptions
-		}, null, null)
+		})
 
 		return {body, html}
 	}
 
-	// FIXME: What was the scanMentions parameter supposed to activate? It's unused.
-	async function addTextEvent(body, html, msgtype, {scanMentions}) {
+	/**
+	 * After converting Discord content to Matrix plaintext and HTML content, post-process the bodies and push the resulting text event
+	 * @param {string} body matrix event plaintext body
+	 * @param {string} html matrix event HTML body
+	 * @param {string} msgtype matrix event msgtype (maybe m.text or m.notice)
+	 */
+	async function addTextEvent(body, html, msgtype) {
 		// Star * prefix for fallback edits
 		if (options.includeEditFallbackStar) {
 			body = "* " + body
@@ -408,14 +448,14 @@ async function messageToEvent(message, guild, options = {}, di) {
 		}
 
 		const flags = message.flags || 0
-		if (flags & 2) {
+		if (flags & DiscordTypes.MessageFlags.IsCrosspost) {
 			body = `[🔀 ${message.author.username}]\n` + body
 			html = `🔀 <strong>${message.author.username}</strong><br>` + html
 		}
 
 		// Fallback body/formatted_body for replies
 		// This branch is optional - do NOT change anything apart from the reply fallback, since it may not be run
-		if (repliedToEventRow && options.includeReplyFallback !== false) {
+		if ((repliedToEventRow || repliedToUnknownEvent) && options.includeReplyFallback !== false) {
 			let repliedToDisplayName
 			let repliedToUserHtml
 			if (repliedToEventRow?.source === 0 && repliedToEventSenderMxid) {
@@ -428,27 +468,50 @@ async function messageToEvent(message, guild, options = {}, di) {
 				repliedToUserHtml = repliedToDisplayName
 			}
 			let repliedToContent = message.referenced_message?.content
-			if (repliedToContent?.startsWith("> <:L1:")) {
+			if (repliedToContent?.match(/^(-# )?> (-# )?<:L1:/)) {
 				// If the Discord user is replying to a Matrix user's reply, the fallback is going to contain the emojis and stuff from the bridged rep of the Matrix user's reply quote.
 				// Need to remove that previous reply rep from this fallback body. The fallbody body should only contain the Matrix user's actual message.
-				repliedToContent = repliedToContent.split("\n").slice(2).join("\n")
+				//                                            ┌──────A─────┐       A reply rep starting with >quote or -#smalltext >quote. Match until the end of the line.
+				//                                            ┆            ┆┌─B─┐  There may be up to 2 reply rep lines in a row if it was created in the old format. Match all lines.
+				repliedToContent = repliedToContent.replace(/^((-# )?> .*\n){1,2}/, "")
 			}
 			if (repliedToContent == "") repliedToContent = "[Media]"
 			else if (!repliedToContent) repliedToContent = "[Replied-to message content wasn't provided by Discord]"
 			const repliedToHtml = markdown.toHTML(repliedToContent, {
 				discordCallback: getDiscordParseCallbacks(message, guild, true)
-			}, null, null)
+			})
 			const repliedToBody = markdown.toHTML(repliedToContent, {
 				discordCallback: getDiscordParseCallbacks(message, guild, false),
 				discordOnly: true,
 				escapeHTML: false,
-			}, null, null)
-			html = `<mx-reply><blockquote><a href="https://matrix.to/#/${repliedToEventRow.room_id}/${repliedToEventRow.event_id}">In reply to</a> ${repliedToUserHtml}`
-				+ `<br>${repliedToHtml}</blockquote></mx-reply>`
-				+ html
-			body = (`${repliedToDisplayName}: ` // scenario 1 part B for mentions
-				+ repliedToBody).split("\n").map(line => "> " + line).join("\n")
-				+ "\n\n" + body
+			})
+			if (repliedToEventRow) {
+				// Generate a reply pointing to the Matrix event we found
+				html = `<mx-reply><blockquote><a href="https://matrix.to/#/${repliedToEventRow.room_id}/${repliedToEventRow.event_id}">In reply to</a> ${repliedToUserHtml}`
+					+ `<br>${repliedToHtml}</blockquote></mx-reply>`
+					+ html
+				body = (`${repliedToDisplayName}: ` // scenario 1 part B for mentions
+					+ repliedToBody).split("\n").map(line => "> " + line).join("\n")
+					+ "\n\n" + body
+			} else { // repliedToUnknownEvent
+				// This reply can't point to the Matrix event because it isn't bridged, we need to indicate this.
+				assert(message.referenced_message)
+				const dateDifference = new Date(message.timestamp).getTime() - new Date(message.referenced_message.timestamp).getTime()
+				const oneHour = 60 * 60 * 1000
+				if (dateDifference < oneHour) {
+					var dateDisplay = "n"
+				} else if (dateDifference < 25 * oneHour) {
+					var dateDisplay = ` ${Math.floor(dateDifference / oneHour)}-hour-old`
+				} else {
+					var dateDisplay = ` ${Math.round(dateDifference / (24 * oneHour))}-day-old`
+				}
+				html = `<blockquote>In reply to a${dateDisplay} unbridged message from ${repliedToDisplayName}:`
+					+ `<br>${repliedToHtml}</blockquote>`
+					+ html
+				body = (`In reply to a${dateDisplay} unbridged message:\n${repliedToDisplayName}: `
+					+ repliedToBody).split("\n").map(line => "> " + line).join("\n")
+					+ "\n\n" + body
+			}
 		}
 
 		const newTextMessageEvent = {
@@ -460,7 +523,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 
 		const isPlaintext = body === html
 
-		if (!isPlaintext) {
+		if (!isPlaintext || options.alwaysReturnFormattedBody) {
 			Object.assign(newTextMessageEvent, {
 				format: "org.matrix.custom.html",
 				formatted_body: html
@@ -478,46 +541,117 @@ async function messageToEvent(message, guild, options = {}, di) {
 		message.content = "changed the channel name to **" + message.content + "**"
 	}
 
-	// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
-	const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
-	if (matches.length && matches.some(m => m[1].match(/[a-z]/i))) {
-		const writtenMentionsText = matches.map(m => m[1].toLowerCase())
-		const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
-		assert(roomID)
-		const {joined} = await di.api.getJoinedMembers(roomID)
-		for (const [mxid, member] of Object.entries(joined)) {
-			if (!userRegex.some(rx => mxid.match(rx))) {
-				const localpart = mxid.match(/@([^:]*)/)
-				assert(localpart)
-				const displayName = member.display_name || localpart[1]
-				if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+	// Forwarded content appears first
+	if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_snapshots?.length) {
+		// Forwarded notice
+		const eventID = select("event_message", "event_id", {message_id: message.message_reference.message_id}).pluck().get()
+		const room = select("channel_room", ["room_id", "name", "nick"], {channel_id: message.message_reference.channel_id}).get()
+		const forwardedNotice = new mxUtils.MatrixStringBuilder()
+		if (room) {
+			const roomName = room && (room.nick || room.name)
+			const via = await getViaServersMemo(room.room_id)
+			if (eventID) {
+				forwardedNotice.addLine(
+					`[🔀 Forwarded from #${roomName}]`,
+					tag`🔀 <em>Forwarded from <a href="https://matrix.to/#/${room.room_id}/${eventID}?${via}">${roomName}</a></em>`
+				)
+			} else {
+				forwardedNotice.addLine(
+					`[🔀 Forwarded from #${roomName}]`,
+					tag`🔀 <em>Forwarded from <a href="https://matrix.to/#/${room.room_id}?${via}">${roomName}</a></em>`
+				)
+			}
+		} else {
+			forwardedNotice.addLine(
+				`[🔀 Forwarded message]`,
+				tag`🔀 <em>Forwarded message</em>`
+			)
+		}
+
+		// Forwarded content
+		// @ts-ignore
+		const forwardedEvents = await messageToEvent(message.message_snapshots[0].message, guild, {includeReplyFallback: false, includeEditFallbackStar: false, alwaysReturnFormattedBody: true, scanTextForMentions: false}, di)
+
+		// Indent
+		for (const event of forwardedEvents) {
+			if (["m.text", "m.notice"].includes(event.msgtype)) {
+				event.msgtype = "m.notice"
+				event.body = event.body.split("\n").map(l => "» " + l).join("\n")
+				event.formatted_body = `<blockquote>${event.formatted_body}</blockquote>`
 			}
 		}
+
+		// Try to merge the forwarded content with the forwarded notice
+		let {body, formatted_body} = forwardedNotice.get()
+		if (forwardedEvents.length >= 1 && ["m.text", "m.notice"].includes(forwardedEvents[0].msgtype)) { // Try to merge the forwarded content and the forwarded notice
+			forwardedEvents[0].body = body + "\n" + forwardedEvents[0].body
+			forwardedEvents[0].formatted_body = formatted_body + "<br>" + forwardedEvents[0].formatted_body
+		} else {
+			await addTextEvent(body, formatted_body, "m.notice")
+		}
+		events.push(...forwardedEvents)
 	}
 
-	// Text content appears first
+	// Then text content
 	if (message.content) {
+		// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
+		const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
+		if (options.scanTextForMentions !== false && matches.length && matches.some(m => m[1].match(/[a-z]/i) && m[1] !== "everyone" && m[1] !== "here")) {
+			const writtenMentionsText = matches.map(m => m[1].toLowerCase())
+			const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
+			assert(roomID)
+			const {joined} = await di.api.getJoinedMembers(roomID)
+			for (const [mxid, member] of Object.entries(joined)) {
+				if (!userRegex.some(rx => mxid.match(rx))) {
+					const localpart = mxid.match(/@([^:]*)/)
+					assert(localpart)
+					const displayName = member.display_name || localpart[1]
+					if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+				}
+			}
+		}
+
 		const {body, html} = await transformContent(message.content)
-		await addTextEvent(body, html, msgtype, {scanMentions: true})
+		await addTextEvent(body, html, msgtype)
 	}
 
 	// Then attachments
-	const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions)))
-	events.push(...attachmentEvents)
+	if (message.attachments) {
+		const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions)))
+		events.push(...attachmentEvents)
+	}
 
 	// Then embeds
+	const urlPreviewEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1
 	for (const embed of message.embeds || []) {
+		if (!urlPreviewEnabled && !message.author?.bot) {
+			continue // show embeds for everyone if enabled, or bot users only if disabled (bots often send content in embeds)
+		}
+
 		if (embed.type === "image") {
 			continue // Matrix's own URL previews are fine for images.
 		}
 
+		if (embed.url?.startsWith("https://discord.com/")) {
+			continue // If discord creates an embed preview for a discord channel link, don't copy that embed
+		}
+
 		// Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once
 		const rep = new mxUtils.MatrixStringBuilder()
 
+		// Provider
+		if (embed.provider?.name) {
+			if (embed.provider.url) {
+				rep.addParagraph(`via ${embed.provider.name} ${embed.provider.url}`, tag`<sub><a href="${embed.provider.url}">${embed.provider.name}</a></sub>`)
+			} else {
+				rep.addParagraph(`via ${embed.provider.name}`, tag`<sub>${embed.provider.name}</sub>`)
+			}
+		}
+
 		// Author and URL into a paragraph
 		let authorNameText = embed.author?.name || ""
 		if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image
-		if (authorNameText || embed.author?.url) {
+		if (authorNameText) {
 			if (embed.author?.url) {
 				const authorURL = await transformContentMessageLinks(embed.author.url)
 				rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`<strong><a href="${authorURL}">${authorNameText}</a></strong>`)
@@ -534,11 +668,11 @@ async function messageToEvent(message, guild, options = {}, di) {
 			} else {
 				rep.addParagraph(`## ${body}`, `<strong>${html}</strong>`)
 			}
-		} else if (embed.url) {
-			rep.addParagraph(`## ${embed.url}`, tag`<strong><a href="${embed.url}">${embed.url}</a></strong>`)
 		}
 
-		if (embed.description) {
+		let embedTypeShouldShowDescription = embed.type !== "video" // Discord doesn't display descriptions for videos
+		if (embed.provider?.name === "YouTube") embedTypeShouldShowDescription = true // But I personally like showing the descriptions for YouTube videos specifically
+		if (embed.description && embedTypeShouldShowDescription) {
 			const {body, html} = await transformContent(embed.description)
 			rep.addParagraph(body, html)
 		}
@@ -552,8 +686,12 @@ async function messageToEvent(message, guild, options = {}, di) {
 			rep.addParagraph(fieldRep.get().body, fieldRep.get().formatted_body)
 		}
 
-		if (embed.image?.url) rep.addParagraph(`📸 ${embed.image.url}`)
-		if (embed.video?.url) rep.addParagraph(`🎞️ ${embed.video.url}`)
+		let chosenImage = embed.image?.url
+		// the thumbnail seems to be used for "article" type but displayed big at the bottom by discord
+		if (embed.type === "article" && embed.thumbnail?.url && !chosenImage) chosenImage = embed.thumbnail.url
+		if (chosenImage) rep.addParagraph(`📸 ${dUtils.getPublicUrlForCdn(chosenImage)}`)
+
+		if (embed.video?.url) rep.addParagraph(`🎞️ ${dUtils.getPublicUrlForCdn(embed.video.url)}`)
 
 		if (embed.footer?.text) rep.addLine(`— ${embed.footer.text}`, tag`— ${embed.footer.text}`)
 		let {body, formatted_body: html} = rep.get()
@@ -561,7 +699,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 		html = `<blockquote>${html}</blockquote>`
 
 		// Send as m.notice to apply the usual automated/subtle appearance, showing this wasn't actually typed by the person
-		await addTextEvent(body, html, "m.notice", {scanMentions: false})
+		await addTextEvent(body, html, "m.notice")
 	}
 
 	// Then stickers

src/d2m/converters/message-to-event.pk.test.js

@@ -1,6 +1,6 @@
 const {test} = require("supertape")
 const {messageToEvent} = require("./message-to-event")
-const data = require("../../test/data")
+const data = require("../../../test/data")
 const Ty = require("../../types")
 
 /**

src/d2m/converters/message-to-event.test.js

@@ -1,6 +1,7 @@
 const {test} = require("supertape")
 const {messageToEvent} = require("./message-to-event")
-const data = require("../../test/data")
+const {MatrixServerError} = require("../../matrix/mreq")
+const data = require("../../../test/data")
 const Ty = require("../../types")
 
 /**
@@ -64,6 +65,44 @@ test("message2event: simple user mention", async t => {
 test("message2event: simple room mention", async t => {
 	let called = 0
 	const events = await messageToEvent(data.message.simple_room_mention, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!BnKuBPCvyfOkhcUjEu:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!BnKuBPCvyfOkhcUjEu:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "#worm-farm",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe?via=cadence.moe&via=matrix.org">#worm-farm</a>'
+	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
+})
+
+test("message2event: nicked room mention", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message.nicked_room_mention, data.guild.general, {}, {
 		api: {
 			async getStateEvent(roomID, type, key) {
 				called++
@@ -229,6 +268,54 @@ test("message2event: message link that OOYE doesn't know about", async t => {
 	t.equal(called, 3, "getEventForTimestamp, getStateEvent, and getJoinedMembers should be called once each")
 })
 
+test("message2event: message timestamp failed to fetch", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message.message_link_to_before_ooye, data.guild.general, {}, {
+		api: {
+			async getEventForTimestamp(roomID, ts) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				throw new MatrixServerError({
+					errcode: "M_NOT_FOUND",
+					error: "Unable to find event from 1726762095974 in direction Direction.FORWARDS"
+				}, {})
+			},
+			async getStateEvent(roomID, type, key) { // for ?via calculation
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) { // for ?via calculation
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "Me: I'll scroll up to find a certain message I'll send\n_scrolls up and clicks message links for god knows how long_\n_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n_stumbles upon:_ "
+			+ "[unknown event, timestamp resolution failed, in room: https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org]",
+		format: "org.matrix.custom.html",
+		formatted_body: "Me: I'll scroll up to find a certain message I'll send<br><em>scrolls up and clicks message links for god knows how long</em><br><em>completely forgets what they were looking for and simply begins scrolling up to find some fun moments</em><br><em>stumbles upon:</em> "
+			+ '[unknown event, timestamp resolution failed, in room: <a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&amp;via=matrix.org">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&amp;via=matrix.org</a>]'
+	}])
+	t.equal(called, 3, "getEventForTimestamp, getStateEvent, and getJoinedMembers should be called once each")
+})
+
 test("message2event: message link from another server", async t => {
 	const events = await messageToEvent(data.message.message_link_from_another_server, data.guild.general)
 	t.deepEqual(events, [{
@@ -250,7 +337,7 @@ test("message2event: attachment with no content", async t => {
 		msgtype: "m.image",
 		url: "mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM",
 		body: "image.png",
-		external_url: "https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png",
+		external_url: "https://bridge.example.org/download/discordcdn/497161332244742154/1124628646431297546/image.png",
 		filename: "image.png",
 		info: {
 			mimetype: "image/png",
@@ -267,9 +354,9 @@ test("message2event: spoiler attachment", async t => {
 		$type: "m.room.message",
 		"m.mentions": {},
 		msgtype: "m.text",
-		body: "📄 Uploaded SPOILER file: https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci (74 KB)",
+		body: "📄 Uploaded SPOILER file: https://bridge.example.org/download/discordcdn/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci (74 KB)",
 		format: "org.matrix.custom.html",
-		formatted_body: "<blockquote>📄 Uploaded SPOILER file: <a href=\"https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci\">https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci</a> (74 KB)</blockquote>"
+		formatted_body: "<blockquote>📄 Uploaded SPOILER file: <a href=\"https://bridge.example.org/download/discordcdn/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci\">https://bridge.example.org/download/discordcdn/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci</a> (74 KB)</blockquote>"
 	}])
 })
 
@@ -286,7 +373,7 @@ test("message2event: stickers", async t => {
 		msgtype: "m.image",
 		url: "mxc://cadence.moe/ZDCNYnkPszxGKgObUIFmvjus",
 		body: "image.png",
-		external_url: "https://cdn.discordapp.com/attachments/122155380120748034/1106366167486038016/image.png",
+		external_url: "https://bridge.example.org/download/discordcdn/122155380120748034/1106366167486038016/image.png",
 		filename: "image.png",
 		info: {
 			mimetype: "image/png",
@@ -340,7 +427,7 @@ test("message2event: skull webp attachment with content", async t => {
 			mimetype: "image/webp",
 			size: 74290
 		},
-		external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1128084747910918195/skull.webp",
+		external_url: "https://bridge.example.org/download/discordcdn/112760669178241024/1128084747910918195/skull.webp",
 		filename: "skull.webp",
 		url: "mxc://cadence.moe/sDxWmDErBhYBxtDcJQgBETes"
 	}])
@@ -374,7 +461,7 @@ test("message2event: reply to skull webp attachment with content", async t => {
 			mimetype: "image/jpeg",
 			size: 85906
 		},
-		external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1128084851023675515/RDT_20230704_0936184915846675925224905.jpg",
+		external_url: "https://bridge.example.org/download/discordcdn/112760669178241024/1128084851023675515/RDT_20230704_0936184915846675925224905.jpg",
 		filename: "RDT_20230704_0936184915846675925224905.jpg",
 		url: "mxc://cadence.moe/WlAbFSiNRIHPDEwKdyPeGywa"
 	}])
@@ -464,7 +551,7 @@ test("message2event: reply with a video", async t => {
 		body: "Ins_1960637570.mp4",
 		filename: "Ins_1960637570.mp4",
 		url: "mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU",
-		external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4?ex=65bbee8f&is=65a9798f&hm=ae14f7824c3d526c5e11c162e012e1ee405fd5776e1e9302ed80ccd86503cfda&",
+		external_url: "https://bridge.example.org/download/discordcdn/112760669178241024/1197621094786531358/Ins_1960637570.mp4",
 		info: {
 			h: 854,
 			mimetype: "video/mp4",
@@ -485,10 +572,10 @@ test("message2event: voice message", async t => {
 	t.deepEqual(events, [{
 		$type: "m.room.message",
       body: "voice-message.ogg",
-      external_url: "https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg?ex=65c92d4c&is=65b6b84c&hm=0654bab5027474cbe23875954fa117cf44d8914c144cd151879590fa1baf8b1c&",
+      external_url: "https://bridge.example.org/download/discordcdn/1099031887500034088/1112476845502365786/voice-message.ogg",
       filename: "voice-message.ogg",
       info: {
-        duration: 3960.0000381469727,
+        duration: 3960,
         mimetype: "audio/ogg",
         size: 10584,
 		},
@@ -508,7 +595,7 @@ test("message2event: misc file", async t => {
 	}, {
 		$type: "m.room.message",
       body: "the.yml",
-      external_url: "https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml?ex=65cd6270&is=65baed70&hm=8c5f1b571784e3c7f99628492298815884e351ae0dc7c2ae40dd22d97caf27d9&",
+      external_url: "https://bridge.example.org/download/discordcdn/122155380120748034/1174514575220158545/the.yml",
       filename: "the.yml",
 		info: {
 			mimetype: "text/plain; charset=utf-8",
@@ -559,6 +646,98 @@ test("message2event: simple reply in thread to a matrix user's reply", async t =
 	}])
 })
 
+test("message2event: infinidoge's reply to ami's matrix smalltext reply to infinidoge", async t => {
+	const events = await messageToEvent(data.message.infinidoge_reply_to_ami_matrix_smalltext_reply_to_infinidoge, data.guild.general, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!BnKuBPCvyfOkhcUjEu:cadence.moe", "$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4", {
+				type: "m.room.message",
+				sender: "@ami:the-apothecary.club",
+				content: {
+					msgtype: "m.text",
+					body: `> <@_ooye_infinidoge:cadence.moe> Neat that they thought of that\n\nlet me guess they got a lot of bug reports like "empty chest with no loot?"`,
+					format: "org.matrix.custom.html",
+					formatted_body: `<mx-reply><blockquote><a href="https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$baby?via=cadence.moe">In reply to</a> <a href="https://matrix.to/#/@_ooye_infinidoge:cadence.moe">@_ooye_infinidoge:cadence.moe</a><br>Neat that they thought of that</blockquote></mx-reply>let me guess they got a lot of bug reports like "empty chest with no loot?"`,
+					"m.relates_to": {
+						"m.in_reply_to": {
+							event_id: "$baby"
+						}
+					}
+				},
+				event_id: "$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4",
+				room_id: "!BnKuBPCvyfOkhcUjEu:cadence.moe"
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4"
+			}
+		},
+		"m.mentions": {
+			user_ids: ["@ami:the-apothecary.club"]
+		},
+		msgtype: "m.text",
+      body: `> Ami (she/her): let me guess they got a lot of bug reports like "empty chest with no loot?"\n\nMost likely`,
+      format: "org.matrix.custom.html",
+      formatted_body: `<mx-reply><blockquote><a href="https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4">In reply to</a> <a href="https://matrix.to/#/@ami:the-apothecary.club">Ami (she/her)</a><br>let me guess they got a lot of bug reports like "empty chest with no loot?"</blockquote></mx-reply>Most likely`,
+	}])
+})
+
+test("message2event: infinidoge's reply to ami's matrix smalltext singleline reply to infinidoge", async t => {
+	const events = await messageToEvent(data.message.infinidoge_reply_to_ami_matrix_smalltext_singleline_reply_to_infinidoge, data.guild.general, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!BnKuBPCvyfOkhcUjEu:cadence.moe", "$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4", {
+				type: "m.room.message",
+				sender: "@ami:the-apothecary.club",
+				content: {
+					msgtype: "m.text",
+					body: `> <@_ooye_infinidoge:cadence.moe> Neat that they thought of that\n\nlet me guess they got a lot of bug reports like "empty chest with no loot?"`,
+					format: "org.matrix.custom.html",
+					formatted_body: `<mx-reply><blockquote><a href="https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$baby?via=cadence.moe">In reply to</a> <a href="https://matrix.to/#/@_ooye_infinidoge:cadence.moe">@_ooye_infinidoge:cadence.moe</a><br>Neat that they thought of that</blockquote></mx-reply>let me guess they got a lot of bug reports like "empty chest with no loot?"`,
+					"m.relates_to": {
+						"m.in_reply_to": {
+							event_id: "$baby"
+						}
+					}
+				},
+				event_id: "$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4",
+				room_id: "!BnKuBPCvyfOkhcUjEu:cadence.moe"
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4"
+			}
+		},
+		"m.mentions": {
+			user_ids: ["@ami:the-apothecary.club"]
+		},
+		msgtype: "m.text",
+      body: `> Ami (she/her): let me guess they got a lot of bug reports like "empty chest with no loot?"\n\nMost likely`,
+      format: "org.matrix.custom.html",
+      formatted_body: `<mx-reply><blockquote><a href="https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe/$W1nsDhNIojWrcQOdnOD9RaEvrz2qyZErQoNhPRs1nK4">In reply to</a> <a href="https://matrix.to/#/@ami:the-apothecary.club">Ami (she/her)</a><br>let me guess they got a lot of bug reports like "empty chest with no loot?"</blockquote></mx-reply>Most likely`,
+	}])
+})
+
+test("message2event: reply to a Discord message that wasn't bridged", async t => {
+	const events = await messageToEvent(data.message.reply_to_unknown_message, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+      body: `> In reply to a 1-day-old unbridged message:`
+			+ `\n> Occimyy: BILLY BOB THE GREAT`
+			+ `\n\nenigmatic`,
+      format: "org.matrix.custom.html",
+      formatted_body: `<blockquote>In reply to a 1-day-old unbridged message from Occimyy:<br>BILLY BOB THE GREAT</blockquote>enigmatic`,
+		"m.mentions": {}
+	}])
+})
+
 test("message2event: simple written @mention for matrix user", async t => {
 	const events = await messageToEvent(data.message.simple_written_at_mention_for_matrix, data.guild.general, {}, {
 		api: {
@@ -672,7 +851,7 @@ test("message2event: very large attachment is linked instead of being uploaded",
 		content: "hey",
 		attachments: [{
 			filename: "hey.jpg",
-			url: "https://discord.com/404/hey.jpg",
+			url: "https://cdn.discordapp.com/attachments/123/456/789.mega",
 			content_type: "application/i-made-it-up",
 			size: 100e6
 		}]
@@ -686,9 +865,9 @@ test("message2event: very large attachment is linked instead of being uploaded",
 		$type: "m.room.message",
 		"m.mentions": {},
 		msgtype: "m.text",
-		body: "📄 Uploaded file: https://discord.com/404/hey.jpg (100 MB)",
+		body: "📄 Uploaded file: https://bridge.example.org/download/discordcdn/123/456/789.mega (100 MB)",
 		format: "org.matrix.custom.html",
-		formatted_body: '📄 Uploaded file: <a href="https://discord.com/404/hey.jpg">hey.jpg</a> (100 MB)'
+		formatted_body: '📄 Uploaded file: <a href="https://bridge.example.org/download/discordcdn/123/456/789.mega">hey.jpg</a> (100 MB)'
 	}])
 })
 
@@ -789,3 +968,200 @@ test("message2event: crossposted announcements say where they are crossposted fr
 		formatted_body: "🔀 <strong>Chewey Bot Official Server #announcements</strong><br>All text based commands are now inactive on Chewey Bot<br>To continue using commands you'll need to use them as slash commands"
 	}])
 })
+
+test("message2event: @everyone", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@room",
+		"m.mentions": {
+			room: true
+		}
+	}])
+})
+
+test("message2event: @here", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_here)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@room",
+		"m.mentions": {
+			room: true
+		}
+	}])
+})
+
+test("message2event: @everyone without permission", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone_without_permission)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@everyone <-- this is testing that it DOESN'T mention. if this mentions everyone then my apologies.",
+		format: "org.matrix.custom.html",
+		formatted_body: "@everyone &lt;-- this is testing that it DOESN'T mention. if this mentions everyone then my apologies.",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: @here without permission", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_here_without_permission)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@here <-- this is testing that it DOESN'T mention. if this mentions people then my apologies.",
+		format: "org.matrix.custom.html",
+		formatted_body: "@here &lt;-- this is testing that it DOESN'T mention. if this mentions people then my apologies.",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: @everyone within a link", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone_within_link)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://github.com/@everyone",
+		format: "org.matrix.custom.html",
+		formatted_body: `<a href="https://github.com/@everyone">https://github.com/@everyone</a>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: forwarded image", async t => {
+	const events = await messageToEvent(data.message.forwarded_image)
+	t.deepEqual(events, [
+		{
+			$type: "m.room.message",
+			body: "[🔀 Forwarded message]",
+			format: "org.matrix.custom.html",
+			formatted_body: "🔀 <em>Forwarded message</em>",
+			"m.mentions": {},
+			msgtype: "m.notice",
+		},
+		{
+			$type: "m.room.message",
+			body: "100km.gif",
+			external_url: "https://bridge.example.org/download/discordcdn/112760669178241024/1296237494987133070/100km.gif",
+			filename: "100km.gif",
+			info: {
+				h: 300,
+				mimetype: "image/gif",
+				size: 2965649,
+				w: 300,
+			},
+			"m.mentions": {},
+			msgtype: "m.image",
+			url: "mxc://cadence.moe/qDAotmebTfEIfsAIVCEZptLh",
+		},
+	])
+})
+
+test("message2event: constructed forwarded message", async t => {
+	const events = await messageToEvent(data.message.constructed_forwarded_message, {}, {}, {
+		api: {
+			async getJoinedMembers() {
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [
+		{
+			$type: "m.room.message",
+			body: "[🔀 Forwarded from #wonderland]"
+				+ "\n» What's cooking, good looking? :hipposcope:",
+			format: "org.matrix.custom.html",
+			formatted_body: `🔀 <em>Forwarded from <a href="https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$tBIT8mO7XTTCgIINyiAIy6M2MSoPAdJenRl_RLyYuaE?via=cadence.moe&amp;via=matrix.org">wonderland</a></em>`
+				+ `<br><blockquote>What's cooking, good looking? <img data-mx-emoticon height="32" src="mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc" title=":hipposcope:" alt=":hipposcope:"></blockquote>`,
+			"m.mentions": {},
+			msgtype: "m.notice",
+		},
+		{
+			$type: "m.room.message",
+			body: "100km.gif",
+			external_url: "https://bridge.example.org/download/discordcdn/112760669178241024/1296237494987133070/100km.gif",
+			filename: "100km.gif",
+			info: {
+				h: 300,
+				mimetype: "image/gif",
+				size: 2965649,
+				w: 300,
+			},
+			"m.mentions": {},
+			msgtype: "m.image",
+			url: "mxc://cadence.moe/qDAotmebTfEIfsAIVCEZptLh",
+		},
+		{
+			$type: "m.room.message",
+			body: "» | ## This man"
+				+ "\n» | "
+				+ "\n» | ## This man is 100 km away from your house"
+				+ "\n» | "
+				+ "\n» | ### Distance away"
+				+ "\n» | 99 km"
+				+ "\n» | "
+				+ "\n» | ### Distance away"
+				+ "\n» | 98 km",
+			format: "org.matrix.custom.html",
+			formatted_body: "<blockquote><blockquote><p><strong>This man</strong></p><p><strong>This man is 100 km away from your house</strong></p><p><strong>Distance away</strong><br>99 km</p><p><strong>Distance away</strong><br>98 km</p></blockquote></blockquote>",
+			"m.mentions": {},
+			msgtype: "m.notice"
+		}
+	])
+})
+
+test("message2event: constructed forwarded text", async t => {
+	const events = await messageToEvent(data.message.constructed_forwarded_text, {}, {}, {
+		api: {
+			async getJoinedMembers() {
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [
+		{
+			$type: "m.room.message",
+			body: "[🔀 Forwarded from #amanda-spam]"
+				+ "\n» What's cooking, good looking?",
+			format: "org.matrix.custom.html",
+			formatted_body: `🔀 <em>Forwarded from <a href="https://matrix.to/#/!CzvdIdUQXgUjDVKxeU:cadence.moe?via=cadence.moe&amp;via=matrix.org">amanda-spam</a></em>`
+				+ `<br><blockquote>What's cooking, good looking?</blockquote>`,
+			"m.mentions": {},
+			msgtype: "m.notice",
+		},
+		{
+			$type: "m.room.message",
+			body: "What's cooking everybody ‼️",
+			"m.mentions": {},
+			msgtype: "m.text",
+		}
+	])
+})
+
+
+test("message2event: don't scan forwarded messages for mentions", async t => {
+	const events = await messageToEvent(data.message.forwarded_dont_scan_for_mentions, {}, {}, {})
+	t.deepEqual(events, [
+		{
+			$type: "m.room.message",
+			body: "[🔀 Forwarded message]"
+				+ "\n» If some folks have spare bandwidth then helping out ArchiveTeam with archiving soon to be deleted research and government data might be worthwhile https://social.luca.run/@luca/113950834185678114",
+			format: "org.matrix.custom.html",
+			formatted_body: `🔀 <em>Forwarded message</em>`
+				+ `<br><blockquote>If some folks have spare bandwidth then helping out ArchiveTeam with archiving soon to be deleted research and government data might be worthwhile <a href="https://social.luca.run/@luca/113950834185678114">https://social.luca.run/@luca/113950834185678114</a></blockquote>`,
+			"m.mentions": {},
+			msgtype: "m.notice"
+		}
+	])
+})

src/d2m/converters/pins-to-list.test.js

@@ -1,5 +1,5 @@
 const {test} = require("supertape")
-const data = require("../../test/data")
+const data = require("../../../test/data")
 const {pinsToList} = require("./pins-to-list")
 
 test("pins2list: converts known IDs, ignores unknown IDs", t => {

src/d2m/converters/remove-reaction.js

@@ -12,7 +12,7 @@ const utils = sync.require("../../m2d/converters/utils")
  * @typedef ReactionRemoveRequest
  * @prop {string} eventID
  * @prop {string | null} mxid
- * @prop {BigInt} [hash]
+ * @prop {bigint} [hash]
  */
 
 /**

src/d2m/converters/remove-reaction.test.js

@@ -10,7 +10,9 @@ function fakeSpecificReactionRemoval(userID, emoji, emojiID) {
 		channel_id: "THE_CHANNEL",
 		message_id: "THE_MESSAGE",
 		user_id: userID,
-		emoji: {id: emojiID, name: emoji}
+		emoji: {id: emojiID, name: emoji},
+		burst: false,
+		type: 0
 	}
 }
 

src/d2m/converters/thread-to-announcement.js

@@ -4,10 +4,9 @@ const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
-/** @type {import("../../matrix/read-registration")} */
-const reg = sync.require("../../matrix/read-registration.js")
 /** @type {import("../../m2d/converters/utils")} */
 const mxUtils = sync.require("../../m2d/converters/utils")
+const {reg} = require("../../matrix/read-registration.js")
 
 const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
 

src/d2m/converters/thread-to-announcement.test.js

@@ -1,6 +1,6 @@
 const {test} = require("supertape")
 const {threadToAnnouncement} = require("./thread-to-announcement")
-const data = require("../../test/data")
+const data = require("../../../test/data")
 const Ty = require("../../types")
 
 /**

src/d2m/converters/user-to-mxid.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 const assert = require("assert")
+const {reg} = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
 const {select} = passthrough
@@ -24,6 +25,10 @@ function downcaseUsername(user) {
 		.replace(/[^a-z0-9._=/-]*/g, "")
 		// remove leading and trailing dashes and underscores...
 		.replace(/(?:^[_-]*|[_-]*$)/g, "")
+	// If requested, also make the Discord user ID part of the username
+	if (reg.ooye.include_user_id_in_mxid) {
+		downcased = user.id + "_" + downcased
+	}
 	// The new length must be at least 2 characters (in other words, it should have some content)
 	if (downcased.length < 2) {
 		downcased = user.id
@@ -59,7 +64,7 @@ function userToSimName(user) {
 	}
 
 	// 1. Is sim user already registered?
-	const existing = select("sim", "sim_name", {user_id: user.id}).pluck().get()
+	const existing = select("sim", "user_id", {user_id: user.id}).pluck().get()
 	assert.equal(existing, null, "Shouldn't try to create a new name for an existing sim")
 
 	// 2. Register based on username (could be new or old format)

src/d2m/converters/user-to-mxid.test.js

@@ -1,7 +1,7 @@
 const {test} = require("supertape")
 const tryToCatch = require("try-to-catch")
 const assert = require("assert")
-const data = require("../../test/data")
+const data = require("../../../test/data")
 const {userToSimName} = require("./user-to-mxid")
 
 test("user2name: cannot create user for a webhook", async t => {
@@ -44,3 +44,11 @@ test("user2name: uses ID when name has only disallowed characters", t => {
 test("user2name: works on special user", t => {
 	t.equal(userToSimName(data.user.clyde_ai), "clyde_ai")
 })
+
+test("user2name: includes ID if requested in config", t => {
+	const {reg} = require("../../matrix/read-registration")
+	reg.ooye.include_user_id_in_mxid = true
+	t.equal(userToSimName({username: "Harry Styles!", discriminator: "0001", id: "123456"}), "123456_harry_styles")
+   t.equal(userToSimName({username: "f***", discriminator: "0001", id: "123456"}), "123456_f")
+	reg.ooye.include_user_id_in_mxid = false
+})

src/d2m/discord-client.js

@@ -1,12 +1,17 @@
 // @ts-check
 
-const { SnowTransfer } = require("snowtransfer")
-const { Client: CloudStorm } = require("cloudstorm")
+const DiscordTypes = require("discord-api-types/v10")
+const {Endpoints, SnowTransfer} = require("snowtransfer")
+const {reg} = require("../matrix/read-registration")
+const {Client: CloudStorm} = require("cloudstorm")
+
+// @ts-ignore
+Endpoints.BASE_HOST = reg.ooye.discord_origin || "https://discord.com"; Endpoints.CDN_URL = reg.ooye.discord_cdn_origin || "https://cdn.discordapp.com"
 
 const passthrough = require("../passthrough")
-const { sync } = passthrough
+const {sync} = passthrough
 
-/** @type {typeof import("./discord-packets")} */
+/** @type {import("./discord-packets")} */
 const discordPackets = sync.require("./discord-packets")
 
 class DiscordClient {
@@ -24,7 +29,7 @@ class DiscordClient {
 			intents: [
 				"DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING",
 				"GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS",
-				"MESSAGE_CONTENT"
+				"MESSAGE_CONTENT", "GUILD_PRESENCES"
 			],
 			ws: {
 				compress: false,
@@ -32,15 +37,15 @@ class DiscordClient {
 			}
 		})
 		this.ready = false
-		/** @type {import("discord-api-types/v10").APIUser} */
+		/** @type {DiscordTypes.APIUser} */
 		// @ts-ignore avoid setting as or null because we know we need to wait for ready anyways
 		this.user = null
-		/** @type {Pick<import("discord-api-types/v10").APIApplication, "id" | "flags">} */
+		/** @type {Pick<DiscordTypes.APIApplication, "id" | "flags">} */
 		// @ts-ignore
 		this.application = null
-		/** @type {Map<string, import("discord-api-types/v10").APIChannel>} */
+		/** @type {Map<string, DiscordTypes.APIChannel>} */
 		this.channels = new Map()
-		/** @type {Map<string, import("discord-api-types/v10").APIGuild>} */
+		/** @type {Map<string, DiscordTypes.APIGuild & {members: DiscordTypes.APIGuildMember[]}>} */ // we get members from the GUILD_CREATE and we do maintain it
 		this.guilds = new Map()
 		/** @type {Map<string, Array<string>>} */
 		this.guildChannelMap = new Map()
@@ -57,6 +62,9 @@ class DiscordClient {
 		addEventLogger("error", "Error")
 		addEventLogger("disconnected", "Disconnected")
 		addEventLogger("ready", "Ready")
+		this.snow.requestHandler.on("requestError", (requestID, error) => {
+			console.error("request error:", error)
+		})
 	}
 }
 

src/d2m/discord-packets.js

@@ -4,7 +4,11 @@
 
 const DiscordTypes = require("discord-api-types/v10")
 const passthrough = require("../passthrough")
-const { sync } = passthrough
+const {sync, db} = passthrough
+
+function populateGuildID(guildID, channelID) {
+	db.prepare("UPDATE channel_room SET guild_id = ? WHERE channel_id = ?").run(guildID, channelID)
+}
 
 const utils = {
 	/**
@@ -16,6 +20,8 @@ const utils = {
 		// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
 		/** @type {typeof import("./event-dispatcher")} */
 		const eventDispatcher = sync.require("./event-dispatcher")
+		/** @type {import("../discord/register-interactions")} */
+		const interactions = sync.require("../discord/register-interactions")
 
 		// Client internals, keep track of the state we need
 		if (message.t === "READY") {
@@ -26,6 +32,7 @@ const utils = {
 			console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`)
 
 		} else if (message.t === "GUILD_CREATE") {
+			message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web
 			client.guilds.set(message.d.id, message.d)
 			const arr = []
 			client.guildChannelMap.set(message.d.id, arr)
@@ -34,17 +41,25 @@ const utils = {
 				channel.guild_id = message.d.id
 				arr.push(channel.id)
 				client.channels.set(channel.id, channel)
+				populateGuildID(message.d.id, channel.id)
 			}
 			for (const thread of message.d.threads || []) {
 				// @ts-ignore
 				thread.guild_id = message.d.id
 				arr.push(thread.id)
 				client.channels.set(thread.id, thread)
+				populateGuildID(message.d.id, thread.id)
 			}
+
 			if (listen === "full") {
-				eventDispatcher.checkMissedExpressions(message.d)
-				eventDispatcher.checkMissedPins(client, message.d)
-				eventDispatcher.checkMissedMessages(client, message.d)
+				try {
+					await eventDispatcher.checkMissedExpressions(message.d)
+					await eventDispatcher.checkMissedPins(client, message.d)
+					await eventDispatcher.checkMissedMessages(client, message.d)
+				} catch (e) {
+					console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:")
+					console.error(e)
+				}
 			}
 
 		} else if (message.t === "GUILD_UPDATE") {
@@ -87,9 +102,20 @@ const utils = {
 				}
 			}
 
+		} else if (message.t === "GUILD_MEMBER_UPDATE") {
+			const guild = client.guilds.get(message.d.guild_id)
+			const member = guild?.members.find(m => m.user.id === message.d.user.id)
+			if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones
+				Object.assign(member, message.d)
+			}
+
 		} else if (message.t === "THREAD_CREATE") {
 			client.channels.set(message.d.id, message.d)
-
+			if (message.d["guild_id"]) {
+				populateGuildID(message.d["guild_id"], message.d.id)
+				const channels = client.guildChannelMap.get(message.d["guild_id"])
+				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
+			}
 
 		} else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") {
 			client.channels.set(message.d.id, message.d)
@@ -111,71 +137,40 @@ const utils = {
 			client.guildChannelMap.delete(message.d.id)
 
 
-		} else if (message.t === "CHANNEL_CREATE" || message.t === "CHANNEL_DELETE") {
-			if (message.t === "CHANNEL_CREATE") {
-				client.channels.set(message.d.id, message.d)
-				if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
-					const channels = client.guildChannelMap.get(message.d["guild_id"])
-					if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
-				}
-			} else {
-				client.channels.delete(message.d.id)
-				if (message.d["guild_id"]) {
-					const channels = client.guildChannelMap.get(message.d["guild_id"])
-					if (channels) {
-						const previous = channels.indexOf(message.d.id)
-						if (previous !== -1) channels.splice(previous, 1)
-					}
+		} else if (message.t === "CHANNEL_CREATE") {
+			client.channels.set(message.d.id, message.d)
+			if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
+				populateGuildID(message.d["guild_id"], message.d.id)
+				const channels = client.guildChannelMap.get(message.d["guild_id"])
+				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
+			}
+
+		} else if (message.t === "CHANNEL_DELETE") {
+			client.channels.delete(message.d.id)
+			if (message.d["guild_id"]) {
+				const channels = client.guildChannelMap.get(message.d["guild_id"])
+				if (channels) {
+					const previous = channels.indexOf(message.d.id)
+					if (previous !== -1) channels.splice(previous, 1)
 				}
 			}
 		}
 
 		// Event dispatcher for OOYE bridge operations
-		if (listen === "full") {
+		if (listen === "full" && message.t) {
 			try {
-				if (message.t === "GUILD_UPDATE") {
-					await eventDispatcher.onGuildUpdate(client, message.d)
-
-				} else if (message.t === "GUILD_EMOJIS_UPDATE" || message.t === "GUILD_STICKERS_UPDATE") {
-					await eventDispatcher.onExpressionsUpdate(client, message.d)
-
-				} else if (message.t === "CHANNEL_UPDATE") {
-					await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false)
-
-				} else if (message.t === "CHANNEL_PINS_UPDATE") {
-					await eventDispatcher.onChannelPinsUpdate(client, message.d)
-
-				} else if (message.t === "THREAD_CREATE") {
-					// @ts-ignore
-					await eventDispatcher.onThreadCreate(client, message.d)
-
-				} else if (message.t === "THREAD_UPDATE") {
-					await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true)
-
-				} else if (message.t === "MESSAGE_CREATE") {
-					await eventDispatcher.onMessageCreate(client, message.d)
-
-				} else if (message.t === "MESSAGE_UPDATE") {
-					await eventDispatcher.onMessageUpdate(client, message.d)
-
-				} else if (message.t === "MESSAGE_DELETE") {
-					await eventDispatcher.onMessageDelete(client, message.d)
-
-				} else if (message.t === "MESSAGE_DELETE_BULK") {
-					await eventDispatcher.onMessageDeleteBulk(client, message.d)
-
-				} else if (message.t === "TYPING_START") {
-					await eventDispatcher.onTypingStart(client, message.d)
-
-				} else if (message.t === "MESSAGE_REACTION_ADD") {
-					await eventDispatcher.onReactionAdd(client, message.d)
-
-				} else if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
+				if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
 					await eventDispatcher.onSomeReactionsRemoved(client, message.d)
+
+				} else if (message.t === "INTERACTION_CREATE") {
+					await interactions.dispatchInteraction(message.d)
+
+				} else if (message.t in eventDispatcher) {
+					await eventDispatcher[message.t](client, message.d)
 				}
 			} catch (e) {
 				// Let OOYE try to handle errors too
-				eventDispatcher.onError(client, e, message)
+				await eventDispatcher.onError(client, e, message)
 			}
 		}
 	}

src/d2m/event-dispatcher.js

@@ -2,7 +2,6 @@
 
 const assert = require("assert").strict
 const DiscordTypes = require("discord-api-types/v10")
-const util = require("util")
 const {sync, db, select, from} = require("../passthrough")
 
 /** @type {import("./actions/send-message")}) */
@@ -27,19 +26,19 @@ const updatePins = sync.require("./actions/update-pins")
 const api = sync.require("../matrix/api")
 /** @type {import("../discord/utils")} */
 const dUtils = sync.require("../discord/utils")
-/** @type {import("../discord/discord-command-handler")}) */
-const discordCommandHandler = sync.require("../discord/discord-command-handler")
-/** @type {import("../m2d/converters/utils")} */
-const mxUtils = require("../m2d/converters/utils")
 /** @type {import("./actions/speedbump")} */
 const speedbump = sync.require("./actions/speedbump")
+/** @type {import("./actions/retrigger")} */
+const retrigger = sync.require("./actions/retrigger")
+/** @type {import("./actions/set-presence")} */
+const setPresence = sync.require("./actions/set-presence")
+/** @type {import("../m2d/event-dispatcher")} */
+const matrixEventDispatcher = sync.require("../m2d/event-dispatcher")
 
 /** @type {any} */ // @ts-ignore bad types from semaphore
 const Semaphore = require("@chriscdn/promise-semaphore")
 const checkMissedPinsSema = new Semaphore()
 
-let lastReportedEvent = 0
-
 // Grab Discord events we care about for the bridge, check them, and pass them on
 
 module.exports = {
@@ -48,49 +47,15 @@ module.exports = {
 	 * @param {Error} e
 	 * @param {import("cloudstorm").IGatewayMessage} gatewayMessage
 	 */
-	onError(client, e, gatewayMessage) {
-		console.error("hit event-dispatcher's error handler with this exception:")
-		console.error(e) // TODO: also log errors into a file or into the database, maybe use a library for this? or just wing it? definitely need to be able to store the formatted event body to load back in later
-		console.error(`while handling this ${gatewayMessage.t} gateway event:`)
-		console.dir(gatewayMessage.d, {depth: null})
-
-		if (gatewayMessage.t === "TYPING_START") return
-
-		if (Date.now() - lastReportedEvent < 5000) return
-		lastReportedEvent = Date.now()
-
+	async onError(client, e, gatewayMessage) {
 		const channelID = gatewayMessage.d["channel_id"]
 		if (!channelID) return
 		const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
 		if (!roomID) return
 
-		let stackLines = null
-		if (e.stack) {
-			stackLines = e.stack.split("\n")
-			let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/"))
-			if (cloudstormLine !== -1) {
-				stackLines = stackLines.slice(0, cloudstormLine - 2)
-			}
-		}
+		if (gatewayMessage.t === "TYPING_START") return
 
-		const builder = new mxUtils.MatrixStringBuilder()
-		builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 <strong>Bridged event from Discord not delivered</strong>")
-		builder.addLine(`Gateway event: ${gatewayMessage.t}`)
-		builder.addLine(e.toString())
-		if (stackLines) {
-			builder.addLine(`Error trace:\n${stackLines.join("\n")}`, `<details><summary>Error trace</summary><pre>${stackLines.join("\n")}</pre></details>`)
-		}
-		builder.addLine("", `<details><summary>Original payload</summary><pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`)
-		api.sendEvent(roomID, "m.room.message", {
-			...builder.get(),
-			"moe.cadence.ooye.error": {
-				source: "discord",
-				payload: gatewayMessage
-			},
-			"m.mentions": {
-				user_ids: ["@cadence:cadence.moe"]
-			}
-		})
+		await matrixEventDispatcher.sendError(roomID, "Discord", gatewayMessage.t, e, gatewayMessage.d)
 	},
 
 	/**
@@ -103,20 +68,28 @@ module.exports = {
 	async checkMissedMessages(client, guild) {
 		if (guild.unavailable) return
 		const bridgedChannels = select("channel_room", "channel_id").pluck().all()
-		const prepared = select("event_message", "event_id", {}, "WHERE message_id = ?").pluck()
-		for (const channel of guild.channels.concat(guild.threads)) {
+		const preparedExists = db.prepare("SELECT channel_id FROM message_channel WHERE channel_id = ? LIMIT 1")
+		const preparedGet = select("event_message", "event_id", {}, "WHERE message_id = ?").pluck()
+		/** @type {(DiscordTypes.APIChannel & {type: DiscordTypes.GuildChannelType})[]} */
+		let channels = []
+		channels = channels.concat(guild.channels, guild.threads)
+		for (const channel of channels) {
 			if (!bridgedChannels.includes(channel.id)) continue
 			if (!("last_message_id" in channel) || !channel.last_message_id) continue
-			const latestWasBridged = prepared.get(channel.last_message_id)
+
+			// Skip if channel is already up-to-date
+			const latestWasBridged = preparedGet.get(channel.last_message_id)
 			if (latestWasBridged) continue
 
+			// Skip if channel was just added to the bridge (there's no place to resume from if it's brand new)
+			if (!preparedExists.get(channel.id)) continue
+
 			// Permissions check
 			const member = guild.members.find(m => m.user?.id === client.user.id)
 			if (!member) return
 			if (!("permission_overwrites" in channel)) continue
 			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
-			if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel
+			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look back in this channel
 
 			/** More recent messages come first. */
 			// console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
@@ -132,7 +105,7 @@ module.exports = {
 				}
 			}
 			let latestBridgedMessageIndex = messages.findIndex(m => {
-				return prepared.get(m.id)
+				return preparedGet.get(m.id)
 			})
 			// console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
 			if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date.
@@ -142,7 +115,7 @@ module.exports = {
 					backfill: true,
 					...messages[i]
 				}
-				await module.exports.onMessageCreate(client, simulatedGatewayDispatchData)
+				await module.exports.MESSAGE_CREATE(client, simulatedGatewayDispatchData)
 			}
 		}
 	},
@@ -164,8 +137,7 @@ module.exports = {
 
 			// Permissions check
 			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
-			if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel
+			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look up the pins in this channel
 
 			const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get()
 			if (!row) continue // Only care about already bridged channels
@@ -181,7 +153,7 @@ module.exports = {
 	 */
 	async checkMissedExpressions(guild) {
 		const data = {guild_id: guild.id, ...guild}
-		createSpace.syncSpaceExpressions(data, true)
+		await createSpace.syncSpaceExpressions(data, true)
 	},
 
 	/**
@@ -190,10 +162,10 @@ module.exports = {
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.APIThreadChannel} thread
 	 */
-	async onThreadCreate(client, thread) {
+	async THREAD_CREATE(client, thread) {
 		const channelID = thread.parent_id || undefined
 		const parentRoomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
-		if (!parentRoomID) return // Not interested in a thread if we aren't interested in its wider channel
+		if (!parentRoomID) return // Not interested in a thread if we aren't interested in its wider channel (won't autocreate)
 		const threadRoomID = await createRoom.syncRoom(thread.id) // Create room (will share the same inflight as the initial message to the thread)
 		await announceThread.announceThread(parentRoomID, threadRoomID, thread)
 	},
@@ -202,7 +174,7 @@ module.exports = {
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayGuildUpdateDispatchData} guild
 	 */
-	async onGuildUpdate(client, guild) {
+	async GUILD_UPDATE(client, guild) {
 		const spaceID = select("guild_space", "space_id", {guild_id: guild.id}).pluck().get()
 		if (!spaceID) return
 		await createSpace.syncSpace(guild)
@@ -211,33 +183,54 @@ module.exports = {
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayChannelUpdateDispatchData} channelOrThread
-	 * @param {boolean} isThread
 	 */
-	async onChannelOrThreadUpdate(client, channelOrThread, isThread) {
+	async CHANNEL_UPDATE(client, channelOrThread) {
 		const roomID = select("channel_room", "room_id", {channel_id: channelOrThread.id}).pluck().get()
 		if (!roomID) return // No target room to update the data on
 		await createRoom.syncRoom(channelOrThread.id)
 	},
 
+	/**
+	 * @param {import("./discord-client")} client
+	 * @param {DiscordTypes.GatewayChannelUpdateDispatchData} thread
+	 */
+	async THREAD_UPDATE(client, thread) {
+		await module.exports.CHANNEL_UPDATE(client, thread)
+	},
+
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayChannelPinsUpdateDispatchData} data
 	 */
-	async onChannelPinsUpdate(client, data) {
+	async CHANNEL_PINS_UPDATE(client, data) {
 		const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
 		if (!roomID) return // No target room to update pins in
 		const convertedTimestamp = updatePins.convertTimestamp(data.last_pin_timestamp)
 		await updatePins.updatePins(data.channel_id, roomID, convertedTimestamp)
 	},
 
+	/**
+	 * @param {import("./discord-client")} client
+	 * @param {DiscordTypes.GatewayChannelDeleteDispatchData} channel
+	 */
+	async CHANNEL_DELETE(client, channel) {
+		const guildID = channel["guild_id"]
+		if (!guildID) return // channel must have been a DM channel or something
+		const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get()
+		if (!roomID) return // channel wasn't being bridged in the first place
+		// @ts-ignore
+		await createRoom.unbridgeDeletedChannel(channel, guildID)
+	},
+
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
 	 */
-	async onMessageCreate(client, message) {
+	async MESSAGE_CREATE(client, message) {
 		if (message.author.username === "Deleted User") return // Nothing we can do for deleted users.
 		const channel = client.channels.get(message.channel_id)
 		if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
+
 		const guild = client.guilds.get(channel.guild_id)
 		assert(guild)
 
@@ -246,51 +239,61 @@ module.exports = {
 			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
 		}
 
+		if (dUtils.isEphemeralMessage(message)) return // Ephemeral messages are for the eyes of the receiver only!
+
+		if (!createRoom.existsOrAutocreatable(channel, guild.id)) return // Check that the sending-to room exists or is autocreatable
+
 		const {affected, row} = await speedbump.maybeDoSpeedbump(message.channel_id, message.id)
 		if (affected) return
 
 		// @ts-ignore
-		await sendMessage.sendMessage(message, guild, row),
-		await discordCommandHandler.execute(message, channel, guild)
+		await sendMessage.sendMessage(message, channel, guild, row)
+
+		retrigger.messageFinishedBridging(message.id)
 	},
 
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageUpdateDispatchData} data
 	 */
-	async onMessageUpdate(client, data) {
+	async MESSAGE_UPDATE(client, data) {
+		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
+		// If the message content is a string then it includes all interesting fields and is meaningful.
+		// Otherwise, if there are embeds, then the system generated URL preview embeds.
+		if (!(typeof data.content === "string" || "embeds" in data)) return
+
 		if (data.webhook_id) {
 			const row = select("webhook", "webhook_id", {webhook_id: data.webhook_id}).pluck().get()
 			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
 		}
 
+		if (dUtils.isEphemeralMessage(data)) return // Ephemeral messages are for the eyes of the receiver only!
+
 		// Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from.
 		const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id)
 		if (affected) return
 
-		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
-		// If the message content is a string then it includes all interesting fields and is meaningful.
-		if (typeof data.content === "string") {
-			/** @type {DiscordTypes.GatewayMessageCreateDispatchData} */
-			// @ts-ignore
-			const message = data
+		// Check that the sending-to room exists, and deal with Eventual Consistency(TM)
+		if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_UPDATE, client, data)) return
 
-			const channel = client.channels.get(message.channel_id)
-			if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
-			const guild = client.guilds.get(channel.guild_id)
-			assert(guild)
-			// @ts-ignore
-			await editMessage.editMessage(message, guild, row)
-		}
+		/** @type {DiscordTypes.GatewayMessageCreateDispatchData} */
+		// @ts-ignore
+		const message = data
+		const channel = client.channels.get(message.channel_id)
+		if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
+		const guild = client.guilds.get(channel.guild_id)
+		assert(guild)
+
+		// @ts-ignore
+		await retrigger.pauseChanges(message.id, editMessage.editMessage(message, guild, row))
 	},
 
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageReactionAddDispatchData} data
 	 */
-	async onReactionAdd(client, data) {
+	async MESSAGE_REACTION_ADD(client, data) {
 		if (data.user_id === client.user.id) return // m2d reactions are added by the discord bot user - do not reflect them back to matrix.
-		discordCommandHandler.onReactionAdd(data)
 		await addReaction.addReaction(data)
 	},
 
@@ -306,24 +309,25 @@ module.exports = {
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageDeleteDispatchData} data
 	 */
-	async onMessageDelete(client, data) {
+	async MESSAGE_DELETE(client, data) {
 		speedbump.onMessageDelete(data.id)
+		if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_DELETE, client, data)) return
 		await deleteMessage.deleteMessage(data)
 	},
 
-		/**
+	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageDeleteBulkDispatchData} data
 	 */
-		async onMessageDeleteBulk(client, data) {
-			await deleteMessage.deleteMessageBulk(data)
-		},
+	async MESSAGE_DELETE_BULK(client, data) {
+		await deleteMessage.deleteMessageBulk(data)
+	},
 
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayTypingStartDispatchData} data
 	 */
-	async onTypingStart(client, data) {
+	async TYPING_START(client, data) {
 		const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
 		if (!roomID) return
 		const mxid = from("sim").join("sim_member", "mxid").where({user_id: data.user_id, room_id: roomID}).pluck("mxid").get()
@@ -336,9 +340,27 @@ module.exports = {
 
 	/**
 	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayGuildEmojisUpdateDispatchData | DiscordTypes.GatewayGuildStickersUpdateDispatchData} data
+	 * @param {DiscordTypes.GatewayGuildEmojisUpdateDispatchData} data
 	 */
-	async onExpressionsUpdate(client, data) {
+	async GUILD_EMOJIS_UPDATE(client, data) {
 		await createSpace.syncSpaceExpressions(data, false)
+	},
+
+	/**
+	 * @param {import("./discord-client")} client
+	 * @param {DiscordTypes.GatewayGuildStickersUpdateDispatchData} data
+	 */
+	async GUILD_STICKERS_UPDATE(client, data) {
+		await createSpace.syncSpaceExpressions(data, false)
+	},
+
+	/**
+	 * @param {import("./discord-client")} client
+	 * @param {DiscordTypes.GatewayPresenceUpdateDispatchData} data
+	 */
+	PRESENCE_UPDATE(client, data) {
+		const status = data.status
+		if (!status) return
+		setPresence.presenceTracker.incomingPresence(data.user.id, data.guild_id, status)
 	}
 }

src/db/migrate.js

@@ -6,7 +6,8 @@ const {join} = require("path")
 async function migrate(db) {
 	let files = fs.readdirSync(join(__dirname, "migrations"))
 	files = files.sort()
-	db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run()
+	db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL, PRIMARY KEY (filename)) WITHOUT ROWID").run()
+	/** @type {string} */
 	let progress = db.prepare("SELECT * FROM migration").pluck().get()
 	if (!progress) {
 		progress = ""
@@ -37,6 +38,8 @@ async function migrate(db) {
 	if (migrationRan) {
 		console.log("Database migrations all done.")
 	}
+
+	db.pragma("foreign_keys = on")
 }
 
 module.exports.migrate = migrate

src/db/migrations/0002-optimise-profile-content.up.js

@@ -3,6 +3,7 @@ module.exports = async function(db) {
 	const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all()
 	const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?")
 	db.transaction(() => {
+		/* c8 ignore next 6 */
 		for (let s of contents) {
 			let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s))
 			const unsignedHash = hasher.h64Raw(b)

src/db/migrations/0011-move-bridge-bot-to-real-id.up.js

@@ -0,0 +1,15 @@
+/*
+	a. If the bridge bot sim already has the correct ID:
+		- No rows updated.
+
+	b. If the bridge bot sim has the wrong ID but there's no duplicate:
+		- One row updated.
+
+	c. If the bridge bot sim has the wrong ID and there's a duplicate:
+		- One row updated (replaces an existing row).
+*/
+
+module.exports = async function(db) {
+	const id = require("../../../addbot").id
+	db.prepare("UPDATE OR REPLACE sim SET user_id = ? WHERE user_id = '0'").run(id)
+}

src/db/migrations/0012-add-member-power.sql

@@ -0,0 +1,14 @@
+BEGIN TRANSACTION;
+
+-- the power we want them to have
+CREATE TABLE IF NOT EXISTS member_power (
+	mxid			TEXT NOT NULL,
+	room_id		TEXT NOT NULL,
+	power_level	INTEGER NOT NULL,
+	PRIMARY KEY(mxid, room_id)
+) WITHOUT ROWID;
+
+-- the power they have
+ALTER TABLE member_cache ADD COLUMN power_level INTEGER NOT NULL DEFAULT 0;
+
+COMMIT;

src/db/migrations/0013-media-proxy.sql

@@ -0,0 +1,8 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE "media_proxy" (
+	"permitted_hash"	INTEGER NOT NULL,
+	PRIMARY KEY("permitted_hash")
+) WITHOUT ROWID;
+
+COMMIT;

src/db/migrations/0014-add-guild-active.sql

@@ -0,0 +1,11 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE "guild_active" (
+	"guild_id"	TEXT NOT NULL,
+	"autocreate"	INTEGER NOT NULL,
+	PRIMARY KEY("guild_id")
+) WITHOUT ROWID;
+
+INSERT INTO guild_active (guild_id, autocreate) SELECT guild_id, 1 FROM guild_space;
+
+COMMIT;

src/db/migrations/0015-add-guild-id-to-channel-room.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE channel_room ADD COLUMN guild_id TEXT;
+
+COMMIT;

src/db/migrations/0016-foreign-keys.sql

@@ -0,0 +1,147 @@
+-- /docs/foreign-keys.md
+
+-- 2
+BEGIN TRANSACTION;
+
+-- *** channel_room ***
+
+-- 4
+-- adding UNIQUE to room_id here will auto-generate the usable index we wanted
+CREATE TABLE "new_channel_room" (
+	"channel_id"	TEXT NOT NULL,
+	"room_id"	TEXT NOT NULL UNIQUE,
+	"name"	TEXT NOT NULL,
+	"nick"	TEXT,
+	"thread_parent"	TEXT,
+	"custom_avatar"	TEXT,
+	"last_bridged_pin_timestamp"	INTEGER,
+	"speedbump_id"	TEXT,
+	"speedbump_checked"	INTEGER,
+	"speedbump_webhook_id"	TEXT,
+	"guild_id"	TEXT,
+	PRIMARY KEY("channel_id"),
+	FOREIGN KEY("guild_id") REFERENCES "guild_active"("guild_id") ON DELETE CASCADE
+) WITHOUT ROWID;
+-- 5
+INSERT INTO new_channel_room (channel_id, room_id, name, nick, thread_parent, custom_avatar, last_bridged_pin_timestamp, speedbump_id, speedbump_checked, speedbump_webhook_id, guild_id) SELECT channel_id, room_id, name, nick, thread_parent, custom_avatar, last_bridged_pin_timestamp, speedbump_id, speedbump_checked, speedbump_webhook_id, guild_id FROM channel_room;
+-- 6
+DROP TABLE channel_room;
+-- 7
+ALTER TABLE new_channel_room RENAME TO channel_room;
+
+-- *** message_channel ***
+
+-- 4
+CREATE TABLE "new_message_channel" (
+	"message_id"	TEXT NOT NULL,
+	"channel_id"	TEXT NOT NULL,
+	PRIMARY KEY("message_id"),
+	FOREIGN KEY("channel_id") REFERENCES "channel_room"("channel_id") ON DELETE CASCADE
+) WITHOUT ROWID;
+-- 5
+-- don't copy any orphaned messages
+INSERT INTO new_message_channel (message_id, channel_id) SELECT message_id, channel_id FROM message_channel WHERE channel_id IN (SELECT channel_id FROM channel_room);
+-- 6
+DROP TABLE message_channel;
+-- 7
+ALTER TABLE new_message_channel RENAME TO message_channel;
+
+-- *** event_message ***
+
+-- clean up any orphaned events
+DELETE FROM event_message WHERE message_id NOT IN (SELECT message_id FROM message_channel);
+-- 4
+CREATE TABLE "new_event_message" (
+	"event_id"	TEXT NOT NULL,
+	"event_type"	TEXT,
+	"event_subtype"	TEXT,
+	"message_id"	TEXT NOT NULL,
+	"part"	INTEGER NOT NULL,
+	"reaction_part"	INTEGER NOT NULL,
+	"source"	INTEGER NOT NULL,
+	PRIMARY KEY("message_id","event_id"),
+	FOREIGN KEY("message_id") REFERENCES "message_channel"("message_id") ON DELETE CASCADE
+) WITHOUT ROWID;
+-- 5
+INSERT INTO new_event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) SELECT event_id, event_type, event_subtype, message_id, part, reaction_part, source FROM event_message;
+-- 6
+DROP TABLE event_message;
+-- 7
+ALTER TABLE new_event_message RENAME TO event_message;
+
+-- *** guild_space ***
+
+-- 4
+CREATE TABLE "new_guild_space" (
+	"guild_id"	TEXT NOT NULL,
+	"space_id"	TEXT NOT NULL,
+	"privacy_level"	INTEGER NOT NULL DEFAULT 0,
+	PRIMARY KEY("guild_id"),
+	FOREIGN KEY("guild_id") REFERENCES "guild_active"("guild_id") ON DELETE CASCADE
+) WITHOUT ROWID;
+-- 5
+INSERT INTO new_guild_space (guild_id, space_id, privacy_level) SELECT guild_id, space_id, privacy_level FROM guild_space;
+-- 6
+DROP TABLE guild_space;
+-- 7
+ALTER TABLE new_guild_space RENAME TO guild_space;
+
+-- *** reaction ***
+
+-- 4
+CREATE TABLE "new_reaction" (
+	"hashed_event_id"	INTEGER NOT NULL,
+	"message_id"	TEXT NOT NULL,
+	"encoded_emoji"	TEXT NOT NULL,
+	PRIMARY KEY("hashed_event_id"),
+	FOREIGN KEY("message_id") REFERENCES "message_channel"("message_id") ON DELETE CASCADE
+) WITHOUT ROWID;
+-- 5
+INSERT INTO new_reaction (hashed_event_id, message_id, encoded_emoji) SELECT hashed_event_id, message_id, encoded_emoji FROM reaction WHERE message_id IN (SELECT message_id FROM message_channel);
+-- 6
+DROP TABLE reaction;
+-- 7
+ALTER TABLE new_reaction RENAME TO reaction;
+
+-- *** webhook ***
+
+-- 4
+-- using RESTRICT instead of CASCADE as a reminder that the webhooks also need to be deleted using the Discord API, it can't just be entirely automatic
+CREATE TABLE "new_webhook" (
+	"channel_id"	TEXT NOT NULL,
+	"webhook_id"	TEXT NOT NULL,
+	"webhook_token"	TEXT NOT NULL,
+	PRIMARY KEY("channel_id"),
+	FOREIGN KEY("channel_id") REFERENCES "channel_room"("channel_id") ON DELETE RESTRICT
+) WITHOUT ROWID;
+-- 5
+INSERT INTO new_webhook (channel_id, webhook_id, webhook_token) SELECT channel_id, webhook_id, webhook_token FROM webhook WHERE channel_id IN (SELECT channel_id FROM channel_room);
+-- 6
+DROP TABLE webhook;
+-- 7
+ALTER TABLE new_webhook RENAME TO webhook;
+
+-- *** sim ***
+
+-- 4
+-- while we're at it, rebuild this table to give it WITHOUT ROWID, remove UNIQUE, and replace the localpart column with username. no foreign keys needed
+CREATE TABLE "new_sim" (
+	"user_id"	TEXT NOT NULL,
+	"username"	TEXT NOT NULL,
+	"sim_name"	TEXT NOT NULL,
+	"mxid"	TEXT NOT NULL,
+	PRIMARY KEY("user_id")
+) WITHOUT ROWID;
+-- 5
+INSERT INTO new_sim (user_id, username, sim_name, mxid) SELECT user_id, sim_name, sim_name, mxid FROM sim;
+-- 6
+DROP TABLE sim;
+-- 7
+ALTER TABLE new_sim RENAME TO sim;
+
+-- *** end ***
+
+-- 10
+PRAGMA foreign_key_check;
+-- 11
+COMMIT;