Added a comment explaining sims to db\orm-defs.d.ts

.gitignore

@@ -1,18 +1,7 @@
-# Secrets
+node_modules
 config.js
 registration.yaml
-ooye.db*
-events.db*
-backfill.db*
-custom-webroot
-
-# Automatically generated
-node_modules
 coverage
+db/ooye.db*
 test/res/*
 !test/res/lottie*
-icon.svg
-*~
-.#*
-\#*#
-launch.json

addbot.js

@@ -1,27 +1,15 @@
-#!/usr/bin/env node
 // @ts-check
 
-const DiscordTypes = require("discord-api-types/v10")
-
-const {reg} = require("./src/matrix/read-registration")
-const token = reg.ooye.discord_token
-const id = Buffer.from(token.split(".")[0], "base64").toString()
-const permissions =
-( DiscordTypes.PermissionFlagsBits.ManageWebhooks
-| DiscordTypes.PermissionFlagsBits.ManageGuildExpressions
-| DiscordTypes.PermissionFlagsBits.ManageMessages
-| DiscordTypes.PermissionFlagsBits.PinMessages
-| DiscordTypes.PermissionFlagsBits.UseExternalEmojis)
+const config = require("./config")
 
 function addbot() {
-	return `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=${id}&scope=bot&permissions=${permissions} `
+	const token = config.discordToken
+	const id = Buffer.from(token.split(".")[0], "base64")
+	return `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=${id}&scope=bot&permissions=1610883072 `
 }
 
-/* c8 ignore next 3 */
 if (process.argv.find(a => a.endsWith("addbot") || a.endsWith("addbot.js"))) {
 	console.log(addbot())
 }
 
-module.exports.id = id
 module.exports.addbot = addbot
-module.exports.permissions = permissions

addbot.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 echo "Open this link to add the bot to a Discord server:"
-echo "https://discord.com/oauth2/authorize?client_id=$(grep discord_token registration.yaml | sed -E 's!.*: ["'\'']([A-Za-z0-9+=/_-]*).*!\1!g' | base64 -d)&scope=bot&permissions=1610883072"
+echo "https://discord.com/oauth2/authorize?client_id=$(grep discordToken config.js | sed -E 's!.*: ["'\'']([A-Za-z0-9+=/_-]*).*!\1!g' | base64 -d)&scope=bot&permissions=1610883072"

config.example.js

@@ -0,0 +1,3 @@
+module.exports = {
+	discordToken: "yes"
+}

d2m/actions/add-reaction.js

@@ -21,11 +21,11 @@ async function addReaction(data) {
 	const user = data.member?.user
 	assert.ok(user && user.username)
 
-	const parentID = select("event_message", "event_id", {message_id: data.message_id}, "ORDER BY reaction_part").pluck().get()
+	const parentID = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get()
 	if (!parentID) return // Nothing can be done if the parent message was never bridged.
 	assert.equal(typeof parentID, "string")
 
-	const key = await emojiToKey.emojiToKey(data.emoji, data.message_id)
+	const key = await emojiToKey.emojiToKey(data.emoji)
 	const shortcode = key.startsWith("mxc://") ? `:${data.emoji.name}:` : undefined
 
 	const roomID = await createRoom.ensureRoom(data.channel_id)

d2m/actions/announce-thread.js

@@ -1,6 +1,6 @@
 // @ts-check
 
-const assert = require("assert").strict
+const assert = require("assert")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough

d2m/actions/create-room.js

@@ -0,0 +1,414 @@
+// @ts-check
+
+const assert = require("assert").strict
+const DiscordTypes = require("discord-api-types/v10")
+const reg = require("../../matrix/read-registration")
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select} = passthrough
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+/** @type {import("../../matrix/kstate")} */
+const ks = sync.require("../../matrix/kstate")
+/** @type {import("./create-space")}) */
+const createSpace = sync.require("./create-space") // watch out for the require loop
+
+/**
+ * There are 3 levels of room privacy:
+ * 0: Room is invite-only.
+ * 1: Anybody can use a link to join.
+ * 2: Room is published in room directory.
+ */
+const PRIVACY_ENUMS = {
+	PRESET: ["private_chat", "public_chat", "public_chat"],
+	VISIBILITY: ["private", "private", "public"],
+	SPACE_HISTORY_VISIBILITY: ["invited", "world_readable", "world_readable"], // copying from element client
+	ROOM_HISTORY_VISIBILITY: ["shared", "shared", "world_readable"], // any events sent after <value> are visible, but for world_readable anybody can read without even joining
+	GUEST_ACCESS: ["can_join", "forbidden", "forbidden"], // whether guests can join space if other conditions are met
+	SPACE_JOIN_RULES: ["invite", "public", "public"],
+	ROOM_JOIN_RULES: ["restricted", "public", "public"]
+}
+
+const DEFAULT_PRIVACY_LEVEL = 0
+
+/** @type {Map<string, Promise<string>>} channel ID -> Promise<room ID> */
+const inflightRoomCreate = new Map()
+
+/**
+ * Async because it gets all room state from the homeserver.
+ * @param {string} roomID
+ */
+async function roomToKState(roomID) {
+	const root = await api.getAllState(roomID)
+	return ks.stateToKState(root)
+}
+
+/**
+ * @param {string} roomID
+ * @param {any} kstate
+ */
+function applyKStateDiffToRoom(roomID, kstate) {
+	const events = ks.kstateToState(kstate)
+	return Promise.all(events.map(({type, state_key, content}) =>
+		api.sendState(roomID, type, state_key, content)
+	))
+}
+
+/**
+ * @param {{id: string, name: string, topic?: string?, type: number}} channel
+ * @param {{id: string}} guild
+ * @param {string | null | undefined} customName
+ */
+function convertNameAndTopic(channel, guild, customName) {
+	let channelPrefix =
+		( channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] "
+		: channel.type === DiscordTypes.ChannelType.PrivateThread ? "[🔒⛓️] "
+		: channel.type === DiscordTypes.ChannelType.GuildVoice ? "[🔊] "
+		: "")
+	const chosenName = customName || (channelPrefix + channel.name);
+	const maybeTopicWithPipe = channel.topic ? ` | ${channel.topic}` : '';
+	const maybeTopicWithNewlines = channel.topic ? `${channel.topic}\n\n` : '';
+	const channelIDPart = `Channel ID: ${channel.id}`;
+	const guildIDPart = `Guild ID: ${guild.id}`;
+
+	const convertedTopic = customName
+		 ? `#${channel.name}${maybeTopicWithPipe}\n\n${channelIDPart}\n${guildIDPart}`
+		 : `${maybeTopicWithNewlines}${channelIDPart}\n${guildIDPart}`;
+
+	return [chosenName, convertedTopic];
+}
+
+/**
+ * Async because it may create the guild and/or upload the guild icon to mxc.
+ * @param {DiscordTypes.APIGuildTextChannel | DiscordTypes.APIThreadChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
+ */
+async function channelToKState(channel, guild) {
+	const spaceID = await createSpace.ensureSpace(guild)
+	assert(typeof spaceID === "string")
+	const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get()
+	assert(typeof privacyLevel === "number")
+
+	const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get()
+	const customName = row?.nick
+	const customAvatar = row?.custom_avatar
+	const [convertedName, convertedTopic] = convertNameAndTopic(channel, guild, customName)
+
+	const avatarEventContent = {}
+	if (customAvatar) {
+		avatarEventContent.url = customAvatar
+	} else if (guild.icon) {
+		avatarEventContent.discord_path = file.guildIcon(guild)
+		avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
+	}
+
+	let history_visibility = PRIVACY_ENUMS.ROOM_HISTORY_VISIBILITY[privacyLevel]
+	if (channel["thread_metadata"]) history_visibility = "world_readable"
+
+	/** @type {{join_rule: string, allow?: any}} */
+	let join_rules = {
+		join_rule: "restricted",
+		allow: [{
+			type: "m.room_membership",
+			room_id: spaceID
+		}]
+	}
+	if (PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel] !== "restricted") {
+		join_rules = {join_rule: PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel]}
+	}
+
+	const channelKState = {
+		"m.room.name/": {name: convertedName},
+		"m.room.topic/": {topic: convertedTopic},
+		"m.room.avatar/": avatarEventContent,
+		"m.room.guest_access/": {guest_access: PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
+		"m.room.history_visibility/": {history_visibility},
+		[`m.space.parent/${spaceID}`]: {
+			via: [reg.ooye.server_name],
+			canonical: true
+		},
+		/** @type {{join_rule: string, [x: string]: any}} */
+		"m.room.join_rules/": join_rules,
+		"m.room.power_levels/": {
+			events: {
+				"m.room.avatar": 0
+			},
+			users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})
+		},
+		"chat.schildi.hide_ui/read_receipts": {
+			hidden: true
+		},
+		[`uk.half-shot.bridge/moe.cadence.ooye://discord/${guild.id}/${channel.id}`]: {
+			bridgebot: `@${reg.sender_localpart}:${reg.ooye.server_name}`,
+			protocol: {
+				id: "discord",
+				displayname: "Discord"
+			},
+			network: {
+				id: guild.id,
+				displayname: guild.name,
+				avatar_url: await file.uploadDiscordFileToMxc(file.guildIcon(guild))
+			},
+			channel: {
+				id: channel.id,
+				displayname: channel.name,
+				external_url: `https://discord.com/channels/${guild.id}/${channel.id}`
+			}
+		}
+	}
+
+	return {spaceID, privacyLevel, channelKState}
+}
+
+/**
+ * Create a bridge room, store the relationship in the database, and add it to the guild's space.
+ * @param {DiscordTypes.APIGuildTextChannel} channel
+ * @param guild
+ * @param {string} spaceID
+ * @param {any} kstate
+ * @param {number} privacyLevel
+ * @returns {Promise<string>} room ID
+ */
+async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
+	let threadParent = null
+	if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id
+
+	// Name and topic can be done earlier in room creation rather than in initial_state
+	// https://spec.matrix.org/latest/client-server-api/#creation
+	const name = kstate["m.room.name/"].name
+	delete kstate["m.room.name/"]
+	assert(name)
+	const topic = kstate["m.room.topic/"].topic
+	delete kstate["m.room.topic/"]
+	assert(topic)
+
+	const roomID = await postApplyPowerLevels(kstate, async kstate => {
+		const roomID = await api.createRoom({
+			name,
+			topic,
+			preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
+			visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
+			invite: [],
+			initial_state: ks.kstateToState(kstate)
+		})
+
+		db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent)
+
+		return roomID
+	})
+
+	// Put the newly created child into the space, no need to await this
+	_syncSpaceMember(channel, spaceID, roomID)
+
+	return roomID
+}
+
+/**
+ * Handling power levels separately. The spec doesn't specify what happens, Dendrite differs,
+ * and Synapse does an absolutely insane *shallow merge* of what I provide on top of what it creates.
+ * We don't want the `events` key to be overridden completely.
+ * https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/room.py#L1170-L1210
+ * https://github.com/matrix-org/matrix-spec/issues/492
+ * @param {any} kstate
+ * @param {(_: any) => Promise<string>} callback must return room ID
+ * @returns {Promise<string>} room ID
+ */
+async function postApplyPowerLevels(kstate, callback) {
+	const powerLevelContent = kstate["m.room.power_levels/"]
+	const kstateWithoutPowerLevels = {...kstate}
+	delete kstateWithoutPowerLevels["m.room.power_levels/"]
+
+	/** @type {string} */
+	const roomID = await callback(kstateWithoutPowerLevels)
+
+	// Now *really* apply the power level overrides on top of what Synapse *really* set
+	if (powerLevelContent) {
+		const newRoomKState = await roomToKState(roomID)
+		const newRoomPowerLevelsDiff = ks.diffKState(newRoomKState, {"m.room.power_levels/": powerLevelContent})
+		await applyKStateDiffToRoom(roomID, newRoomPowerLevelsDiff)
+	}
+
+	return roomID
+}
+
+/**
+ * @param {DiscordTypes.APIGuildChannel} channel
+ */
+function channelToGuild(channel) {
+	const guildID = channel.guild_id
+	assert(guildID)
+	const guild = discord.guilds.get(guildID)
+	assert(guild)
+	return guild
+}
+
+/*
+	Ensure flow:
+	1. Get IDs
+	2. Does room exist? If so great!
+	(it doesn't, so it needs to be created)
+	3. Get kstate for channel
+	4. Create room, return new ID
+
+	Ensure + sync flow:
+	1. Get IDs
+	2. Does room exist?
+	2.5: If room does exist AND wasn't asked to sync: return here
+	3. Get kstate for channel
+	4. Create room with kstate if room doesn't exist
+	5. Get and update room state with kstate if room does exist
+*/
+
+/**
+ * @param {string} channelID
+ * @param {boolean} shouldActuallySync false if just need to ensure room exists (which is a quick database check), true if also want to sync room data when it does exist (slow)
+ * @returns {Promise<string>} room ID
+ */
+async function _syncRoom(channelID, shouldActuallySync) {
+	/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
+	const channel = discord.channels.get(channelID)
+	assert.ok(channel)
+	const guild = channelToGuild(channel)
+
+	if (inflightRoomCreate.has(channelID)) {
+		await inflightRoomCreate.get(channelID) // just waiting, and then doing a new db query afterwards, is the simplest way of doing it
+	}
+
+	const existing = select("channel_room", ["room_id", "thread_parent"], {channel_id: channelID}).get()
+
+	if (!existing) {
+		const creation = (async () => {
+			const {spaceID, privacyLevel, channelKState} = await channelToKState(channel, guild)
+			const roomID = await createRoom(channel, guild, spaceID, channelKState, privacyLevel)
+			inflightRoomCreate.delete(channelID) // OK to release inflight waiters now. they will read the correct `existing` row
+			return roomID
+		})()
+		inflightRoomCreate.set(channelID, creation)
+		return creation // Naturally, the newly created room is already up to date, so we can always skip syncing here.
+	}
+
+	const roomID = existing.room_id
+
+	if (!shouldActuallySync) {
+		return existing.room_id // only need to ensure room exists, and it does. return the room ID
+	}
+
+	console.log(`[room sync] to matrix: ${channel.name}`)
+
+	const {spaceID, channelKState} = await channelToKState(channel, guild) // calling this in both branches because we don't want to calculate this if not syncing
+
+	// sync channel state to room
+	const roomKState = await roomToKState(roomID)
+	if (+roomKState["m.room.create/"].room_version <= 8) {
+		// join_rule `restricted` is not available in room version < 8 and not working properly in version == 8
+		// read more: https://spec.matrix.org/v1.8/rooms/v9/
+		// we have to use `public` instead, otherwise the room will be unjoinable.
+		channelKState["m.room.join_rules/"] = {join_rule: "public"}
+	}
+	const roomDiff = ks.diffKState(roomKState, channelKState)
+	const roomApply = applyKStateDiffToRoom(roomID, roomDiff)
+	db.prepare("UPDATE channel_room SET name = ? WHERE room_id = ?").run(channel.name, roomID)
+
+	// sync room as space member
+	const spaceApply = _syncSpaceMember(channel, spaceID, roomID)
+	await Promise.all([roomApply, spaceApply])
+
+	return roomID
+}
+
+/** Ensures the room exists. If it doesn't, creates the room with an accurate initial state. */
+function ensureRoom(channelID) {
+	return _syncRoom(channelID, false)
+}
+
+/** Actually syncs. Gets all room state from the homeserver in order to diff, and uploads the icon to mxc if it has changed. */
+function syncRoom(channelID) {
+	return _syncRoom(channelID, true)
+}
+
+async function _unbridgeRoom(channelID) {
+	/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
+	const channel = discord.channels.get(channelID)
+	assert.ok(channel)
+	return unbridgeDeletedChannel(channel.id, channel.guild_id)
+}
+
+async function unbridgeDeletedChannel(channelID, guildID) {
+	const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
+	assert.ok(roomID)
+	const spaceID = select("guild_space", "space_id", {guild_id: guildID}).pluck().get()
+	assert.ok(spaceID)
+
+	// remove room from being a space member
+	await api.sendState(roomID, "m.space.parent", spaceID, {})
+	await api.sendState(spaceID, "m.space.child", roomID, {})
+
+	// remove declaration that the room is bridged
+	await api.sendState(roomID, "uk.half-shot.bridge", `moe.cadence.ooye://discord/${guildID}/${channelID}`, {})
+
+	// send a notification in the room
+	await api.sendEvent(roomID, "m.room.message", {
+		msgtype: "m.notice",
+		body: "⚠️ This room was removed from the bridge."
+	})
+
+	// leave room
+	await api.leaveRoom(roomID)
+
+	// delete room from database
+	const {changes} = db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channelID)
+	assert.equal(changes, 1)
+}
+
+/**
+ * Async because it gets all space state from the homeserver, then if necessary sends one state event back.
+ * @param {DiscordTypes.APIGuildTextChannel} channel
+ * @param {string} spaceID
+ * @param {string} roomID
+ * @returns {Promise<string[]>}
+ */
+async function _syncSpaceMember(channel, spaceID, roomID) {
+	const spaceKState = await roomToKState(spaceID)
+	let spaceEventContent = {}
+	if (
+		channel.type !== DiscordTypes.ChannelType.PrivateThread // private threads do not belong in the space (don't offer people something they can't join)
+		&& !channel["thread_metadata"]?.archived // archived threads do not belong in the space (don't offer people conversations that are no longer relevant)
+	) {
+		spaceEventContent = {
+			via: [reg.ooye.server_name]
+		}
+	}
+	const spaceDiff = ks.diffKState(spaceKState, {
+		[`m.space.child/${roomID}`]: spaceEventContent
+	})
+	return applyKStateDiffToRoom(spaceID, spaceDiff)
+}
+
+async function createAllForGuild(guildID) {
+	const channelIDs = discord.guildChannelMap.get(guildID)
+	assert.ok(channelIDs)
+	for (const channelID of channelIDs) {
+		const allowedTypes = [DiscordTypes.ChannelType.GuildText, DiscordTypes.ChannelType.PublicThread]
+		// @ts-ignore
+		if (allowedTypes.includes(discord.channels.get(channelID)?.type)) {
+			const roomID = await syncRoom(channelID)
+			console.log(`synced ${channelID} <-> ${roomID}`)
+		}
+	}
+}
+
+module.exports.DEFAULT_PRIVACY_LEVEL = DEFAULT_PRIVACY_LEVEL
+module.exports.PRIVACY_ENUMS = PRIVACY_ENUMS
+module.exports.createRoom = createRoom
+module.exports.ensureRoom = ensureRoom
+module.exports.syncRoom = syncRoom
+module.exports.createAllForGuild = createAllForGuild
+module.exports.channelToKState = channelToKState
+module.exports.roomToKState = roomToKState
+module.exports.applyKStateDiffToRoom = applyKStateDiffToRoom
+module.exports.postApplyPowerLevels = postApplyPowerLevels
+module.exports._convertNameAndTopic = convertNameAndTopic
+module.exports._unbridgeRoom = _unbridgeRoom
+module.exports.unbridgeDeletedChannel = unbridgeDeletedChannel

d2m/actions/create-room.test.js

@@ -0,0 +1,89 @@
+// @ts-check
+
+const {channelToKState, _convertNameAndTopic} = require("./create-room")
+const {kstateStripConditionals} = require("../../matrix/kstate")
+const {test} = require("supertape")
+const testData = require("../../test/data")
+
+const passthrough = require("../../passthrough")
+const {db} = passthrough
+
+test("channel2room: discoverable privacy room", async t => {
+	db.prepare("UPDATE guild_space SET privacy_level = 2").run()
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
+		Object.assign({}, testData.room.general, {
+			"m.room.guest_access/": {guest_access: "forbidden"},
+			"m.room.join_rules/": {join_rule: "public"},
+			"m.room.history_visibility/": {history_visibility: "world_readable"}
+		})
+	)
+})
+
+test("channel2room: linkable privacy room", async t => {
+	db.prepare("UPDATE guild_space SET privacy_level = 1").run()
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
+		Object.assign({}, testData.room.general, {
+			"m.room.guest_access/": {guest_access: "forbidden"},
+			"m.room.join_rules/": {join_rule: "public"}
+		})
+	)
+})
+
+test("channel2room: invite-only privacy room", async t => {
+	db.prepare("UPDATE guild_space SET privacy_level = 0").run()
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
+		testData.room.general
+	)
+})
+
+test("convertNameAndTopic: custom name and topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, "hauntings"),
+		["hauntings", "#the-twilight-zone | Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: custom name, no topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, "hauntings"),
+		["hauntings", "#the-twilight-zone\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: original name and topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, null),
+		["the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: original name, no topic", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, null),
+		["the-twilight-zone", "Channel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: public thread icon", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 11}, {id: "456"}, null),
+		["[⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: private thread icon", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 12}, {id: "456"}, null),
+		["[🔒⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})
+
+test("convertNameAndTopic: voice channel icon", t => {
+	t.deepEqual(
+		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 2}, {id: "456"}, null),
+		["[🔊] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"]
+	)
+})

d2m/actions/create-space.js

@@ -1,10 +1,9 @@
 // @ts-check
 
 const assert = require("assert").strict
-const {isDeepStrictEqual} = require("util")
 const DiscordTypes = require("discord-api-types/v10")
-const Ty = require("../../types")
-const {reg} = require("../../matrix/read-registration")
+const deepEqual = require("deep-equal")
+const reg = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
@@ -31,12 +30,8 @@ async function createSpace(guild, kstate) {
 	const topic = kstate["m.room.topic/"]?.topic || undefined
 	assert(name)
 
-	const memberCount = guild["member_count"] ?? guild.approximate_member_count ?? 0
-	const enablePresenceByDefault = +(memberCount < 50) // scary! all active users in a presence-enabled guild will be pinging the server every <30 seconds to stay online
-	const globalAdmins = select("member_power", "mxid", {room_id: "*"}).pluck().all()
-
-	const roomCreate = await createRoom.postApplyPowerLevels(kstate, async kstate => {
-		const roomID = await api.createRoom({
+	const roomID = await createRoom.postApplyPowerLevels(kstate, async kstate => {
+		return api.createRoom({
 			name,
 			preset: createRoom.PRIVACY_ENUMS.PRESET[createRoom.DEFAULT_PRIVACY_LEVEL], // New spaces will have to use the default privacy level; we obviously can't look up the existing entry
 			visibility: createRoom.PRIVACY_ENUMS.VISIBILITY[createRoom.DEFAULT_PRIVACY_LEVEL],
@@ -44,17 +39,15 @@ async function createSpace(guild, kstate) {
 				events_default: 100, // space can only be managed by bridge
 				invite: 0 // any existing member can invite others
 			},
-			invite: globalAdmins,
+			invite: reg.ooye.invite,
 			topic,
-			initial_state: await ks.kstateToState(kstate),
-			creation_content: ks.kstateToCreationContent(kstate)
+			creation_content: {
+				type: "m.space"
+			},
+			initial_state: ks.kstateToState(kstate)
 		})
-		const roomCreate = await api.getStateEventOuter(roomID, "m.room.create", "")
-		return roomCreate
 	})
-	const roomID = roomCreate.room_id
-
-	db.prepare("INSERT INTO guild_space (guild_id, space_id, presence) VALUES (?, ?, ?)").run(guild.id, roomID, enablePresenceByDefault)
+	db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, roomID)
 	return roomID
 }
 
@@ -63,24 +56,19 @@ async function createSpace(guild, kstate) {
  * @param {number} privacyLevel
  */
 async function guildToKState(guild, privacyLevel) {
-	assert.equal(typeof privacyLevel, "number")
-	const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all()
-	const additionalCreators = select("member_power", "mxid", {room_id: "*"}, "AND power_level > 100").pluck().all()
+	const avatarEventContent = {}
+	if (guild.icon) {
+		avatarEventContent.discord_path = file.guildIcon(guild)
+		avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
+	}
 
 	const guildKState = {
-		"m.room.create/": {
-			type: "m.space",
-			additional_creators: additionalCreators
-		},
 		"m.room.name/": {name: guild.name},
-		"m.room.avatar/": {
-			$if: guild.icon,
-			url: {$url: file.guildIcon(guild)}
-		},
+		"m.room.avatar/": avatarEventContent,
 		"m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
 		"m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]},
 		"m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]},
-		"m.room.power_levels/": {users: globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {})} // used in guild initial creation postApplyPowerLevels
+		"m.room.power_levels/": {users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})}
 	}
 
 	return guildKState
@@ -102,9 +90,6 @@ async function _syncSpace(guild, shouldActuallySync) {
 	const row = select("guild_space", ["space_id", "privacy_level"], {guild_id: guild.id}).get()
 
 	if (!row) {
-		const autocreate = select("guild_active", "autocreate", {guild_id: guild.id}).pluck().get()
-		assert.equal(autocreate, 1, `refusing to implicitly create a space for guild ${guild.id}. set the guild_active data first before calling ensureSpace/syncSpace.`)
-
 		const creation = (async () => {
 			const guildKState = await guildToKState(guild, createRoom.DEFAULT_PRIVACY_LEVEL) // New spaces will have to use the default privacy level; we obviously can't look up the existing entry
 			const spaceID = await createSpace(guild, guildKState)
@@ -124,13 +109,11 @@ async function _syncSpace(guild, shouldActuallySync) {
 	console.log(`[space sync] to matrix: ${guild.name}`)
 
 	const guildKState = await guildToKState(guild, privacy_level) // calling this in both branches because we don't want to calculate this if not syncing
-	ks.kstateStripConditionals(guildKState) // pre-upload icons before diffing
-	await ks.kstateUploadMxc(guildKState)
 
 	// sync guild state to space
-	const spaceKState = await ks.roomToKState(spaceID)
+	const spaceKState = await createRoom.roomToKState(spaceID)
 	const spaceDiff = ks.diffKState(spaceKState, guildKState)
-	await ks.applyKStateDiffToRoom(spaceID, spaceDiff)
+	await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff)
 
 	// guild icon was changed, so room avatars need to be updated as well as the space ones
 	// doing it this way rather than calling syncRoom for great efficiency gains
@@ -139,10 +122,15 @@ async function _syncSpace(guild, shouldActuallySync) {
 		// don't try to update rooms with custom avatars though
 		const roomsWithCustomAvatars = select("channel_room", "room_id", {}, "WHERE custom_avatar IS NOT NULL").pluck().all()
 
-		for await (const room of api.generateFullHierarchy(spaceID)) {
-			if (room.avatar_url === newAvatarState.url) continue
-			if (roomsWithCustomAvatars.includes(room.room_id)) continue
-			await api.sendState(room.room_id, "m.room.avatar", "", newAvatarState)
+		const childRooms = ks.kstateToState(spaceKState).filter(({type, state_key, content}) => {
+			return type === "m.space.child" && "via" in content && !roomsWithCustomAvatars.includes(state_key)
+		}).map(({state_key}) => state_key)
+
+		for (const roomID of childRooms) {
+			const avatarEventContent = await api.getStateEvent(roomID, "m.room.avatar", "")
+			if (avatarEventContent.url !== newAvatarState.url) {
+				await api.sendState(roomID, "m.room.avatar", "", newAvatarState)
+			}
 		}
 	}
 
@@ -187,23 +175,23 @@ async function syncSpaceFully(guildID) {
 	console.log(`[space sync] to matrix: ${guild.name}`)
 
 	const guildKState = await guildToKState(guild, privacy_level)
-	ks.kstateStripConditionals(guildKState) // pre-upload icons before diffing
-	await ks.kstateUploadMxc(guildKState)
 
 	// sync guild state to space
-	const spaceKState = await ks.roomToKState(spaceID)
+	const spaceKState = await createRoom.roomToKState(spaceID)
 	const spaceDiff = ks.diffKState(spaceKState, guildKState)
-	await ks.applyKStateDiffToRoom(spaceID, spaceDiff)
+	await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff)
 
-	const childRooms = await api.getFullHierarchy(spaceID)
+	const childRooms = ks.kstateToState(spaceKState).filter(({type, content}) => {
+		return type === "m.space.child" && "via" in content
+	}).map(({state_key}) => state_key)
 
-	for (const {room_id} of childRooms) {
-		const channelID = select("channel_room", "channel_id", {room_id}).pluck().get()
+	for (const roomID of childRooms) {
+		const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
 		if (!channelID) continue
 		if (discord.channels.has(channelID)) {
 			await createRoom.syncRoom(channelID)
 		} else {
-			await createRoom.unbridgeChannel({id: channelID}, guildID)
+			await createRoom.unbridgeDeletedChannel(channelID, guildID)
 		}
 	}
 
@@ -229,24 +217,22 @@ async function syncSpaceExpressions(data, checkBeforeSync) {
 	 */
 	async function update(spaceID, key, eventKey, fn) {
 		if (!(key in data) || !data[key].length) return
-		const guild = discord.guilds.get(data.guild_id)
-		assert(guild)
-		const content = await fn(data[key], guild)
+		const content = await fn(data[key])
 		if (checkBeforeSync) {
 			let existing
 			try {
 				existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey)
 			} catch (e) {
 				// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
-				existing = fn([], guild)
+				existing = fn([])
 			}
-			if (isDeepStrictEqual(existing, content)) return
+			if (deepEqual(existing, content, {strict: true})) return
 		}
-		await api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content)
+		api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content)
 	}
 
-	await update(spaceID, "emojis", "moe.cadence.ooye.pack.emojis", expression.emojisToState)
-	await update(spaceID, "stickers", "moe.cadence.ooye.pack.stickers", expression.stickersToState)
+	update(spaceID, "emojis", "moe.cadence.ooye.pack.emojis", expression.emojisToState)
+	update(spaceID, "stickers", "moe.cadence.ooye.pack.stickers", expression.stickersToState)
 }
 
 module.exports.createSpace = createSpace

d2m/actions/delete-message.js

@@ -14,13 +14,12 @@ async function deleteMessage(data) {
 	const row = select("channel_room", ["room_id", "speedbump_checked", "thread_parent"], {channel_id: data.channel_id}).get()
 	if (!row) return
 
-	// Assume we can redact from tombstoned rooms.
-	const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
-		.select("event_id", "room_id").where({message_id: data.id}).all()
-	db.prepare("DELETE FROM message_room WHERE message_id = ?").run(data.id)
-	for (const {event_id, room_id} of eventsToRedact) {
+	const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all()
+	db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id)
+	db.prepare("DELETE FROM event_message WHERE message_id = ?").run(data.id)
+	for (const eventID of eventsToRedact) {
 		// Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs
-		await api.redactEvent(room_id, event_id)
+		await api.redactEvent(row.room_id, eventID)
 	}
 
 	await speedbump.updateCache(row.thread_parent || data.channel_id, row.speedbump_checked)
@@ -30,17 +29,16 @@ async function deleteMessage(data) {
  * @param {import("discord-api-types/v10").GatewayMessageDeleteBulkDispatchData} data
  */
 async function deleteMessageBulk(data) {
-	const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get()
-	if (!row) return
+	const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
+	if (!roomID) return
 
 	const sids = JSON.stringify(data.ids)
-	// Assume we can redact from tombstoned rooms.
-	const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
-		.select("event_id", "room_id").and("WHERE message_id IN (SELECT value FROM json_each(?))").all(sids)
-	db.prepare("DELETE FROM message_room WHERE message_id IN (SELECT value FROM json_each(?))").run(sids)
-	for (const {event_id, room_id} of eventsToRedact) {
+	const eventsToRedact = from("event_message").pluck("event_id").and("WHERE message_id IN (SELECT value FROM json_each(?)").all(sids)
+	db.prepare("DELETE FROM message_channel WHERE message_id IN (SELECT value FROM json_each(?)").run(sids)
+	db.prepare("DELETE FROM event_message WHERE message_id IN (SELECT value FROM json_each(?)").run(sids)
+	for (const eventID of eventsToRedact) {
 		// Awaiting will make it go slower, but since this could be a long-running operation either way, we want to leave rate limit capacity for other operations
-		await api.redactEvent(room_id, event_id)
+		await api.redactEvent(roomID, eventID)
 	}
 }
 

d2m/actions/edit-message.js

@@ -3,15 +3,13 @@
 const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
-const {sync, db, select, from} = passthrough
+const {sync, db, select} = passthrough
 /** @type {import("../converters/edit-to-changes")} */
 const editToChanges = sync.require("../converters/edit-to-changes")
 /** @type {import("./register-pk-user")} */
 const registerPkUser = sync.require("./register-pk-user")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
-/** @type {import("../../matrix/mreq")} */
-const mreq = sync.require("../../matrix/mreq")
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
@@ -19,18 +17,14 @@ const mreq = sync.require("../../matrix/mreq")
  * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
 async function editMessage(message, guild, row) {
-	const historicalRoomOfMessage = from("message_room").join("historical_channel_room", "historical_room_index").where({message_id: message.id}).select("room_id").get()
-	const currentRoom = from("channel_room").join("historical_channel_room", "room_id").where({channel_id: message.channel_id}).select("room_id", "historical_room_index").get()
-	if (!currentRoom) return
-
-	if (historicalRoomOfMessage && historicalRoomOfMessage.room_id !== currentRoom.room_id) return // tombstoned rooms should not have new events (including edits) sent to them
-
 	let {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api)
 
 	if (row && row.speedbump_webhook_id === message.webhook_id) {
 		// Handle the PluralKit public instance
 		if (row.speedbump_id === "466378653216014359") {
-			senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, true)
+			const root = await registerPkUser.fetchMessage(message.id)
+			assert(root.member)
+			senderMxid = await registerPkUser.ensureSimJoined(root, roomID)
 		}
 	}
 
@@ -59,7 +53,7 @@ async function editMessage(message, guild, row) {
 	const sendNewEventParts = new Set()
 	for (const promotion of promotions) {
 		if ("eventID" in promotion) {
-			db.prepare(`UPDATE event_message SET ${promotion.column} = ? WHERE event_id = ?`).run(promotion.value ?? 0, promotion.eventID)
+			db.prepare(`UPDATE event_message SET ${promotion.column} = 0 WHERE event_id = ?`).run(promotion.eventID)
 		} else if ("nextEvent" in promotion) {
 			sendNewEventParts.add(promotion.column)
 		}
@@ -67,7 +61,7 @@ async function editMessage(message, guild, row) {
 
 	// 4. Send all the things.
 	if (eventsToSend.length) {
-		db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(message.id, currentRoom.historical_room_index)
+		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
 	}
 	for (const content of eventsToSend) {
 		const eventType = content.$type
@@ -78,17 +72,8 @@ async function editMessage(message, guild, row) {
 
 		const part = sendNewEventParts.has("part") && eventsToSend[0] === content ? 0 : 1
 		const reactionPart = sendNewEventParts.has("reaction_part") && eventsToSend[eventsToSend.length - 1] === content ? 0 : 1
-
-		try {
-			const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid)
-			db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord
-		} catch (e) {
-			if (e instanceof mreq.MatrixServerError && e.errcode === "M_FORBIDDEN") {
-				// sending user doesn't have permission to update message, e.g. because Discord generated an embed in a read-only room
-			} else {
-				throw e
-			}
-		}
+		const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid)
+		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord
 	}
 }
 

d2m/actions/expression.js

@@ -9,12 +9,11 @@ const file = sync.require("../../matrix/file")
 
 /**
  * @param {DiscordTypes.APIEmoji[]} emojis
- * @param {DiscordTypes.APIGuild} guild
  */
-async function emojisToState(emojis, guild) {
+async function emojisToState(emojis) {
 	const result = {
 		pack: {
-			display_name: `${guild.name} (Discord Emojis)`,
+			display_name: "Discord Emojis",
 			usage: ["emoticon"] // we'll see...
 		},
 		images: {
@@ -25,13 +24,13 @@ async function emojisToState(emojis, guild) {
 		file.uploadDiscordFileToMxc(file.emoji(emoji.id, emoji.animated)).then(url => {
 			result.images[emoji.name] = {
 				info: {
-					mimetype: "image/webp"
+					mimetype: emoji.animated ? "image/gif" : "image/png"
 				},
 				url
 			}
 			db.prepare("INSERT OR IGNORE INTO emoji (emoji_id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(emoji.id, emoji.name, +!!emoji.animated, url)
 		}).catch(e => {
-			if (e.data?.errcode === "M_TOO_LARGE") { // Very unlikely to happen. Only possible for 3x-series emojis uploaded shortly after animated emojis were introduced, when there was no 256 KB size limit.
+			if (e.data.errcode === "M_TOO_LARGE") { // Very unlikely to happen. Only possible for 3x-series emojis uploaded shortly after animated emojis were introduced, when there was no 256 KB size limit.
 				return
 			}
 			console.error(`Trying to handle emoji ${emoji.name} (${emoji.id}), but...`)
@@ -43,12 +42,11 @@ async function emojisToState(emojis, guild) {
 
 /**
  * @param {DiscordTypes.APISticker[]} stickers
- * @param {DiscordTypes.APIGuild} guild
  */
-async function stickersToState(stickers, guild) {
+async function stickersToState(stickers) {
 	const result = {
 		pack: {
-			display_name: `${guild.name} (Discord Stickers)`,
+			display_name: "Discord Stickers",
 			usage: ["sticker"] // we'll see...
 		},
 		images: {
@@ -68,7 +66,7 @@ async function stickersToState(stickers, guild) {
 			while (shortcodes.includes(shortcode)) shortcode = shortcode + "~"
 			shortcodes.push(shortcode)
 
-			result.images[shortcode] = {
+			result.images[shortcodes] = {
 				info: {
 					mimetype: file.stickerFormat.get(sticker.format_type)?.mime || "image/png"
 				},

d2m/actions/lottie.js

@@ -33,7 +33,7 @@ async function convert(stickerItem) {
 	if (res.status !== 200) throw new Error("Sticker data file not found.")
 	const text = await res.text()
 
-	// Convert to PNG (stream.Readable)
+	// Convert to PNG (readable stream)
 	const readablePng = await convertLottie.convert(text)
 
 	// Upload to MXC

d2m/actions/register-pk-user.js

@@ -1,11 +1,12 @@
 // @ts-check
 
 const assert = require("assert")
-const {reg} = require("../../matrix/read-registration")
+const reg = require("../../matrix/read-registration")
 const Ty = require("../../types")
+const fetch = require("node-fetch").default
 
 const passthrough = require("../../passthrough")
-const {sync, db, select, from} = passthrough
+const {discord, sync, db, select} = passthrough
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
@@ -13,20 +14,12 @@ const file = sync.require("../../matrix/file")
 /** @type {import("./register-user")} */
 const registerUser = sync.require("./register-user")
 
-/** @returns {Promise<Ty.PkMessage>} */
-async function fetchMessage(messageID) {
-	try {
-		var res = await fetch(`https://api.pluralkit.me/v2/messages/${messageID}`)
-	} catch (networkError) {
-		// Network issue, raise a more readable message
-		throw new Error(`Failed to connect to PK API: ${networkError.toString()}`)
-	}
-	if (!res.ok) throw new Error(`PK API returned an error: ${await res.text()}`)
-	/** @type {any} */
-	const root = await res.json()
-	if (!root.member) throw new Error(`PK API didn't return member data: ${JSON.stringify(root)}`)
-	return root
-}
+/**
+ * @typedef WebhookAuthor Discord API message->author. A webhook as an author.
+ * @prop {string} username
+ * @prop {string?} avatar
+ * @prop {string} id
+ */
 
 /**
  * A sim is an account that is being simulated by the bridge to copy events from the other side.
@@ -40,7 +33,7 @@ async function createSim(pkMessage) {
 	const mxid = `@${localpart}:${reg.ooye.server_name}`
 
 	// Save chosen name in the database forever
-	db.prepare("INSERT INTO sim (user_id, username, sim_name, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, simName, mxid)
+	db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, localpart, mxid)
 
 	// Register matrix user with that name
 	try {
@@ -87,17 +80,24 @@ async function ensureSimJoined(pkMessage, roomID) {
 	// Ensure joined
 	const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get()
 	if (!existing) {
-		await api.inviteToRoom(roomID, mxid)
-		await api.joinRoom(roomID, mxid)
+		try {
+			await api.inviteToRoom(roomID, mxid)
+			await api.joinRoom(roomID, mxid)
+		} catch (e) {
+			if (e.message.includes("is already in the room.")) {
+				// Sweet!
+			} else {
+				throw e
+			}
+		}
 		db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid)
 	}
 	return mxid
 }
 
 /**
- * Generate profile data based on webhook displayname and configured avatar.
  * @param {Ty.PkMessage} pkMessage
- * @param {Ty.WebhookAuthor} author
+ * @param {WebhookAuthor} author
  */
 async function memberToStateContent(pkMessage, author) {
 	// We prefer to use the member's avatar URL data since the image upload can be cached across channels,
@@ -116,40 +116,38 @@ async function memberToStateContent(pkMessage, author) {
 
 /**
  * Sync profile data for a sim user. This function follows the following process:
- * 1. Look up data about proxy user from API
- * 2. If this fails, try to use previously cached data (won't sync)
- * 3. Create and join the sim to the room if needed
- * 4. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
- * 5. Compare against the previously known state content, which is helpfully stored in the database
- * 6. If the state content has changed, send it to Matrix and update it in the database for next time
- * @param {string} messageID to call API with
- * @param {Ty.WebhookAuthor} author for profile data
- * @param {string} roomID room to join member to
- * @param {boolean} shouldActuallySync whether to actually sync updated user data or just ensure it's joined
+ * 1. Join the sim to the room if needed
+ * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
+ * 3. Compare against the previously known state content, which is helpfully stored in the database
+ * 4. If the state content has changed, send it to Matrix and update it in the database for next time
+ * @param {WebhookAuthor} author
+ * @param {Ty.PkMessage} pkMessage
+ * @param {string} roomID
  * @returns {Promise<string>} mxid of the updated sim
  */
-async function syncUser(messageID, author, roomID, shouldActuallySync) {
-	try {
-		// API lookup
-		var pkMessage = await fetchMessage(messageID)
-		db.prepare("REPLACE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username)
-	} catch (e) {
-		// Fall back to offline cache
-		const senderMxid = from("sim_proxy").join("sim", "user_id").join("sim_member", "mxid").where({displayname: author.username, room_id: roomID}).pluck("mxid").get()
-		if (!senderMxid) throw e
-		return senderMxid
-	}
-
-	// Create and join the sim to the room if needed
+async function syncUser(author, pkMessage, roomID) {
 	const mxid = await ensureSimJoined(pkMessage, roomID)
-
-	if (shouldActuallySync) {
-		// Build current profile data and sync if the hash has changed
-		const content = await memberToStateContent(pkMessage, author)
-		await registerUser._sendSyncUser(roomID, mxid, content, null)
+	// Update the sim_proxy table, so mentions can look up the original sender later
+	db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username)
+	// Sync the member state
+	const content = await memberToStateContent(pkMessage, author)
+	const currentHash = registerUser._hashProfileContent(content)
+	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
+	// only do the actual sync if the hash has changed since we last looked
+	if (existingHash !== currentHash) {
+		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
 	}
-
 	return mxid
 }
 
+/** @returns {Promise<Ty.PkMessage>} */
+function fetchMessage(messageID) {
+	return fetch(`https://api.pluralkit.me/v2/messages/${messageID}`).then(res => res.json())
+}
+
+module.exports._memberToStateContent = memberToStateContent
+module.exports.ensureSim = ensureSim
+module.exports.ensureSimJoined = ensureSimJoined
 module.exports.syncUser = syncUser
+module.exports.fetchMessage = fetchMessage

d2m/actions/register-user.js

@@ -0,0 +1,186 @@
+// @ts-check
+
+const assert = require("assert")
+const reg = require("../../matrix/read-registration")
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select} = passthrough
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("../converters/user-to-mxid")} */
+const userToMxid = sync.require("../converters/user-to-mxid")
+/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
+let hasher = null
+// @ts-ignore
+require("xxhash-wasm")().then(h => hasher = h)
+
+/**
+ * A sim is an account that is being simulated by the bridge to copy events from the other side.
+ * @param {import("discord-api-types/v10").APIUser} user
+ * @returns mxid
+ */
+async function createSim(user) {
+	// Choose sim name
+	const simName = userToMxid.userToSimName(user)
+	const localpart = reg.ooye.namespace_prefix + simName
+	const mxid = `@${localpart}:${reg.ooye.server_name}`
+
+	// Save chosen name in the database forever
+	// Making this database change right away so that in a concurrent registration, the 2nd registration will already have generated a different localpart because it can see this row when it generates
+	db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(user.id, simName, localpart, mxid)
+
+	// Register matrix user with that name
+	try {
+		await api.register(localpart)
+	} catch (e) {
+		// If user creation fails, manually undo the database change. Still isn't perfect, but should help.
+		// (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.)
+		db.prepare("DELETE FROM sim WHERE user_id = ?").run(user.id)
+		throw e
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user.
+ * If there is already a sim, use that one. If there isn't one yet, register a new sim.
+ * @param {import("discord-api-types/v10").APIUser} user
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSim(user) {
+	let mxid = null
+	const existing = select("sim", "mxid", {user_id: user.id}).pluck().get()
+	if (existing) {
+		mxid = existing
+	} else {
+		mxid = await createSim(user)
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user and is joined to the room.
+ * @param {import("discord-api-types/v10").APIUser} user
+ * @param {string} roomID
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSimJoined(user, roomID) {
+	// Ensure room ID is really an ID, not an alias
+	assert.ok(roomID[0] === "!")
+
+	// Ensure user
+	const mxid = await ensureSim(user)
+
+	// Ensure joined
+	const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get()
+	if (!existing) {
+		try {
+			await api.inviteToRoom(roomID, mxid)
+			await api.joinRoom(roomID, mxid)
+		} catch (e) {
+			if (e.message.includes("is already in the room.")) {
+				// Sweet!
+			} else {
+				throw e
+			}
+		}
+		db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid)
+	}
+	return mxid
+}
+
+/**
+ * @param {import("discord-api-types/v10").APIUser} user
+ * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ */
+async function memberToStateContent(user, member, guildID) {
+	let displayname = user.username
+	if (user.global_name) displayname = user.global_name
+	if (member.nick) displayname = member.nick
+
+	const content = {
+		displayname,
+		membership: "join",
+		"moe.cadence.ooye.member": {
+		},
+		"uk.half-shot.discord.member": {
+			bot: !!user.bot,
+			displayColor: user.accent_color,
+			id: user.id,
+			username: user.discriminator.length === 4 ? `${user.username}#${user.discriminator}` : `@${user.username}`
+		}
+	}
+
+	if (member.avatar || user.avatar) {
+		// const avatarPath = file.userAvatar(user) // the user avatar only
+		const avatarPath = file.memberAvatar(guildID, user, member) // the member avatar or the user avatar
+		content["moe.cadence.ooye.member"].avatar = avatarPath
+		content.avatar_url = await file.uploadDiscordFileToMxc(avatarPath)
+	}
+
+	return content
+}
+
+function _hashProfileContent(content) {
+	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
+	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
+	return signedHash
+}
+
+/**
+ * Sync profile data for a sim user. This function follows the following process:
+ * 1. Join the sim to the room if needed
+ * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
+ * 3. Compare against the previously known state content, which is helpfully stored in the database
+ * 4. If the state content has changed, send it to Matrix and update it in the database for next time
+ * @param {import("discord-api-types/v10").APIUser} user
+ * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ * @returns {Promise<string>} mxid of the updated sim
+ */
+async function syncUser(user, member, guildID, roomID) {
+	const mxid = await ensureSimJoined(user, roomID)
+	const content = await memberToStateContent(user, member, guildID)
+	const currentHash = _hashProfileContent(content)
+	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
+	// only do the actual sync if the hash has changed since we last looked
+	if (existingHash !== currentHash) {
+		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
+	}
+	return mxid
+}
+
+async function syncAllUsersInRoom(roomID) {
+	const mxids = select("sim_member", "mxid", {room_id: roomID}).pluck().all()
+
+	const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
+	assert.ok(typeof channelID === "string")
+
+	/** @ts-ignore @type {import("discord-api-types/v10").APIGuildChannel} */
+	const channel = discord.channels.get(channelID)
+	const guildID = channel.guild_id
+	assert.ok(typeof guildID === "string")
+
+	for (const mxid of mxids) {
+		const userID = select("sim", "user_id", {mxid}).pluck().get()
+		assert.ok(typeof userID === "string")
+
+		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIGuildMember>} */
+		const member = await discord.snow.guild.getGuildMember(guildID, userID)
+		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIUser>} user */
+		const user = member.user
+		assert.ok(user)
+
+		console.log(`[user sync] to matrix: ${user.username} in ${channel.name}`)
+		await syncUser(user, member, guildID, roomID)
+	}
+}
+
+module.exports._memberToStateContent = memberToStateContent
+module.exports._hashProfileContent = _hashProfileContent
+module.exports.ensureSim = ensureSim
+module.exports.ensureSimJoined = ensureSimJoined
+module.exports.syncUser = syncUser
+module.exports.syncAllUsersInRoom = syncAllUsersInRoom

d2m/actions/register-user.test.js

@@ -0,0 +1,63 @@
+const {_memberToStateContent} = require("./register-user")
+const {test} = require("supertape")
+const testData = require("../../test/data")
+
+test("member2state: without member nick or avatar", async t => {
+	t.deepEqual(
+		await _memberToStateContent(testData.member.kumaccino.user, testData.member.kumaccino, testData.guild.general.id),
+		{
+			avatar_url: "mxc://cadence.moe/UpAeIqeclhKfeiZNdIWNcXXL",
+			displayname: "kumaccino",
+			membership: "join",
+			"moe.cadence.ooye.member": {
+				avatar: "/avatars/113340068197859328/b48302623a12bc7c59a71328f72ccb39.png?size=1024"
+			},
+			"uk.half-shot.discord.member": {
+				bot: false,
+				displayColor: 10206929,
+				id: "113340068197859328",
+				username: "@kumaccino"
+			}
+		}
+	)
+})
+
+test("member2state: with global name, without member nick or avatar", async t => {
+	t.deepEqual(
+		await _memberToStateContent(testData.member.papiophidian.user, testData.member.papiophidian, testData.guild.general.id),
+		{
+			avatar_url: "mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX",
+			displayname: "PapiOphidian",
+			membership: "join",
+			"moe.cadence.ooye.member": {
+				avatar: "/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024"
+			},
+			"uk.half-shot.discord.member": {
+				bot: false,
+				displayColor: 1579292,
+				id: "320067006521147393",
+				username: "@papiophidian"
+			}
+		}
+	)
+})
+
+test("member2state: with member nick and avatar", async t => {
+	t.deepEqual(
+		await _memberToStateContent(testData.member.sheep.user, testData.member.sheep, testData.guild.general.id),
+		{
+			avatar_url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl",
+			displayname: "The Expert's Submarine",
+			membership: "join",
+			"moe.cadence.ooye.member": {
+				avatar: "/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"
+			},
+			"uk.half-shot.discord.member": {
+				bot: false,
+				displayColor: null,
+				id: "134826546694193153",
+				username: "@aprilsong"
+			}
+		}
+	)
+})

d2m/actions/remove-reaction.js

@@ -4,7 +4,7 @@ const Ty = require("../../types")
 const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, from, select} = passthrough
+const {discord, sync, db, select} = passthrough
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 /** @type {import("../converters/emoji-to-key")} */
@@ -18,15 +18,21 @@ const converter = sync.require("../converters/remove-reaction")
  * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData | DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData | DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
  */
 async function removeSomeReactions(data) {
-	const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get()
-	if (!row) return
+	const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
+	if (!roomID) return
+	const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get()
+	if (!eventIDForMessage) return
 
-	const eventReactedTo = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
-		.where({message_id: data.message_id}).and("ORDER BY reaction_part").select("event_id", "room_id").get()
-	if (!eventReactedTo) return
-
-	// Due to server restrictions, all relations (i.e. reactions) have to be in the same room as the original event.
-	const reactions = await api.getFullRelations(eventReactedTo.room_id, eventReactedTo.event_id, "m.annotation")
+	/** @type {Ty.Event.Outer<Ty.Event.M_Reaction>[]} */
+	let reactions = []
+	/** @type {string | undefined} */
+	let nextBatch = undefined
+	do {
+		/** @type {Ty.Pagination<Ty.Event.Outer<Ty.Event.M_Reaction>>} */
+		const res = await api.getRelations(roomID, eventIDForMessage, {from: nextBatch}, "m.annotation")
+		reactions = reactions.concat(res.chunk)
+		nextBatch = res.next_batch
+	} while (nextBatch)
 
 	// Run the proper strategy and any strategy-specific database changes
 	const removals = await
@@ -36,7 +42,7 @@ async function removeSomeReactions(data) {
 
 	// Redact the events and delete individual stored events in the database
 	for (const removal of removals) {
-		await api.redactEvent(eventReactedTo.room_id, removal.eventID, removal.mxid)
+		await api.redactEvent(roomID, removal.eventID, removal.mxid)
 		if (removal.hash) db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(removal.hash)
 	}
 }
@@ -46,7 +52,7 @@ async function removeSomeReactions(data) {
  * @param {Ty.Event.Outer<Ty.Event.M_Reaction>[]} reactions
  */
 async function removeReaction(data, reactions) {
-	const key = await emojiToKey.emojiToKey(data.emoji, data.message_id)
+	const key = await emojiToKey.emojiToKey(data.emoji)
 	return converter.removeReaction(data, reactions, key)
 }
 
@@ -55,8 +61,8 @@ async function removeReaction(data, reactions) {
  * @param {Ty.Event.Outer<Ty.Event.M_Reaction>[]} reactions
  */
 async function removeEmojiReaction(data, reactions) {
-	const key = await emojiToKey.emojiToKey(data.emoji, data.message_id)
-	const discordPreferredEncoding = await emoji.encodeEmoji(key, undefined)
+	const key = await emojiToKey.emojiToKey(data.emoji)
+	const discordPreferredEncoding = emoji.encodeEmoji(key, undefined)
 	db.prepare("DELETE FROM reaction WHERE message_id = ? AND encoded_emoji = ?").run(data.message_id, discordPreferredEncoding)
 
 	return converter.removeEmojiReaction(data, reactions, key)

d2m/actions/send-message.js

@@ -0,0 +1,83 @@
+// @ts-check
+
+const assert = require("assert")
+
+const passthrough = require("../../passthrough")
+const { discord, sync, db } = passthrough
+/** @type {import("../converters/message-to-event")} */
+const messageToEvent = sync.require("../converters/message-to-event")
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+/** @type {import("./register-user")} */
+const registerUser = sync.require("./register-user")
+/** @type {import("./register-pk-user")} */
+const registerPkUser = sync.require("./register-pk-user")
+/** @type {import("../actions/create-room")} */
+const createRoom = sync.require("../actions/create-room")
+/** @type {import("../../discord/utils")} */
+const dUtils = sync.require("../../discord/utils")
+
+/**
+ * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
+ * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
+ */
+async function sendMessage(message, guild, row) {
+	const roomID = await createRoom.ensureRoom(message.channel_id)
+
+	let senderMxid = null
+	if (!dUtils.isWebhookMessage(message)) {
+		if (message.member) { // available on a gateway message create event
+			senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID)
+		} else { // well, good enough...
+			senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
+		}
+	} else if (row && row.speedbump_webhook_id === message.webhook_id) {
+		// Handle the PluralKit public instance
+		if (row.speedbump_id === "466378653216014359") {
+			const root = await registerPkUser.fetchMessage(message.id)
+			// Member is null if member was deleted. We just got this message, so member surely exists.
+			if (!root.member) {
+				const e = new Error("PK API did not return a member")
+				message["__pk_response__"] = root
+				console.error(root)
+				throw e
+			}
+			senderMxid = await registerPkUser.syncUser(message.author, root, roomID)
+		}
+	}
+
+	const events = await messageToEvent.messageToEvent(message, guild, {}, {api})
+	const eventIDs = []
+	if (events.length) {
+		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
+		if (senderMxid) api.sendTyping(roomID, false, senderMxid)
+	}
+	for (const event of events) {
+		const part = event === events[0] ? 0 : 1
+		const reactionPart = event === events[events.length - 1] ? 0 : 1
+
+		const eventType = event.$type
+		if ("$sender" in event) senderMxid = event.$sender
+		/** @type {Pick<typeof event, Exclude<keyof event, "$type" | "$sender">> & { $type?: string, $sender?: string }} */
+		const eventWithoutType = {...event}
+		delete eventWithoutType.$type
+		delete eventWithoutType.$sender
+
+		const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined
+		const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp)
+		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, part, reactionPart) // source 1 = discord
+
+		// The primary event is part = 0 and has the most important and distinct information. It is used to provide reply previews, be pinned, and possibly future uses.
+		// The first event is chosen to be the primary part because it is usually the message text content and is more likely to be distinct.
+		// For example, "Reply to 'this meme made me think of you'" is more useful than "Replied to image".
+
+		// The last event gets reaction_part = 0. Reactions are managed there because reactions are supposed to appear at the bottom.
+
+		eventIDs.push(eventID)
+	}
+
+	return eventIDs
+}
+
+module.exports.sendMessage = sendMessage

d2m/actions/speedbump.js

@@ -4,14 +4,6 @@ const DiscordTypes = require("discord-api-types/v10")
 const passthrough = require("../../passthrough")
 const {discord, select, db} = passthrough
 
-const DEBUG_SPEEDBUMP = false
-
-function debugSpeedbump(message) {
-	if (DEBUG_SPEEDBUMP) {
-		console.log(message)
-	}
-}
-
 const SPEEDBUMP_SPEED = 4000 // 4 seconds delay
 const SPEEDBUMP_UPDATE_FREQUENCY = 2 * 60 * 60 // 2 hours
 
@@ -35,8 +27,8 @@ async function updateCache(channelID, lastChecked) {
 	db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_webhook_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(foundApplication, foundWebhook, now, channelID)
 }
 
-/** @type {Map<string, number>} messageID -> number of gateway events currently bumping */
-const bumping = new Map()
+/** @type {Set<string>} set of messageID */
+const bumping = new Set()
 
 /**
  * Slow down a message. After it passes the speedbump, return whether it's okay or if it's been deleted.
@@ -44,26 +36,9 @@ const bumping = new Map()
  * @returns whether it was deleted
  */
 async function doSpeedbump(messageID) {
-	let value = (bumping.get(messageID) ?? 0) + 1
-	bumping.set(messageID, value)
-	debugSpeedbump(`[speedbump] WAIT ${messageID}++ = ${value}`)
-
+	bumping.add(messageID)
 	await new Promise(resolve => setTimeout(resolve, SPEEDBUMP_SPEED))
-
-	if (!bumping.has(messageID)) {
-		debugSpeedbump(`[speedbump] DELETED ${messageID}`)
-		return true
-	}
-	value = (bumping.get(messageID) ?? 0) - 1
-	if (value <= 0) {
-		debugSpeedbump(`[speedbump] OK ${messageID}-- = ${value}`)
-		bumping.delete(messageID)
-		return false
-	} else {
-		debugSpeedbump(`[speedbump] MULTI ${messageID}-- = ${value}`)
-		bumping.set(messageID, value)
-		return true
-	}
+	return !bumping.delete(messageID)
 }
 
 /**

d2m/actions/update-pins.js

@@ -6,8 +6,6 @@ const {discord, sync, db} = passthrough
 const pinsToList = sync.require("../converters/pins-to-list")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
-/** @type {import("../../matrix/kstate")} */
-const ks = sync.require("../../matrix/kstate")
 
 /**
  * @template {string | null | undefined} T
@@ -25,21 +23,13 @@ function convertTimestamp(timestamp) {
  * @param {number?} convertedTimestamp
  */
 async function updatePins(channelID, roomID, convertedTimestamp) {
-	try {
-		var discordPins = await discord.snow.channel.getChannelPinnedMessages(channelID)
-	} catch (e) {
-		if (e.message === `{"message": "Missing Access", "code": 50001}`) {
-			return // Discord sends channel pins update events even for channels that the bot can't view/get pins in, just ignore it
-		}
-		throw e
+	const pins = await discord.snow.channel.getChannelPinnedMessages(channelID)
+	const eventIDs = pinsToList.pinsToList(pins)
+	if (pins.length === eventIDs.length || eventIDs.length) {
+		await api.sendState(roomID, "m.room.pinned_events", "", {
+			pinned: eventIDs
+		})
 	}
-
-	const kstate = await ks.roomToKState(roomID, [["m.room.pinned_events", ""]])
-	const pinned = pinsToList.pinsToList(discordPins, kstate)
-
-	const diff = ks.diffKState(kstate, {"m.room.pinned_events/": {pinned}})
-	await ks.applyKStateDiffToRoom(roomID, diff)
-
 	db.prepare("UPDATE channel_room SET last_bridged_pin_timestamp = ? WHERE channel_id = ?").run(convertedTimestamp || 0, channelID)
 }
 

d2m/converters/edit-to-changes.js

@@ -0,0 +1,157 @@
+// @ts-check
+
+const assert = require("assert").strict
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select, from} = passthrough
+/** @type {import("./message-to-event")} */
+const messageToEvent = sync.require("../converters/message-to-event")
+/** @type {import("../actions/register-user")} */
+const registerUser = sync.require("../actions/register-user")
+/** @type {import("../actions/create-room")} */
+const createRoom = sync.require("../actions/create-room")
+
+/**
+ * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
+ * IMPORTANT: This may not have all the normal fields! The API documentation doesn't provide possible types, just says it's all optional!
+ * Since I don't have a spec, I will have to capture some real traffic and add it as test cases... I hope they don't change anything later...
+ * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {import("../../matrix/api")} api simple-as-nails dependency injection for the matrix API
+ */
+async function editToChanges(message, guild, api) {
+	// Figure out what events we will be replacing
+
+	const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
+	assert(roomID)
+	/** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */
+	const senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
+
+	const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all()
+
+	// Figure out what we will be replacing them with
+
+	const newFallbackContent = await messageToEvent.messageToEvent(message, guild, {includeEditFallbackStar: true}, {api})
+	const newInnerContent = await messageToEvent.messageToEvent(message, guild, {includeReplyFallback: false}, {api})
+	assert.ok(newFallbackContent.length === newInnerContent.length)
+
+	// Match the new events to the old events
+
+	/*
+		Rules:
+			+ The events must have the same type.
+			+ The events must have the same subtype.
+		Events will therefore be divided into four categories:
+	*/
+	/** 1. Events that are matched, and should be edited by sending another m.replace event */
+	let eventsToReplace = []
+	/** 2. Events that are present in the old version only, and should be blanked or redacted */
+	let eventsToRedact = []
+	/** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */
+	let eventsToSend = []
+	//  4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. This is represented as nothing.
+
+	function shift() {
+		newFallbackContent.shift()
+		newInnerContent.shift()
+	}
+
+	// For each old event...
+	outer: while (newFallbackContent.length) {
+		const newe = newFallbackContent[0]
+		// Find a new event to pair it with...
+		for (let i = 0; i < oldEventRows.length; i++) {
+			const olde = oldEventRows[i]
+			if (olde.event_type === newe.$type && olde.event_subtype === (newe.msgtype || null)) { // The spec does allow subtypes to change, so I can change this condition later if I want to
+				// Found one!
+				// Set up the pairing
+				eventsToReplace.push({
+					old: olde,
+					newFallbackContent: newFallbackContent[0],
+					newInnerContent: newInnerContent[0]
+				})
+				// These events have been handled now, so remove them from the source arrays
+				shift()
+				oldEventRows.splice(i, 1)
+				// Go all the way back to the start of the next iteration of the outer loop
+				continue outer
+			}
+		}
+		// If we got this far, we could not pair it to an existing event, so it'll have to be a new one
+		eventsToSend.push(newInnerContent[0])
+		shift()
+	}
+	// Anything remaining in oldEventRows is present in the old version only and should be redacted.
+	eventsToRedact = oldEventRows
+
+	// We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times.
+	/** @type {({column: string, eventID: string} | {column: string, nextEvent: true})[]} */
+	const promotions = []
+	for (const column of ["part", "reaction_part"]) {
+		// If no events with part = 0 exist (or will exist), we need to do some management.
+		if (!eventsToReplace.some(e => e.old[column] === 0)) {
+			if (eventsToReplace.length) {
+				// We can choose an existing event to promote. Bigger order is better.
+				const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text")
+				eventsToReplace.sort((a, b) => order(b) - order(a))
+				if (column === "part") {
+					promotions.push({column, eventID: eventsToReplace[0].old.event_id}) // part should be the first one
+				} else {
+					promotions.push({column, eventID: eventsToReplace[eventsToReplace.length - 1].old.event_id}) // reaction_part should be the last one
+				}
+			} else {
+				// No existing events to promote, but new events are being sent. Whatever gets sent will be the next part = 0.
+				promotions.push({column, nextEvent: true})
+			}
+		}
+	}
+
+	// Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed!
+	// (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.)
+	// So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed.
+	eventsToReplace = eventsToReplace.filter(ev => {
+		// Discord does not allow files, images, attachments, or videos to be edited.
+		if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
+			return false
+		}
+		// Discord does not allow stickers to be edited.
+		if (ev.old.event_type === "m.sticker") {
+			return false
+		}
+		// Anything else is fair game.
+		return true
+	})
+
+	// Removing unnecessary properties before returning
+	eventsToRedact = eventsToRedact.map(e => e.event_id)
+	eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)}))
+
+	return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions}
+}
+
+/**
+ * @template T
+ * @param {string} oldID
+ * @param {T} newFallbackContent
+ * @param {T} newInnerContent
+ * @returns {import("../../types").Event.ReplacementContent<T>} content
+ */
+function makeReplacementEventContent(oldID, newFallbackContent, newInnerContent) {
+	const content = {
+		...newFallbackContent,
+		"m.mentions": {},
+		"m.new_content": {
+			...newInnerContent
+		},
+		"m.relates_to": {
+			rel_type: "m.replace",
+			event_id: oldID
+		}
+	}
+	delete content["m.new_content"]["$type"]
+	// Client-Server API spec 11.37.3: Any m.relates_to property within m.new_content is ignored.
+	delete content["m.new_content"]["m.relates_to"]
+	return content
+}
+
+module.exports.editToChanges = editToChanges
+module.exports.makeReplacementEventContent = makeReplacementEventContent

d2m/converters/edit-to-changes.test.js

@@ -0,0 +1,237 @@
+const {test} = require("supertape")
+const {editToChanges} = require("./edit-to-changes")
+const data = require("../../test/data")
+const Ty = require("../../types")
+
+test("edit2changes: edit by webhook", async t => {
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edit_by_webhook, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* test 2",
+			"m.mentions": {},
+			"m.new_content": {
+				// *** Replaced With: ***
+				msgtype: "m.text",
+				body: "test 2",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10"
+			}
+		}
+	}])
+	t.equal(senderMxid, null)
+	t.deepEqual(promotions, [])
+})
+
+test("edit2changes: bot response", async t => {
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.bot_response, data.guild.general, {
+		async getJoinedMembers(roomID) {
+			t.equal(roomID, "!hYnGGlPHlbujVVfktC:cadence.moe")
+			return new Promise(resolve => {
+				setTimeout(() => {
+					resolve({
+						joined: {
+							"@cadence:cadence.moe": {
+								displayname: "cadence [they]",
+								avatar_url: "whatever"
+							},
+							"@_ooye_botrac4r:cadence.moe": {
+								displayname: "botrac4r",
+								avatar_url: "whatever"
+							}
+						}
+					})
+				})
+			})
+		}
+	})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* :ae_botrac4r: @cadence asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.",
+			format: "org.matrix.custom.html",
+			formatted_body: '* <img data-mx-emoticon height="32" src="mxc://cadence.moe/skqfuItqxNmBYekzmVKyoLzs" title=":ae_botrac4r:" alt=":ae_botrac4r:"> @cadence asked <code>­</code>, I respond: Stop drinking paint. (No)<br><br>Hit <img data-mx-emoticon height="32" src="mxc://cadence.moe/OIpqpfxTnHKokcsYqDusxkBT" title=":bn_re:" alt=":bn_re:"> to reroll.',
+			"m.mentions": {
+				// Client-Server API spec 11.37.7: Copy Discord's behaviour by not re-notifying anyone that an *edit occurred*
+			},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: ":ae_botrac4r: @cadence asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.",
+				format: "org.matrix.custom.html",
+				formatted_body: '<img data-mx-emoticon height="32" src="mxc://cadence.moe/skqfuItqxNmBYekzmVKyoLzs" title=":ae_botrac4r:" alt=":ae_botrac4r:"> @cadence asked <code>­</code>, I respond: Stop drinking paint. (No)<br><br>Hit <img data-mx-emoticon height="32" src="mxc://cadence.moe/OIpqpfxTnHKokcsYqDusxkBT" title=":bn_re:" alt=":bn_re:"> to reroll.',
+				"m.mentions": {
+					// Client-Server API spec 11.37.7: This should contain the mentions for the final version of the event
+					"user_ids": ["@cadence:cadence.moe"]
+				}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY"
+			}
+		}
+	}])
+	t.equal(senderMxid, "@_ooye_bojack_horseman:cadence.moe")
+	t.deepEqual(promotions, [])
+})
+
+test("edit2changes: remove caption from image", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.removed_caption_from_image, data.guild.general, {})
+	t.deepEqual(eventsToRedact, ["$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA"])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [])
+	t.deepEqual(promotions, [{column: "part", eventID: "$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI"}])
+})
+
+test("edit2changes: change file type", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.changed_file_type, data.guild.general, {})
+	t.deepEqual(eventsToRedact, ["$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ"])
+	t.deepEqual(eventsToSend, [{
+		$type: "m.room.message",
+		body: "📝 Uploaded file: https://cdn.discordapp.com/attachments/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt (20 MB)",
+		format: "org.matrix.custom.html",
+		formatted_body: "📝 Uploaded file: <a href=\"https://cdn.discordapp.com/attachments/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt\">gaze_into_my_dark_mind.txt</a> (20 MB)",
+		"m.mentions": {},
+		msgtype: "m.text"
+	}])
+	t.deepEqual(eventsToReplace, [])
+	t.deepEqual(promotions, [{column: "part", nextEvent: true}, {column: "reaction_part", nextEvent: true}])
+})
+
+test("edit2changes: add caption back to that image", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.added_caption_to_image, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "some text",
+		"m.mentions": {}
+	}])
+	t.deepEqual(eventsToReplace, [])
+	t.deepEqual(promotions, [])
+})
+
+test("edit2changes: stickers and attachments are not changed, only the content can be edited", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* only the content can be edited",
+			"m.mentions": {},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "only the content can be edited",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4"
+			}
+		}
+	}])
+})
+
+test("edit2changes: edit of reply to skull webp attachment with content", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edit_of_reply_to_skull_webp_attachment_with_content, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "> Extremity: Image\n\n* Edit",
+			format: "org.matrix.custom.html",
+			formatted_body:
+				'<mx-reply><blockquote><a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q">In reply to</a> Extremity'
+				+ '<br>Image</blockquote></mx-reply>'
+				+ '* Edit',
+			"m.mentions": {},
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "Edit",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M"
+			}
+		}
+	}])
+})
+
+test("edit2changes: edits the text event when multiple rows have part = 0 (should never happen in real life, but make sure the safety net works)", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_0, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* only the content can be edited",
+			"m.mentions": {},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "only the content can be edited",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999"
+			}
+		}
+	}])
+})
+
+test("edit2changes: promotes the text event when multiple rows have part = 1 (should never happen in real life, but make sure the safety net works)", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_1, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* only the content can be edited",
+			"m.mentions": {},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "only the content can be edited",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111"
+			}
+		}
+	}])
+	t.deepEqual(promotions, [
+		{
+			column: "part",
+			eventID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111"
+		},
+		{
+			column: "reaction_part",
+			eventID: "$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd111"
+		}
+	])
+})

d2m/converters/emoji-to-key.js

@@ -8,10 +8,9 @@ const file = sync.require("../../matrix/file")
 
 /**
  * @param {import("discord-api-types/v10").APIEmoji} emoji
- * @param {string} message_id
  * @returns {Promise<string>}
  */
-async function emojiToKey(emoji, message_id) {
+async function emojiToKey(emoji) {
 	let key
 	if (emoji.id) {
 		// Custom emoji
@@ -31,10 +30,7 @@ async function emojiToKey(emoji, message_id) {
 		// Default emoji
 		const name = emoji.name
 		assert(name)
-		// If the reaction was used on Matrix already, it might be using a different arrangement of Variation Selector 16 characters.
-		// We'll use the same arrangement that was originally used, otherwise a duplicate of the emoji will appear as a separate reaction.
-		const originalEncoding = select("reaction", "original_encoding", {message_id, encoded_emoji: encodeURIComponent(name)}).pluck().get()
-		key = originalEncoding || name
+		key = name
 	}
 	return key
 }

d2m/converters/emoji-to-key.test.js

@@ -2,7 +2,7 @@
 
 const {test} = require("supertape")
 const {emojiToKey} = require("./emoji-to-key")
-const data = require("../../../test/data")
+const data = require("../../test/data")
 const Ty = require("../../types")
 
 test("emoji2key: unicode emoji works", async t => {

d2m/converters/lottie.js

@@ -2,7 +2,7 @@
 
 const assert = require("assert")
 const stream = require("stream")
-const {PNG} = require("@cloudrac3r/pngjs")
+const {PNG} = require("pngjs")
 
 const SIZE = 160 // Discord's display size on 1x displays is 160
 
@@ -21,7 +21,7 @@ const Rlottie = (async () => {
 
 /**
  * @param {string} text
- * @returns {Promise<stream.Readable>}
+ * @returns {Promise<import("stream").Readable>}
  */
 async function convert(text) {
 	const r = await Rlottie
@@ -41,7 +41,6 @@ async function convert(text) {
 	png.data = Buffer.from(rendered)
 	// png.pack() is a bad stream and will throw away any data it sends if it's not connected to a destination straight away.
 	// We use Duplex.from to convert it into a good stream.
-	// @ts-ignore
 	return stream.Duplex.from(png.pack())
 }
 

d2m/converters/message-to-event.embeds.test.js

@@ -0,0 +1,143 @@
+const {test} = require("supertape")
+const {messageToEvent} = require("./message-to-event")
+const data = require("../../test/data")
+const Ty = require("../../types")
+
+test("message2event embeds: nothing but a field", async t => {
+	const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.notice",
+		body: "| ### Amanda 🎵#2192 :online:"
+			+ "\n| willow tree, branch 0"
+			+ "\n| **❯ Uptime:**\n| 3m 55s\n| **❯ Memory:**\n| 64.45MB",
+		format: "org.matrix.custom.html",
+		formatted_body: '<blockquote><p><strong>Amanda 🎵#2192 <img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ\" title=\":online:\" alt=\":online:\">'
+			+ '<br>willow tree, branch 0</strong>'
+			+ '<br><strong>❯ Uptime:</strong><br>3m 55s'
+			+ '<br><strong>❯ Memory:</strong><br>64.45MB</p></blockquote>'
+	}])
+})
+
+test("message2event embeds: reply with just an embed", async t => {
+	const events = await messageToEvent(data.message_with_embeds.reply_with_only_embed, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		"m.mentions": {},
+		body: "| ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
+			+ "\n| \n| ## https://twitter.com/i/status/1707484191963648161"
+			+ "\n| \n| does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
+			+ "\n| \n| ### Retweets"
+			+ "\n| 119"
+			+ "\n| \n| ### Likes"
+			+ "\n| 5581"
+			+ "\n| — Twitter",
+		format: "org.matrix.custom.html",
+		formatted_body: '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong></p>'
+			+ '<p><strong><a href="https://twitter.com/i/status/1707484191963648161">https://twitter.com/i/status/1707484191963648161</a></strong>'
+			+ '</p><p>does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
+			+ '</p><p><strong>Retweets</strong><br>119</p><p><strong>Likes</strong><br>5581</p>— Twitter</blockquote>'
+	}])
+})
+
+test("message2event embeds: image embed and attachment", async t => {
+	const events = await messageToEvent(data.message_with_embeds.image_embed_and_attachment, data.guild.general, {}, {
+		api: {
+			async getJoinedMembers(roomID) {
+				return {joined: []}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://tootsuite.net/Warp-Gate2.gif\ntanget: @ monster spawner",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://tootsuite.net/Warp-Gate2.gif">https://tootsuite.net/Warp-Gate2.gif</a><br>tanget: @ monster spawner',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.image",
+		url: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR",
+		body: "Screenshot_20231001_034036.jpg",
+		external_url: "https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg?ex=651a1faa&is=6518ce2a&hm=eb5ca80a3fa7add8765bf404aea2028a28a2341e4a62435986bcdcf058da82f3&",
+		filename: "Screenshot_20231001_034036.jpg",
+		info: {
+			h: 1170,
+			w: 1080,
+			size: 51981,
+			mimetype: "image/jpeg"
+		},
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: blockquote in embed", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:example.invalid": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: ":emoji: **4 |** #wonderland",
+		format: "org.matrix.custom.html",
+		formatted_body: `<img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO\" title=\":emoji:\" alt=\":emoji:\"> <strong>4 |</strong> <a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe?via=cadence.moe&via=example.invalid\">#wonderland</a>`,
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n| \n| reply draft\n| > The following is a message composed via consensus of the Stinker Council.\n| > \n| > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n| > \n| > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n| > \n| > There will be no further communication.\n| \n| [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote><p><strong><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">⏺️ minimus</a></strong></p><p>reply draft<br><blockquote>The following is a message composed via consensus of the Stinker Council.<br><br>For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.<br><br>Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.<br><br>There will be no further communication.</blockquote></p><p><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">Go to Message</a></p></blockquote>",
+		"m.mentions": {}
+	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
+})
+
+test("message2event embeds: crazy html is all escaped", async t => {
+	const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## ⏺️ <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n| \n| ## <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n| \n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| \n| ### <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| — <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote>`
+			+ `<p><strong><a href="https://a.co/&amp;amp;&lt;script&gt;">⏺️ &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</a></strong></p>`
+			+ `<p><strong><a href=\"https://a.co/&amp;amp;&lt;script&gt;">&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href="https://a.co/&amp;amp">https://a.co/&amp;amp</a>;)</a></strong></p>`
+			+ `<p>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
+			+ `<p><strong>&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href=\"https://a.co/&amp;amp\">https://a.co/&amp;amp</a>;)</strong>`
+			+ `<br>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
+			+ `— &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</blockquote>`,
+		"m.mentions": {}
+	}])
+})

d2m/converters/message-to-event.js

@@ -0,0 +1,613 @@
+// @ts-check
+
+const assert = require("assert").strict
+const markdown = require("discord-markdown")
+const pb = require("prettier-bytes")
+const DiscordTypes = require("discord-api-types/v10")
+const {tag} = require("html-template-tag")
+
+const passthrough = require("../../passthrough")
+const {sync, db, discord, select, from} = passthrough
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("./emoji-to-key")} */
+const emojiToKey = sync.require("./emoji-to-key")
+/** @type {import("../actions/lottie")} */
+const lottie = sync.require("../actions/lottie")
+/** @type {import("../../m2d/converters/utils")} */
+const mxUtils = sync.require("../../m2d/converters/utils")
+/** @type {import("../../discord/utils")} */
+const dUtils = sync.require("../../discord/utils")
+const reg = require("../../matrix/read-registration")
+
+const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
+
+/**
+ * @param {DiscordTypes.APIMessage} message
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {boolean} useHTML
+ */
+function getDiscordParseCallbacks(message, guild, useHTML) {
+	return {
+		/** @param {{id: string, type: "discordUser"}} node */
+		user: node => {
+			const mxid = select("sim", "mxid", {user_id: node.id}).pluck().get()
+			const username = message.mentions.find(ment => ment.id === node.id)?.username || node.id
+			if (mxid && useHTML) {
+				return `<a href="https://matrix.to/#/${mxid}">@${username}</a>`
+			} else {
+				return `@${username}:`
+			}
+		},
+		/** @param {{id: string, type: "discordChannel", row: {room_id: string, name: string, nick: string?}?, via: string}} node */
+		channel: node => {
+			if (!node.row) { // fallback for when this channel is not bridged
+				const channel = discord.channels.get(node.id)
+				if (channel) {
+					return `#${channel.name} [channel not bridged]`
+				} else {
+					return `#unknown-channel [channel from an unbridged server]`
+				}
+			} else if (useHTML) {
+				return `<a href="https://matrix.to/#/${node.row.room_id}?${node.via}">#${node.row.nick || node.row.name}</a>`
+			} else {
+				return `#${node.row.nick || node.row.name}`
+			}
+		},
+		/** @param {{animated: boolean, name: string, id: string, type: "discordEmoji"}} node */
+		emoji: node => {
+			if (useHTML) {
+				const mxc = select("emoji", "mxc_url", {emoji_id: node.id}).pluck().get()
+				assert(mxc) // All emojis should have been added ahead of time in the messageToEvent function.
+				return `<img data-mx-emoticon height="32" src="${mxc}" title=":${node.name}:" alt=":${node.name}:">`
+			} else {
+				return `:${node.name}:`
+			}
+		},
+		role: node => {
+			const role = guild.roles.find(r => r.id === node.id)
+			if (!role) {
+				// This fallback should only trigger if somebody manually writes a silly message, or if the cache breaks (hasn't happened yet).
+				// If the cache breaks, fix discord-packets.js to store role info properly.
+				return "@&" + node.id
+			} else if (useHTML && role.color) {
+				return `<font color="#${role.color.toString(16)}">@${role.name}</font>`
+			} else if (useHTML) {
+				return `<span data-mx-color="#ffffff" data-mx-bg-color="#414eef">@${role.name}</span>`
+			} else {
+				return `@${role.name}:`
+			}
+		},
+		everyone: () => {
+			if (message.mention_everyone) return "@room"
+			return "@everyone"
+		},
+		here: () => {
+			if (message.mention_everyone) return "@room"
+			return "@here"
+		}
+	}
+}
+
+const embedTitleParser = markdown.markdownEngine.parserFor({
+	...markdown.rules,
+	autolink: undefined,
+	link: undefined
+})
+
+/**
+ * @param {{room?: boolean, user_ids?: string[]}} mentions
+ * @param {DiscordTypes.APIAttachment} attachment
+ */
+async function attachmentToEvent(mentions, attachment) {
+	const emoji =
+		attachment.content_type?.startsWith("image/jp") ? "📸"
+		: attachment.content_type?.startsWith("image/") ? "🖼️"
+		: attachment.content_type?.startsWith("video/") ? "🎞️"
+		: attachment.content_type?.startsWith("text/") ? "📝"
+		: attachment.content_type?.startsWith("audio/") ? "🎶"
+		: "📄"
+	// no native media spoilers in Element, so we'll post a link instead, forcing it to not preview using a blockquote
+	if (attachment.filename.startsWith("SPOILER_")) {
+		return {
+			$type: "m.room.message",
+			"m.mentions": mentions,
+			msgtype: "m.text",
+			body: `${emoji} Uploaded SPOILER file: ${attachment.url} (${pb(attachment.size)})`,
+			format: "org.matrix.custom.html",
+			formatted_body: `<blockquote>${emoji} Uploaded SPOILER file: <a href="${attachment.url}">${attachment.url}</a> (${pb(attachment.size)})</blockquote>`
+		}
+	}
+	// for large files, always link them instead of uploading so I don't use up all the space in the content repo
+	else if (attachment.size > reg.ooye.max_file_size) {
+		return {
+			$type: "m.room.message",
+			"m.mentions": mentions,
+			msgtype: "m.text",
+			body: `${emoji} Uploaded file: ${attachment.url} (${pb(attachment.size)})`,
+			format: "org.matrix.custom.html",
+			formatted_body: `${emoji} Uploaded file: <a href="${attachment.url}">${attachment.filename}</a> (${pb(attachment.size)})`
+		}
+	} else if (attachment.content_type?.startsWith("image/") && attachment.width && attachment.height) {
+		return {
+			$type: "m.room.message",
+			"m.mentions": mentions,
+			msgtype: "m.image",
+			url: await file.uploadDiscordFileToMxc(attachment.url),
+			external_url: attachment.url,
+			body: attachment.description || attachment.filename,
+			filename: attachment.filename,
+			info: {
+				mimetype: attachment.content_type,
+				w: attachment.width,
+				h: attachment.height,
+				size: attachment.size
+			}
+		}
+	} else if (attachment.content_type?.startsWith("video/") && attachment.width && attachment.height) {
+		return {
+			$type: "m.room.message",
+			"m.mentions": mentions,
+			msgtype: "m.video",
+			url: await file.uploadDiscordFileToMxc(attachment.url),
+			external_url: attachment.url,
+			body: attachment.description || attachment.filename,
+			filename: attachment.filename,
+			info: {
+				mimetype: attachment.content_type,
+				w: attachment.width,
+				h: attachment.height,
+				size: attachment.size
+			}
+		}
+	} else if (attachment.content_type?.startsWith("audio/")) {
+		return {
+			$type: "m.room.message",
+			"m.mentions": mentions,
+			msgtype: "m.audio",
+			url: await file.uploadDiscordFileToMxc(attachment.url),
+			external_url: attachment.url,
+			body: attachment.description || attachment.filename,
+			filename: attachment.filename,
+			info: {
+				mimetype: attachment.content_type,
+				size: attachment.size,
+				duration: attachment.duration_secs ? attachment.duration_secs * 1000 : undefined
+			}
+		}
+	} else {
+		return {
+			$type: "m.room.message",
+			"m.mentions": mentions,
+			msgtype: "m.file",
+			url: await file.uploadDiscordFileToMxc(attachment.url),
+			external_url: attachment.url,
+			body: attachment.description || attachment.filename,
+			filename: attachment.filename,
+			info: {
+				mimetype: attachment.content_type,
+				size: attachment.size
+			}
+		}
+	}
+}
+
+/**
+ * @param {import("discord-api-types/v10").APIMessage} message
+ * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {{includeReplyFallback?: boolean, includeEditFallbackStar?: boolean}} options default values:
+ * - includeReplyFallback: true
+ * - includeEditFallbackStar: false
+ * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API
+ */
+async function messageToEvent(message, guild, options = {}, di) {
+	const events = []
+
+	/* c8 ignore next 7 */
+	if (message.type === DiscordTypes.MessageType.ThreadCreated) {
+		// This is the kind of message that appears when somebody makes a thread which isn't close enough to the message it's based off.
+		// It lacks the lines and the pill, so it looks kind of like a member join message, and it says:
+		// [#] NICKNAME started a thread: __THREAD NAME__. __See all threads__
+		// We're already bridging the THREAD_CREATED gateway event to make a comparable message, so drop this one.
+		return []
+	}
+
+	if (message.type === DiscordTypes.MessageType.ThreadStarterMessage) {
+		// This is the message that appears at the top of a thread when the thread was based off an existing message.
+		// It's just a message reference, no content.
+		const ref = message.message_reference
+		assert(ref)
+		assert(ref.message_id)
+		const eventID = select("event_message", "event_id", {message_id: ref.message_id}).pluck().get()
+		const roomID = select("channel_room", "room_id", {channel_id: ref.channel_id}).pluck().get()
+		if (!eventID || !roomID) return []
+		const event = await di.api.getEvent(roomID, eventID)
+		return [{
+			...event.content,
+			$type: event.type,
+			$sender: null
+		}]
+	}
+
+	/**
+	   @type {{room?: boolean, user_ids?: string[]}}
+		We should consider the following scenarios for mentions:
+		1. A discord user rich-replies to a matrix user with a text post
+			+ The matrix user needs to be m.mentioned in the text event
+			+ The matrix user needs to have their name/mxid/link in the text event (notification fallback)
+				- So prepend their `@name:` to the start of the plaintext body
+		2. A discord user rich-replies to a matrix user with an image event only
+			+ The matrix user needs to be m.mentioned in the image event
+			+ TODO The matrix user needs to have their name/mxid in the image event's body field, alongside the filename (notification fallback)
+				- So append their name to the filename body, I guess!!!
+		3. A discord user `@`s a matrix user in the text body of their text box
+			+ The matrix user needs to be m.mentioned in the text event
+			+ No change needed to the text event content: it already has their name
+				- So make sure we don't do anything in this case.
+	*/
+	const mentions = {}
+	let repliedToEventRow = null
+	let repliedToEventSenderMxid = null
+
+	function addMention(mxid) {
+		if (!mentions.user_ids) mentions.user_ids = []
+		if (!mentions.user_ids.includes(mxid)) mentions.user_ids.push(mxid)
+	}
+
+	// Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions
+	// (Still need to do scenarios 1 and 2 part B, and scenario 3.)
+	if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) {
+		const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
+		if (row) {
+			repliedToEventRow = row
+		}
+	} else if (dUtils.isWebhookMessage(message) && message.embeds[0]?.author?.name?.endsWith("↩️")) {
+		// It could be a PluralKit emulated reply, let's see if it has a message link
+		const isEmulatedReplyToText = message.embeds[0].description?.startsWith("**[Reply to:]")
+		const isEmulatedReplyToAttachment = message.embeds[0].description?.startsWith("*[(click to see attachment")
+		if (isEmulatedReplyToText || isEmulatedReplyToAttachment) {
+			assert(message.embeds[0].description)
+			const match = message.embeds[0].description.match(/\/channels\/[0-9]*\/[0-9]*\/([0-9]{2,})/)
+			if (match) {
+				const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1])
+				if (row) {
+					/*
+						we generate a partial referenced_message based on what PK provided. we don't need everything, since this will only be used for further message-to-event converting.
+						the following properties are necessary:
+						- content: used for generating the reply fallback
+						- author: used for the top of the reply fallback (only used for discord authors. for matrix authors, repliedToEventSenderMxid is set.)
+					*/
+					const emulatedMessageContent =
+						( isEmulatedReplyToAttachment ? "[Media]"
+						: message.embeds[0].description.replace(/^.*?\)\*\*\s*/, ""))
+					message.referenced_message = {
+						content: emulatedMessageContent,
+						// @ts-ignore
+						author: {
+							username: message.embeds[0].author.name.replace(/\s*↩️\s*$/, "")
+						}
+					}
+					message.embeds.shift()
+					repliedToEventRow = row
+				}
+			}
+		}
+	}
+	if (repliedToEventRow && repliedToEventRow.source === 0) { // reply was originally from Matrix
+		// Need to figure out who sent that event...
+		const event = await di.api.getEvent(repliedToEventRow.room_id, repliedToEventRow.event_id)
+		repliedToEventSenderMxid = event.sender
+		// Need to add the sender to m.mentions
+		addMention(repliedToEventSenderMxid)
+	}
+
+	/** @type {Map<string, Promise<string>>} */
+	const viaMemo = new Map()
+	/**
+	 * @param {string} roomID
+	 * @returns {Promise<string>} string encoded URLSearchParams
+	 */
+	function getViaServersMemo(roomID) {
+		// @ts-ignore
+		if (viaMemo.has(roomID)) return viaMemo.get(roomID)
+		const promise = mxUtils.getViaServersQuery(roomID, di.api).then(p => p.toString())
+		viaMemo.set(roomID, promise)
+		return promise
+	}
+
+	/**
+	 * Translate Discord message links to Matrix event links.
+	 * If OOYE has handled this message in the past, this is an instant database lookup.
+	 * Otherwise, if OOYE knows the channel, this is a multi-second request to /timestamp_to_event to approximate.
+	 * @param {string} content Partial or complete Discord message content
+	 */
+	async function transformContentMessageLinks(content) {
+		let offset = 0
+		for (const match of [...content.matchAll(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/[0-9]+\/([0-9]+)\/([0-9]+)/g)]) {
+			assert(typeof match.index === "number")
+			const [_, channelID, messageID] = match
+			let result
+
+			const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
+			if (roomID) {
+				const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get()
+				const via = await getViaServersMemo(roomID)
+				if (eventID && roomID) {
+					result = `https://matrix.to/#/${roomID}/${eventID}?${via}`
+				} else {
+					const ts = dUtils.snowflakeToTimestampExact(messageID)
+					const {event_id} = await di.api.getEventForTimestamp(roomID, ts)
+					result = `https://matrix.to/#/${roomID}/${event_id}?${via}`
+				}
+			} else {
+				result = `${match[0]} [event is from another server]`
+			}
+
+			content = content.slice(0, match.index + offset) + result + content.slice(match.index + match[0].length + offset)
+			offset += result.length - match[0].length
+		}
+		return content
+	}
+
+	/**
+	 * Translate links and emojis and mentions and stuff. Give back the text and HTML so they can be combined into bigger events.
+	 * @param {string} content Partial or complete Discord message content
+	 * @param {any} customOptions
+	 * @param {any} customParser
+	 * @param {any} customHtmlOutput
+	 */
+	async function transformContent(content, customOptions = {}, customParser = null, customHtmlOutput = null) {
+		content = await transformContentMessageLinks(content)
+
+		// Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter.
+		// So we scan the message ahead of time for all its emojis and ensure they are in the DB.
+		const emojiMatches = [...content.matchAll(/<(a?):([^:>]{2,64}):([0-9]+)>/g)]
+		await Promise.all(emojiMatches.map(match => {
+			const id = match[3]
+			const name = match[2]
+			const animated = !!match[1]
+			return emojiToKey.emojiToKey({id, name, animated}) // Register the custom emoji if needed
+		}))
+
+		async function transformParsedVia(parsed) {
+			for (const node of parsed) {
+				if (node.type === "discordChannel") {
+					node.row = select("channel_room", ["room_id", "name", "nick"], {channel_id: node.id}).get()
+					if (node.row?.room_id) {
+						node.via = await getViaServersMemo(node.row.room_id)
+					}
+				}
+				if (Array.isArray(node.content)) {
+					await transformParsedVia(node.content)
+				}
+			}
+			return parsed
+		}
+
+		let html = await markdown.toHtmlWithPostParser(content, transformParsedVia, {
+			discordCallback: getDiscordParseCallbacks(message, guild, true),
+			...customOptions
+		}, customParser, customHtmlOutput)
+
+		let body = await markdown.toHtmlWithPostParser(content, transformParsedVia, {
+			discordCallback: getDiscordParseCallbacks(message, guild, false),
+			discordOnly: true,
+			escapeHTML: false,
+			...customOptions
+		}, null, null)
+
+		return {body, html}
+	}
+
+	// FIXME: What was the scanMentions parameter supposed to activate? It's unused.
+	async function addTextEvent(body, html, msgtype, {scanMentions}) {
+		// Star * prefix for fallback edits
+		if (options.includeEditFallbackStar) {
+			body = "* " + body
+			html = "* " + html
+		}
+
+		const flags = message.flags || 0
+		if (flags & 2) {
+			body = `[🔀 ${message.author.username}]\n` + body
+			html = `🔀 <strong>${message.author.username}</strong><br>` + html
+		}
+
+		// Fallback body/formatted_body for replies
+		// This branch is optional - do NOT change anything apart from the reply fallback, since it may not be run
+		if (repliedToEventRow && options.includeReplyFallback !== false) {
+			let repliedToDisplayName
+			let repliedToUserHtml
+			if (repliedToEventRow?.source === 0 && repliedToEventSenderMxid) {
+				const match = repliedToEventSenderMxid.match(/^@([^:]*)/)
+				assert(match)
+				repliedToDisplayName = message.referenced_message?.author.username || match[1] || "a Matrix user" // grab the localpart as the display name, whatever
+				repliedToUserHtml = `<a href="https://matrix.to/#/${repliedToEventSenderMxid}">${repliedToDisplayName}</a>`
+			} else {
+				repliedToDisplayName = message.referenced_message?.author.global_name || message.referenced_message?.author.username || "a Discord user"
+				repliedToUserHtml = repliedToDisplayName
+			}
+			let repliedToContent = message.referenced_message?.content
+			if (repliedToContent?.startsWith("> <:L1:")) {
+				// If the Discord user is replying to a Matrix user's reply, the fallback is going to contain the emojis and stuff from the bridged rep of the Matrix user's reply quote.
+				// Need to remove that previous reply rep from this fallback body. The fallbody body should only contain the Matrix user's actual message.
+				repliedToContent = repliedToContent.split("\n").slice(2).join("\n")
+			}
+			if (repliedToContent == "") repliedToContent = "[Media]"
+			else if (!repliedToContent) repliedToContent = "[Replied-to message content wasn't provided by Discord]"
+			const repliedToHtml = markdown.toHTML(repliedToContent, {
+				discordCallback: getDiscordParseCallbacks(message, guild, true)
+			}, null, null)
+			const repliedToBody = markdown.toHTML(repliedToContent, {
+				discordCallback: getDiscordParseCallbacks(message, guild, false),
+				discordOnly: true,
+				escapeHTML: false,
+			}, null, null)
+			html = `<mx-reply><blockquote><a href="https://matrix.to/#/${repliedToEventRow.room_id}/${repliedToEventRow.event_id}">In reply to</a> ${repliedToUserHtml}`
+				+ `<br>${repliedToHtml}</blockquote></mx-reply>`
+				+ html
+			body = (`${repliedToDisplayName}: ` // scenario 1 part B for mentions
+				+ repliedToBody).split("\n").map(line => "> " + line).join("\n")
+				+ "\n\n" + body
+		}
+
+		const newTextMessageEvent = {
+			$type: "m.room.message",
+			"m.mentions": mentions,
+			msgtype,
+			body: body
+		}
+
+		const isPlaintext = body === html
+
+		if (!isPlaintext) {
+			Object.assign(newTextMessageEvent, {
+				format: "org.matrix.custom.html",
+				formatted_body: html
+			})
+		}
+
+		events.push(newTextMessageEvent)
+	}
+
+
+	let msgtype = "m.text"
+	// Handle message type 4, channel name changed
+	if (message.type === DiscordTypes.MessageType.ChannelNameChange) {
+		msgtype = "m.emote"
+		message.content = "changed the channel name to **" + message.content + "**"
+	}
+
+	// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
+	const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
+	if (matches.length && matches.some(m => m[1].match(/[a-z]/i))) {
+		const writtenMentionsText = matches.map(m => m[1].toLowerCase())
+		const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
+		assert(roomID)
+		const {joined} = await di.api.getJoinedMembers(roomID)
+		for (const [mxid, member] of Object.entries(joined)) {
+			if (!userRegex.some(rx => mxid.match(rx))) {
+				const localpart = mxid.match(/@([^:]*)/)
+				assert(localpart)
+				const displayName = member.display_name || localpart[1]
+				if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+			}
+		}
+	}
+
+	// Text content appears first
+	if (message.content) {
+		const {body, html} = await transformContent(message.content)
+		await addTextEvent(body, html, msgtype, {scanMentions: true})
+	}
+
+	// Then attachments
+	const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions)))
+	events.push(...attachmentEvents)
+
+	// Then embeds
+	for (const embed of message.embeds || []) {
+		if (embed.type === "image") {
+			continue // Matrix's own URL previews are fine for images.
+		}
+
+		// Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once
+		const rep = new mxUtils.MatrixStringBuilder()
+
+		// Author and URL into a paragraph
+		let authorNameText = embed.author?.name || ""
+		if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image
+		if (authorNameText || embed.author?.url) {
+			if (embed.author?.url) {
+				const authorURL = await transformContentMessageLinks(embed.author.url)
+				rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`<strong><a href="${authorURL}">${authorNameText}</a></strong>`)
+			} else {
+				rep.addParagraph(`## ${authorNameText}`, tag`<strong>${authorNameText}</strong>`)
+			}
+		}
+
+		// Title and URL into a paragraph
+		if (embed.title) {
+			const {body, html} = await transformContent(embed.title, {}, embedTitleParser, markdown.htmlOutput)
+			if (embed.url) {
+				rep.addParagraph(`## ${body} ${embed.url}`, tag`<strong><a href="${embed.url}">$${html}</a></strong>`)
+			} else {
+				rep.addParagraph(`## ${body}`, `<strong>${html}</strong>`)
+			}
+		} else if (embed.url) {
+			rep.addParagraph(`## ${embed.url}`, tag`<strong><a href="${embed.url}">${embed.url}</a></strong>`)
+		}
+
+		if (embed.description) {
+			const {body, html} = await transformContent(embed.description)
+			rep.addParagraph(body, html)
+		}
+
+		for (const field of embed.fields || []) {
+			const name = field.name.match(/^[\s​­]*$/) ? {body: "", html: ""} : await transformContent(field.name, {}, embedTitleParser, markdown.htmlOutput)
+			const value = await transformContent(field.value)
+			const fieldRep = new mxUtils.MatrixStringBuilder()
+				.addLine(`### ${name.body}`, `<strong>${name.html}</strong>`, name.body)
+				.addLine(value.body, value.html, !!value.body)
+			rep.addParagraph(fieldRep.get().body, fieldRep.get().formatted_body)
+		}
+
+		if (embed.image?.url) rep.addParagraph(`📸 ${embed.image.url}`)
+		if (embed.video?.url) rep.addParagraph(`🎞️ ${embed.video.url}`)
+
+		if (embed.footer?.text) rep.addLine(`— ${embed.footer.text}`, tag`— ${embed.footer.text}`)
+		let {body, formatted_body: html} = rep.get()
+		body = body.split("\n").map(l => "| " + l).join("\n")
+		html = `<blockquote>${html}</blockquote>`
+
+		// Send as m.notice to apply the usual automated/subtle appearance, showing this wasn't actually typed by the person
+		await addTextEvent(body, html, "m.notice", {scanMentions: false})
+	}
+
+	// Then stickers
+	if (message.sticker_items) {
+		const stickerEvents = await Promise.all(message.sticker_items.map(async stickerItem => {
+			const format = file.stickerFormat.get(stickerItem.format_type)
+			assert(format?.mime)
+			if (format?.mime === "lottie") {
+				const {mxc_url, info} = await lottie.convert(stickerItem)
+				return {
+					$type: "m.sticker",
+					"m.mentions": mentions,
+					body: stickerItem.name,
+					info,
+					url: mxc_url
+				}
+			} else {
+				let body = stickerItem.name
+				const sticker = guild.stickers.find(sticker => sticker.id === stickerItem.id)
+				if (sticker && sticker.description) body += ` - ${sticker.description}`
+				return {
+					$type: "m.sticker",
+					"m.mentions": mentions,
+					body,
+					info: {
+						mimetype: format.mime
+					},
+					url: await file.uploadDiscordFileToMxc(file.sticker(stickerItem))
+				}
+			}
+		}))
+		events.push(...stickerEvents)
+	}
+
+	// Rich replies
+	if (repliedToEventRow) {
+		Object.assign(events[0], {
+			"m.relates_to": {
+				"m.in_reply_to": {
+					event_id: repliedToEventRow.event_id
+				}
+			}
+		})
+	}
+
+	return events
+}
+
+module.exports.messageToEvent = messageToEvent

d2m/converters/message-to-event.pk.test.js

@@ -1,6 +1,6 @@
 const {test} = require("supertape")
 const {messageToEvent} = require("./message-to-event")
-const data = require("../../../test/data")
+const data = require("../../test/data")
 const Ty = require("../../types")
 
 /**
@@ -50,7 +50,11 @@ test("message2event: pk reply to matrix is converted to native matrix reply", as
 			]
 		},
 		msgtype: "m.text",
-		body: "this is a reply",
+		body: "> cadence [they]: now for my next experiment:\n\nthis is a reply",
+		format: "org.matrix.custom.html",
+		formatted_body: '<mx-reply><blockquote><a href="https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU">In reply to</a> <a href="https://matrix.to/#/@cadence:cadence.moe">cadence [they]</a><br>'
+			+ "now for my next experiment:</blockquote></mx-reply>"
+			+ "this is a reply",
 		"m.relates_to": {
 			"m.in_reply_to": {
 				event_id: "$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU"
@@ -76,7 +80,11 @@ test("message2event: pk reply to discord is converted to native matrix reply", a
 		$type: "m.room.message",
 		msgtype: "m.text",
 		"m.mentions": {},
-		body: "this is a reply",
+		body: "> wing: some text\n\nthis is a reply",
+		format: "org.matrix.custom.html",
+		formatted_body: '<mx-reply><blockquote><a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA">In reply to</a> wing<br>'
+			+ "some text</blockquote></mx-reply>"
+			+ "this is a reply",
 		"m.relates_to": {
 			"m.in_reply_to": {
 				event_id: "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA"
@@ -112,7 +120,11 @@ test("message2event: pk reply to matrix attachment is converted to native matrix
 		"m.mentions": {
 			user_ids: ["@ampflower:matrix.org"]
 		},
-		body: "Cat nod",
+		body: "> Ampflower 🌺: [Media]\n\nCat nod",
+		format: "org.matrix.custom.html",
+		formatted_body: '<mx-reply><blockquote><a href="https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y">In reply to</a> <a href="https://matrix.to/#/@ampflower:matrix.org">Ampflower 🌺</a><br>'
+			+ "[Media]</blockquote></mx-reply>"
+			+ "Cat nod",
 		"m.relates_to": {
 			"m.in_reply_to": {
 				event_id: "$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y"

d2m/converters/message-to-event.test.js

@@ -0,0 +1,791 @@
+const {test} = require("supertape")
+const {messageToEvent} = require("./message-to-event")
+const data = require("../../test/data")
+const Ty = require("../../types")
+
+/**
+ * @param {string} roomID
+ * @param {string} eventID
+ * @returns {(roomID: string, eventID: string) => Promise<Ty.Event.Outer<Ty.Event.M_Room_Message>>}
+ */
+function mockGetEvent(t, roomID_in, eventID_in, outer) {
+	return async function(roomID, eventID) {
+		t.equal(roomID, roomID_in)
+		t.equal(eventID, eventID_in)
+		return new Promise(resolve => {
+			setTimeout(() => {
+				resolve({
+					event_id: eventID_in,
+					room_id: roomID_in,
+					origin_server_ts: 1680000000000,
+					unsigned: {
+						age: 2245,
+						transaction_id: "$local.whatever"
+					},
+					...outer
+				})
+			})
+		})
+	}
+}
+
+test("message2event: simple plaintext", async t => {
+	const events = await messageToEvent(data.message.simple_plaintext, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "ayy lmao"
+	}])
+})
+
+test("message2event: simple plaintext with quotes", async t => {
+	const events = await messageToEvent(data.message.simple_plaintext_with_quotes, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: `then he said, "you and her aren't allowed in here!"`
+	}])
+})
+
+test("message2event: simple user mention", async t => {
+	const events = await messageToEvent(data.message.simple_user_mention, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "@crunch god: Tell me about Phil, renowned martial arts master and creator of the Chin Trick",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://matrix.to/#/@_ooye_crunch_god:cadence.moe">@crunch god</a> Tell me about Phil, renowned martial arts master and creator of the Chin Trick'
+	}])
+})
+
+test("message2event: simple room mention", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message.simple_room_mention, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "#main",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org">#main</a>'
+	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
+})
+
+test("message2event: unknown room mention", async t => {
+	const events = await messageToEvent(data.message.unknown_room_mention, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "#unknown-channel [channel from an unbridged server]"
+	}])
+})
+
+test("message2event: unbridged room mention", async t => {
+	const events = await messageToEvent(data.message.unbridged_room_mention, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "#bad-boots-prison [channel not bridged]"
+	}])
+})
+
+test("message2event: simple role mentions", async t => {
+	const events = await messageToEvent(data.message.simple_role_mentions, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "I'm just @!!DLCS!!: testing a few role pings @Master Wonder Mage: don't mind me",
+		format: "org.matrix.custom.html",
+		formatted_body: `I'm just <font color="#a901ff">@!!DLCS!!</font> testing a few role pings <span data-mx-color="#ffffff" data-mx-bg-color="#414eef">@Master Wonder Mage</span> don't mind me`
+	}])
+})
+
+test("message2event: manually constructed unknown roles should use fallback", async t => {
+	const events = await messageToEvent(data.message.unknown_role, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "I'm just @&4 testing a few role pings <@&B> don't mind me",
+		format: "org.matrix.custom.html",
+		formatted_body: "I'm just @&4 testing a few role pings &lt;@&amp;B&gt; don't mind me"
+	}])
+})
+
+test("message2event: simple message link", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message.simple_message_link, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:super.invalid": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg?via=cadence.moe&via=super.invalid",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg?via=cadence.moe&amp;via=super.invalid">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg?via=cadence.moe&amp;via=super.invalid</a>'
+	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
+})
+
+test("message2event: message link that OOYE doesn't know about", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message.message_link_to_before_ooye, data.guild.general, {}, {
+		api: {
+			async getEventForTimestamp(roomID, ts) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					event_id: "$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U",
+					origin_server_ts: 1613287812754
+				}
+			},
+			async getStateEvent(roomID, type, key) { // for ?via calculation
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) { // for ?via calculation
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "Me: I'll scroll up to find a certain message I'll send\n_scrolls up and clicks message links for god knows how long_\n_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n_stumbles upon:_ "
+			+ "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U?via=cadence.moe&via=matrix.org",
+		format: "org.matrix.custom.html",
+		formatted_body: "Me: I'll scroll up to find a certain message I'll send<br><em>scrolls up and clicks message links for god knows how long</em><br><em>completely forgets what they were looking for and simply begins scrolling up to find some fun moments</em><br><em>stumbles upon:</em> "
+			+ '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U?via=cadence.moe&amp;via=matrix.org">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U?via=cadence.moe&amp;via=matrix.org</a>'
+	}])
+	t.equal(called, 3, "getEventForTimestamp, getStateEvent, and getJoinedMembers should be called once each")
+})
+
+test("message2event: message link from another server", async t => {
+	const events = await messageToEvent(data.message.message_link_from_another_server, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "Neither of these servers are known to OOYE: https://discord.com/channels/111/222/333 [event is from another server] https://canary.discordapp.com/channels/444/555/666 [event is from another server]",
+		format: "org.matrix.custom.html",
+		formatted_body: 'Neither of these servers are known to OOYE: <a href="https://discord.com/channels/111/222/333">https://discord.com/channels/111/222/333</a> [event is from another server]'
+			+ ' <a href="https://canary.discordapp.com/channels/444/555/666">https://canary.discordapp.com/channels/444/555/666</a> [event is from another server]'
+	}])
+})
+
+test("message2event: attachment with no content", async t => {
+	const events = await messageToEvent(data.message.attachment_no_content, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.image",
+		url: "mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM",
+		body: "image.png",
+		external_url: "https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png",
+		filename: "image.png",
+		info: {
+			mimetype: "image/png",
+			w: 466,
+			h: 85,
+			size: 12919,
+		},
+	}])
+})
+
+test("message2event: spoiler attachment", async t => {
+	const events = await messageToEvent(data.message.spoiler_attachment, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "📄 Uploaded SPOILER file: https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci (74 KB)",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote>📄 Uploaded SPOILER file: <a href=\"https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci\">https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci</a> (74 KB)</blockquote>"
+	}])
+})
+
+test("message2event: stickers", async t => {
+	const events = await messageToEvent(data.message.sticker, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "can have attachments too"
+	}, {
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.image",
+		url: "mxc://cadence.moe/ZDCNYnkPszxGKgObUIFmvjus",
+		body: "image.png",
+		external_url: "https://cdn.discordapp.com/attachments/122155380120748034/1106366167486038016/image.png",
+		filename: "image.png",
+		info: {
+			mimetype: "image/png",
+			w: 333,
+			h: 287,
+			size: 127373,
+		},
+	}, {
+		$type: "m.sticker",
+		"m.mentions": {},
+		body: "pomu puff - damn that tiny lil bitch really chuffing. puffing that fat ass dart",
+		info: {
+			mimetype: "image/png"
+			// thumbnail_url
+			// thumbnail_info
+		},
+		url: "mxc://cadence.moe/UuUaLwXhkxFRwwWCXipDlBHn"
+	}])
+})
+
+test("message2event: lottie sticker", async t => {
+	const events = await messageToEvent(data.message.lottie_sticker, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.sticker",
+		"m.mentions": {},
+		body: "8",
+		info: {
+			mimetype: "image/png",
+			w: 160,
+			h: 160
+		},
+		url: "mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR"
+	}])
+})
+
+test("message2event: skull webp attachment with content", async t => {
+	const events = await messageToEvent(data.message.skull_webp_attachment_with_content, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "Image"
+	}, {
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.image",
+		body: "skull.webp",
+		info: {
+			w: 1200,
+			h: 628,
+			mimetype: "image/webp",
+			size: 74290
+		},
+		external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1128084747910918195/skull.webp",
+		filename: "skull.webp",
+		url: "mxc://cadence.moe/sDxWmDErBhYBxtDcJQgBETes"
+	}])
+})
+
+test("message2event: reply to skull webp attachment with content", async t => {
+	const events = await messageToEvent(data.message.reply_to_skull_webp_attachment_with_content, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q"
+			}
+		},
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "> Extremity: Image\n\nReply",
+		format: "org.matrix.custom.html",
+		formatted_body:
+			'<mx-reply><blockquote><a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q">In reply to</a> Extremity'
+			+ '<br>Image</blockquote></mx-reply>'
+			+ 'Reply'
+	}, {
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.image",
+		body: "RDT_20230704_0936184915846675925224905.jpg",
+		info: {
+			w: 2048,
+			h: 1536,
+			mimetype: "image/jpeg",
+			size: 85906
+		},
+		external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1128084851023675515/RDT_20230704_0936184915846675925224905.jpg",
+		filename: "RDT_20230704_0936184915846675925224905.jpg",
+		url: "mxc://cadence.moe/WlAbFSiNRIHPDEwKdyPeGywa"
+	}])
+})
+
+test("message2event: simple reply to matrix user", async t => {
+	const events = await messageToEvent(data.message.simple_reply_to_matrix_user, data.guild.general, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!kLRqKKUQXcibIMtOpl:cadence.moe", "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4", {
+				type: "m.room.message",
+				content: {
+					msgtype: "m.text",
+					body: "so can you reply to my webhook uwu"
+				},
+				sender: "@cadence:cadence.moe"
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4"
+			}
+		},
+		"m.mentions": {
+			user_ids: [
+				"@cadence:cadence.moe"
+			]
+		},
+		msgtype: "m.text",
+		body: "> cadence: so can you reply to my webhook uwu\n\nReply",
+		format: "org.matrix.custom.html",
+		formatted_body:
+			'<mx-reply><blockquote><a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4">In reply to</a> <a href="https://matrix.to/#/@cadence:cadence.moe">cadence</a>'
+			+ '<br>so can you reply to my webhook uwu</blockquote></mx-reply>'
+			+ 'Reply'
+	}])
+})
+
+test("message2event: simple reply to matrix user, reply fallbacks disabled", async t => {
+	const events = await messageToEvent(data.message.simple_reply_to_matrix_user, data.guild.general, {includeReplyFallback: false}, {
+		api: {
+			getEvent: mockGetEvent(t, "!kLRqKKUQXcibIMtOpl:cadence.moe", "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4", {
+				type: "m.room.message",
+				content: {
+					msgtype: "m.text",
+					body: "so can you reply to my webhook uwu"
+				},
+				sender: "@cadence:cadence.moe"
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4"
+			}
+		},
+		"m.mentions": {
+			user_ids: [
+				"@cadence:cadence.moe"
+			]
+		},
+		msgtype: "m.text",
+		body: "Reply"
+	}])
+})
+
+test("message2event: reply with a video", async t => {
+	const events = await messageToEvent(data.message.reply_with_video, data.guild.general, {
+		api: {
+			getEvent: mockGetEvent(t, "!kLRqKKUQXcibIMtOpl:cadence.moe", "$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw", {
+				type: "m.room.message",
+				content: {
+					msgtype: "m.text",
+					body: 'deadpicord "extremity you woke up at 4 am"'
+				},
+				sender: "@_ooye_extremity:cadence.moe"
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.video",
+		body: "Ins_1960637570.mp4",
+		filename: "Ins_1960637570.mp4",
+		url: "mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU",
+		external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4?ex=65bbee8f&is=65a9798f&hm=ae14f7824c3d526c5e11c162e012e1ee405fd5776e1e9302ed80ccd86503cfda&",
+		info: {
+			h: 854,
+			mimetype: "video/mp4",
+			size: 860559,
+			w: 480,
+		},
+		"m.mentions": {},
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw"
+			}
+		}
+	}])
+})
+
+test("message2event: voice message", async t => {
+	const events = await messageToEvent(data.message.voice_message)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+      body: "voice-message.ogg",
+      external_url: "https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg?ex=65c92d4c&is=65b6b84c&hm=0654bab5027474cbe23875954fa117cf44d8914c144cd151879590fa1baf8b1c&",
+      filename: "voice-message.ogg",
+      info: {
+        duration: 3960.0000381469727,
+        mimetype: "audio/ogg",
+        size: 10584,
+		},
+      "m.mentions": {},
+      msgtype: "m.audio",
+      url: "mxc://cadence.moe/MRRPDggXQMYkrUjTpxQbmcxB"
+	}])
+})
+
+test("message2event: misc file", async t => {
+	const events = await messageToEvent(data.message.misc_file)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "final final final revised draft",
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+      body: "the.yml",
+      external_url: "https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml?ex=65cd6270&is=65baed70&hm=8c5f1b571784e3c7f99628492298815884e351ae0dc7c2ae40dd22d97caf27d9&",
+      filename: "the.yml",
+		info: {
+			mimetype: "text/plain; charset=utf-8",
+			size: 2274
+		},
+      "m.mentions": {},
+      msgtype: "m.file",
+      url: "mxc://cadence.moe/HnQIYQmmlIKwOQsbFsIGpzPP"
+	}])
+})
+
+test("message2event: simple reply in thread to a matrix user's reply", async t => {
+	const events = await messageToEvent(data.message.simple_reply_to_reply_in_thread, data.guild.general, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!FuDZhlOAtqswlyxzeR:cadence.moe", "$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo", {
+				type: "m.room.message",
+				sender: "@cadence:cadence.moe",
+				content: {
+					msgtype: "m.text",
+					body: "> <@_ooye_cadence:cadence.moe> So what I'm wondering is about replies.\n\nWhat about them?",
+					format: "org.matrix.custom.html",
+					formatted_body: "<mx-reply><blockquote><a href=\"https://matrix.to/#/!FuDZhlOAtqswlyxzeR:cadence.moe/$fWQT8uOrzLzAXNVXz88VkGx7Oo724iS5uD8Qn5KUy9w?via=cadence.moe\">In reply to</a> <a href=\"https://matrix.to/#/@_ooye_cadence:cadence.moe\">@_ooye_cadence:cadence.moe</a><br>So what I&#39;m wondering is about replies.</blockquote></mx-reply>What about them?",
+					"m.relates_to": {
+						"m.in_reply_to": {
+							event_id: "$fWQT8uOrzLzAXNVXz88VkGx7Oo724iS5uD8Qn5KUy9w"
+						}
+					}
+				},
+				event_id: "$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo",
+				room_id: "!FuDZhlOAtqswlyxzeR:cadence.moe"
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo"
+			}
+		},
+		"m.mentions": {
+			user_ids: ["@cadence:cadence.moe"]
+		},
+		msgtype: "m.text",
+      body: "> cadence [they]: What about them?\n\nWell, they don't seem to...",
+      format: "org.matrix.custom.html",
+      formatted_body: "<mx-reply><blockquote><a href=\"https://matrix.to/#/!FuDZhlOAtqswlyxzeR:cadence.moe/$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo\">In reply to</a> <a href=\"https://matrix.to/#/@cadence:cadence.moe\">cadence [they]</a><br>What about them?</blockquote></mx-reply>Well, they don't seem to...",
+	}])
+})
+
+test("message2event: simple written @mention for matrix user", async t => {
+	const events = await messageToEvent(data.message.simple_written_at_mention_for_matrix, data.guild.general, {}, {
+		api: {
+			async getJoinedMembers(roomID) {
+				t.equal(roomID, "!rEOspnYqdOalaIFniV:cadence.moe")
+				return new Promise(resolve => {
+					setTimeout(() => {
+						resolve({
+							joined: {
+								"@she_who_brings_destruction:matrix.org": {
+									avatar_url: "mxc://matrix.org/FKcfnfFZlEhspeMsERfYtCuO",
+									display_name: "ash (Old)"
+								},
+								"@tomskeleton:cadence.moe": {
+									avatar_url: "mxc://cadence.moe/OvYYicuOwfAACKaXKJCUPbVz",
+									display_name: "tomskeleton"
+								},
+								"@she_who_brings_destruction:cadence.moe": {
+									avatar_url: "mxc://cadence.moe/XDXLMbkieETPrjFupoeiwyyq",
+									display_name: "ash"
+								},
+								"@_ooye_bot:cadence.moe": {
+									avatar_url: "mxc://cadence.moe/jlrgFjYQHzfBvORedOmYqXVz",
+									display_name: "Out Of Your Element"
+								},
+								"@cadence:cadence.moe": {
+									avatar_url: "mxc://cadence.moe/GJDPWiryxIhyRBNJzRNYzAlh",
+									display_name: "cadence [they]"
+								},
+								"@_ooye_tomskeleton:cadence.moe": {
+									avatar_url: "mxc://cadence.moe/SdSrjjsrNVdyPTAKEGQUhKUK",
+									display_name: "tomskeleton"
+								},
+								"@_ooye_queergasm:cadence.moe": {
+									avatar_url: "mxc://cadence.moe/KqXYGbUqhPPJKifLmfpoLnmB",
+									display_name: "queergasm"
+								},
+								"@_ooye_.subtext:cadence.moe": {
+									avatar_url: "mxc://cadence.moe/heoCvaUmfCdpxdzaChwwkpEp",
+									display_name: ".subtext"
+								}
+							}
+						})
+					})
+				})
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {
+			user_ids: [
+				"@she_who_brings_destruction:cadence.moe"
+			]
+		},
+		msgtype: "m.text",
+		body: "@ash do you need anything from the store btw as I'm heading there after gym"
+	}])
+})
+
+test("message2event: advanced written @mentions for matrix users", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message.advanced_written_at_mention_for_matrix, data.guild.general, {}, {
+		api: {
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return new Promise(resolve => {
+					setTimeout(() => {
+						resolve({
+							joined: {
+								"@cadence:cadence.moe": {
+									display_name: "cadence [they]",
+									avatar_url: "whatever"
+								},
+								"@huckleton:cadence.moe": {
+									display_name: "huck",
+									avatar_url: "whatever"
+								},
+								"@_ooye_botrac4r:cadence.moe": {
+									display_name: "botrac4r",
+									avatar_url: "whatever"
+								},
+								"@_ooye_bot:cadence.moe": {
+									display_name: "Out Of Your Element",
+									avatar_url: "whatever"
+								}
+							}
+						})
+					})
+				})
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {
+			user_ids: [
+				"@cadence:cadence.moe",
+				"@huckleton:cadence.moe"
+			]
+		},
+		msgtype: "m.text",
+		body: "@Cadence, tell me about @Phil, the creator of the Chin Trick, who has become ever more powerful under the mentorship of @botrac4r and @huck"
+	}])
+	t.equal(called, 1, "should only look up the member list once")
+})
+
+test("message2event: very large attachment is linked instead of being uploaded", async t => {
+	const events = await messageToEvent({
+		content: "hey",
+		attachments: [{
+			filename: "hey.jpg",
+			url: "https://discord.com/404/hey.jpg",
+			content_type: "application/i-made-it-up",
+			size: 100e6
+		}]
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "hey"
+	}, {
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "📄 Uploaded file: https://discord.com/404/hey.jpg (100 MB)",
+		format: "org.matrix.custom.html",
+		formatted_body: '📄 Uploaded file: <a href="https://discord.com/404/hey.jpg">hey.jpg</a> (100 MB)'
+	}])
+})
+
+test("message2event: type 4 channel name change", async t => {
+	const events = await messageToEvent(data.special_message.thread_name_change, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.emote",
+		body: "changed the channel name to **worming**",
+		format: "org.matrix.custom.html",
+		formatted_body: "changed the channel name to <strong>worming</strong>"
+	}])
+})
+
+test("message2event: thread start message reference", async t => {
+	const events = await messageToEvent(data.special_message.thread_start_context, data.guild.general, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!BnKuBPCvyfOkhcUjEu:cadence.moe", "$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo", {
+				"type": "m.room.message",
+				"sender": "@_ooye_kyuugryphon:cadence.moe",
+				"content": {
+					"m.mentions": {},
+					"msgtype": "m.text",
+					"body": "layer 4"
+				}
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		$sender: null,
+		msgtype: "m.text",
+		body: "layer 4",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: single large bridged emoji", async t => {
+	const events = await messageToEvent(data.message.single_emoji, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: ":hippo:",
+		format: "org.matrix.custom.html",
+		formatted_body: '<img data-mx-emoticon height="32" src="mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC" title=":hippo:" alt=":hippo:">'
+	}])
+})
+
+test("message2event: mid-message small bridged emoji", async t => {
+	const events = await messageToEvent(data.message.surrounded_emoji, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "h is for :hippo:!",
+		format: "org.matrix.custom.html",
+		formatted_body: 'h is for <img data-mx-emoticon height="32" src="mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC" title=":hippo:" alt=":hippo:">!'
+	}])
+})
+
+test("message2event: emoji that hasn't been registered yet", async t => {
+	const events = await messageToEvent(data.message.not_been_registered_emoji, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: ":Yeah:",
+		format: "org.matrix.custom.html",
+		formatted_body: '<img data-mx-emoticon height="32" src="mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP" title=":Yeah:" alt=":Yeah:">'
+	}])
+})
+
+test("message2event: emoji triple long name", async t => {
+	const events = await messageToEvent(data.message.emoji_triple_long_name, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: ":brillillillilliant_move::brillillillilliant_move::brillillillilliant_move:",
+		format: "org.matrix.custom.html",
+		formatted_body:
+			  '<img data-mx-emoticon height="32" src="mxc://cadence.moe/scfRIDOGKWFDEBjVXocWYQHik" title=":brillillillilliant_move:" alt=":brillillillilliant_move:">'
+			+ '<img data-mx-emoticon height="32" src="mxc://cadence.moe/scfRIDOGKWFDEBjVXocWYQHik" title=":brillillillilliant_move:" alt=":brillillillilliant_move:">'
+			+ '<img data-mx-emoticon height="32" src="mxc://cadence.moe/scfRIDOGKWFDEBjVXocWYQHik" title=":brillillillilliant_move:" alt=":brillillillilliant_move:">'
+	}])
+})
+
+test("message2event: crossposted announcements say where they are crossposted from", async t => {
+	const events = await messageToEvent(data.special_message.crosspost_announcement, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "[🔀 Chewey Bot Official Server #announcements]\nAll text based commands are now inactive on Chewey Bot\nTo continue using commands you'll need to use them as slash commands",
+		format: "org.matrix.custom.html",
+		formatted_body: "🔀 <strong>Chewey Bot Official Server #announcements</strong><br>All text based commands are now inactive on Chewey Bot<br>To continue using commands you'll need to use them as slash commands"
+	}])
+})

d2m/converters/pins-to-list.js

@@ -0,0 +1,19 @@
+// @ts-check
+
+const {select} = require("../../passthrough")
+
+/**
+ * @param {import("discord-api-types/v10").RESTGetAPIChannelPinsResult} pins
+ */
+function pinsToList(pins) {
+	/** @type {string[]} */
+	const result = []
+	for (const message of pins) {
+		const eventID = select("event_message", "event_id", {message_id: message.id, part: 0}).pluck().get()
+		if (eventID) result.push(eventID)
+	}
+	result.reverse()
+	return result
+}
+
+module.exports.pinsToList = pinsToList

d2m/converters/pins-to-list.test.js

@@ -0,0 +1,12 @@
+const {test} = require("supertape")
+const data = require("../../test/data")
+const {pinsToList} = require("./pins-to-list")
+
+test("pins2list: converts known IDs, ignores unknown IDs", t => {
+	const result = pinsToList(data.pins.faked)
+	t.deepEqual(result, [
+		"$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4",
+		"$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA",
+		"$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg"
+	])
+})

d2m/converters/remove-reaction.js

@@ -5,14 +5,14 @@ const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, select} = passthrough
-/** @type {import("../../matrix/utils")} */
-const utils = sync.require("../../matrix/utils")
+/** @type {import("../../m2d/converters/utils")} */
+const utils = sync.require("../../m2d/converters/utils")
 
 /**
  * @typedef ReactionRemoveRequest
  * @prop {string} eventID
  * @prop {string | null} mxid
- * @prop {bigint} [hash]
+ * @prop {BigInt} [hash]
  */
 
 /**

d2m/converters/remove-reaction.test.js

@@ -10,9 +10,7 @@ function fakeSpecificReactionRemoval(userID, emoji, emojiID) {
 		channel_id: "THE_CHANNEL",
 		message_id: "THE_MESSAGE",
 		user_id: userID,
-		emoji: {id: emojiID, name: emoji},
-		burst: false,
-		type: 0
+		emoji: {id: emojiID, name: emoji}
 	}
 }
 

d2m/converters/thread-to-announcement.js

@@ -4,9 +4,10 @@ const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
-/** @type {import("../../matrix/utils")} */
-const mxUtils = sync.require("../../matrix/utils")
-const {reg} = require("../../matrix/read-registration.js")
+/** @type {import("../../matrix/read-registration")} */
+const reg = sync.require("../../matrix/read-registration.js")
+/** @type {import("../../m2d/converters/utils")} */
+const mxUtils = sync.require("../../m2d/converters/utils")
 
 const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
 
@@ -32,10 +33,13 @@ async function threadToAnnouncement(parentRoomID, threadRoomID, creatorMxid, thr
 	const template = creatorMxid ? "started a thread:" : "Thread started:"
 	const via = await mxUtils.getViaServersQuery(threadRoomID, di.api)
 	let body = `${template} ${thread.name} https://matrix.to/#/${threadRoomID}?${via.toString()}`
+	let html = `${template} <a href="https://matrix.to/#/${threadRoomID}?${via.toString()}">${thread.name}</a>`
 
 	return {
 		msgtype,
 		body,
+		format: "org.matrix.custom.html",
+		formatted_body: html,
 		"m.mentions": {},
 		...context
 	}

d2m/converters/thread-to-announcement.test.js

@@ -1,8 +1,7 @@
 const {test} = require("supertape")
 const {threadToAnnouncement} = require("./thread-to-announcement")
-const data = require("../../../test/data")
+const data = require("../../test/data")
 const Ty = require("../../types")
-const {mockGetEffectivePower} = require("../../matrix/utils.test")
 
 /**
  * @param {string} roomID
@@ -31,7 +30,13 @@ function mockGetEvent(t, roomID_in, eventID_in, outer) {
 }
 
 const viaApi = {
-	getEffectivePower: mockGetEffectivePower(),
+	async getStateEvent(roomID, type, key) {
+		return {
+			users: {
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}
+	},
 	async getJoinedMembers(roomID) {
 		return {
 			joined: {
@@ -50,6 +55,8 @@ test("thread2announcement: no known creator, no branched from event", async t =>
 	t.deepEqual(content, {
 		msgtype: "m.text",
 		body: "Thread started: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
+		format: "org.matrix.custom.html",
+		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {}
 	})
 })
@@ -62,6 +69,8 @@ test("thread2announcement: known creator, no branched from event", async t => {
 	t.deepEqual(content, {
 		msgtype: "m.emote",
 		body: "started a thread: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
+		format: "org.matrix.custom.html",
+		formatted_body: `started a thread: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {}
 	})
 })
@@ -86,6 +95,8 @@ test("thread2announcement: no known creator, branched from discord event", async
 	t.deepEqual(content, {
 		msgtype: "m.text",
 		body: "Thread started: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
+		format: "org.matrix.custom.html",
+		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {},
 		"m.relates_to": {
 			"m.in_reply_to": {
@@ -115,6 +126,8 @@ test("thread2announcement: known creator, branched from discord event", async t
 	t.deepEqual(content, {
 		msgtype: "m.emote",
 		body: "started a thread: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
+		format: "org.matrix.custom.html",
+		formatted_body: `started a thread: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {},
 		"m.relates_to": {
 			"m.in_reply_to": {
@@ -144,6 +157,8 @@ test("thread2announcement: no known creator, branched from matrix event", async
 	t.deepEqual(content, {
 		msgtype: "m.text",
 		body: "Thread started: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
+		format: "org.matrix.custom.html",
+		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {
 			user_ids: ["@cadence:cadence.moe"]
 		},

d2m/converters/user-to-mxid.js

@@ -1,8 +1,6 @@
 // @ts-check
 
 const assert = require("assert")
-const {reg} = require("../../matrix/read-registration")
-const Ty = require("../../types")
 
 const passthrough = require("../../passthrough")
 const {select} = passthrough
@@ -14,22 +12,18 @@ const SPECIAL_USER_MAPPINGS = new Map([
 /**
  * Downcased and stripped username. Can only include a basic set of characters.
  * https://spec.matrix.org/v1.6/appendices/#user-identifiers
- * @param {import("discord-api-types/v10").APIUser | Ty.WebhookAuthor} user
+ * @param {import("discord-api-types/v10").APIUser} user
  * @returns {string} localpart
  */
 function downcaseUsername(user) {
 	// First, try to convert the username to the set of allowed characters
 	let downcased = user.username.toLowerCase()
-		// spaces and slashes to underscores...
-		.replace(/[ /]/g, "_")
+		// spaces to underscores...
+		.replace(/ /g, "_")
 		// remove disallowed characters...
-		.replace(/[^a-z0-9._=-]*/g, "")
+		.replace(/[^a-z0-9._=/-]*/g, "")
 		// remove leading and trailing dashes and underscores...
 		.replace(/(?:^[_-]*|[_-]*$)/g, "")
-	// If requested, also make the Discord user ID part of the username
-	if (reg.ooye.include_user_id_in_mxid) {
-		downcased = user.id + "_" + downcased
-	}
 	// The new length must be at least 2 characters (in other words, it should have some content)
 	if (downcased.length < 2) {
 		downcased = user.id
@@ -65,7 +59,7 @@ function userToSimName(user) {
 	}
 
 	// 1. Is sim user already registered?
-	const existing = select("sim", "user_id", {user_id: user.id}).pluck().get()
+	const existing = select("sim", "sim_name", {user_id: user.id}).pluck().get()
 	assert.equal(existing, null, "Shouldn't try to create a new name for an existing sim")
 
 	// 2. Register based on username (could be new or old format)
@@ -86,49 +80,4 @@ function userToSimName(user) {
 	throw new Error(`Ran out of suggestions when generating sim name. downcased: "${downcased}"`)
 }
 
-/**
- * Webhooks have an ID specific to that webhook, but a single webhook can send messages with any user name.
- * The point of this feature (gated by guild_space webhook_profile) is to create persistent Matrix accounts for individual webhook "users".
- * This is convenient when using a bridge to a platform that does not assign persistent user IDs (e.g. IRC, Minecraft).
- * In this case, webhook "users" are disambiguated by their username (downcased).
- * @param {Ty.WebhookAuthor} author
- * @returns {string}
- */
-function webhookAuthorToFakeUserID(author) {
-	const downcased = downcaseUsername(author)
-	return `webhook_${downcased}`
-}
-
-function isWebhookUserID(userID) {
-	return userID.match(/^webhook_[a-z90-9._=/-]+$/)
-}
-
-/**
- * @param {Ty.WebhookAuthor} author
- * @returns {string}
- */
-function webhookAuthorToSimName(author) {
-	assert(!SPECIAL_USER_MAPPINGS.has(author.id), "Special users should have followed the other code path.")
-
-	// 1. Is sim user already registered?
-	const fakeUserID = webhookAuthorToFakeUserID(author)
-	const existing = select("sim", "user_id", {user_id: fakeUserID}).pluck().get()
-	assert.equal(existing, null, "Shouldn't try to create a new name for an existing sim")
-
-	// 2. Register based on username (could be new or old format)
-	const downcased = "webhook_" + downcaseUsername(author)
-
-	// Check for conflicts with already registered sims
-	const matches = select("sim", "sim_name", {}, "WHERE sim_name LIKE ? ESCAPE '@'").pluck().all(downcased + "%")
-	// Keep generating until we get a suggestion that doesn't conflict
-	for (const suggestion of generateLocalpartAlternatives([downcased])) {
-		if (!matches.includes(suggestion)) return suggestion
-	}
-	/* c8 ignore next */
-	throw new Error(`Ran out of suggestions when generating sim name. downcased: "${downcased}"`)
-}
-
 module.exports.userToSimName = userToSimName
-module.exports.webhookAuthorToFakeUserID = webhookAuthorToFakeUserID
-module.exports.webhookAuthorToSimName = webhookAuthorToSimName
-module.exports.isWebhookUserID = isWebhookUserID

d2m/converters/user-to-mxid.test.js

@@ -1,8 +1,8 @@
 const {test} = require("supertape")
 const tryToCatch = require("try-to-catch")
 const assert = require("assert")
-const data = require("../../../test/data")
-const {userToSimName, webhookAuthorToSimName} = require("./user-to-mxid")
+const data = require("../../test/data")
+const {userToSimName} = require("./user-to-mxid")
 
 test("user2name: cannot create user for a webhook", async t => {
    const [error] = await tryToCatch(() => userToSimName({discriminator: "0000"}))
@@ -21,12 +21,8 @@ test("user2name: works on single emoji at the end", t => {
    t.equal(userToSimName({username: "Melody 🎵", discriminator: "2192"}), "melody")
 })
 
-test("user2name: works on really weird name", t => {
-   t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7")
-})
-
-test("user2name: treats slashes", t => {
-	t.equal(userToSimName({username: "Evil Lillith (she/her)", discriminator: "5892"}), "evil_lillith_she_her")
+test("user2name: works on crazy name", t => {
+   t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7//")
 })
 
 test("user2name: adds discriminator if name is unavailable (old tag format)", t => {
@@ -48,18 +44,3 @@ test("user2name: uses ID when name has only disallowed characters", t => {
 test("user2name: works on special user", t => {
 	t.equal(userToSimName(data.user.clyde_ai), "clyde_ai")
 })
-
-test("webhook author: can generate sim names", t => {
-	t.equal(webhookAuthorToSimName({
-		username: "Cadence, Maid of Creation, Eye of Clarity, Empress of Hope ☆",
-		avatar: null,
-		id: "123"
-	}), "webhook_cadence_maid_of_creation_eye_of_clarity_empress_of_hope")
-})
-
-test("user2name: includes ID if requested in config", t => {
-	const {reg} = require("../../matrix/read-registration")
-	reg.ooye.include_user_id_in_mxid = true
-	t.equal(userToSimName({username: "Harry Styles!", discriminator: "0001", id: "123456"}), "123456_harry_styles")
-	t.equal(userToSimName({username: "f***", discriminator: "0001", id: "123456"}), "123456_f")
-})

d2m/discord-client.js

@@ -1,17 +1,12 @@
 // @ts-check
 
-const DiscordTypes = require("discord-api-types/v10")
-const {Endpoints, SnowTransfer} = require("snowtransfer")
-const {reg} = require("../matrix/read-registration")
-const {Client: CloudStorm} = require("cloudstorm")
-
-// @ts-ignore
-Endpoints.BASE_HOST = reg.ooye.discord_origin || "https://discord.com"; Endpoints.CDN_URL = reg.ooye.discord_cdn_origin || "https://cdn.discordapp.com"
+const { SnowTransfer } = require("snowtransfer")
+const { Client: CloudStorm } = require("cloudstorm")
 
 const passthrough = require("../passthrough")
-const {sync} = passthrough
+const { sync } = passthrough
 
-/** @type {import("./discord-packets")} */
+/** @type {typeof import("./discord-packets")} */
 const discordPackets = sync.require("./discord-packets")
 
 class DiscordClient {
@@ -20,34 +15,32 @@ class DiscordClient {
 	 * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
 	 */
 	constructor(discordToken, listen = "full") {
-		/** @type {import("cloudstorm").IClientOptions["intents"]} */
-		const intents = [
-			"DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING",
-			"GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS", "GUILD_MESSAGE_POLLS",
-			"MESSAGE_CONTENT"
-		]
-		if (reg.ooye.receive_presences !== false) intents.push("GUILD_PRESENCES")
 		this.discordToken = discordToken
 		this.snow = new SnowTransfer(discordToken)
 		this.cloud = new CloudStorm(discordToken, {
 			shards: [0],
+			reconnect: true,
 			snowtransferInstance: this.snow,
-			intents,
+			intents: [
+				"DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING",
+				"GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS",
+				"MESSAGE_CONTENT"
+			],
 			ws: {
 				compress: false,
 				encoding: "json"
 			}
 		})
 		this.ready = false
-		/** @type {DiscordTypes.APIUser} */
+		/** @type {import("discord-api-types/v10").APIUser} */
 		// @ts-ignore avoid setting as or null because we know we need to wait for ready anyways
 		this.user = null
-		/** @type {Pick<DiscordTypes.APIApplication, "id" | "flags">} */
+		/** @type {Pick<import("discord-api-types/v10").APIApplication, "id" | "flags">} */
 		// @ts-ignore
 		this.application = null
-		/** @type {Map<string, DiscordTypes.APIChannel>} */
+		/** @type {Map<string, import("discord-api-types/v10").APIChannel>} */
 		this.channels = new Map()
-		/** @type {Map<string, DiscordTypes.APIGuild & {members: DiscordTypes.APIGuildMember[]}>} */ // we get members from the GUILD_CREATE and we do maintain it
+		/** @type {Map<string, import("discord-api-types/v10").APIGuild>} */
 		this.guilds = new Map()
 		/** @type {Map<string, Array<string>>} */
 		this.guildChannelMap = new Map()

d2m/discord-packets.js

@@ -4,7 +4,7 @@
 
 const DiscordTypes = require("discord-api-types/v10")
 const passthrough = require("../passthrough")
-const {sync, db} = passthrough
+const { sync } = passthrough
 
 const utils = {
 	/**
@@ -16,8 +16,6 @@ const utils = {
 		// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
 		/** @type {typeof import("./event-dispatcher")} */
 		const eventDispatcher = sync.require("./event-dispatcher")
-		/** @type {import("../discord/register-interactions")} */
-		const interactions = sync.require("../discord/register-interactions")
 
 		// Client internals, keep track of the state we need
 		if (message.t === "READY") {
@@ -28,7 +26,6 @@ const utils = {
 			console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`)
 
 		} else if (message.t === "GUILD_CREATE") {
-			message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web
 			client.guilds.set(message.d.id, message.d)
 			const arr = []
 			client.guildChannelMap.set(message.d.id, arr)
@@ -44,17 +41,10 @@ const utils = {
 				arr.push(thread.id)
 				client.channels.set(thread.id, thread)
 			}
-
 			if (listen === "full") {
-				try {
-					interactions.registerInteractions()
-					await eventDispatcher.checkMissedExpressions(message.d)
-					await eventDispatcher.checkMissedPins(client, message.d)
-					await eventDispatcher.checkMissedMessages(client, message.d)
-				} catch (e) {
-					console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:")
-					console.error(e)
-				}
+				eventDispatcher.checkMissedExpressions(message.d)
+				eventDispatcher.checkMissedPins(client, message.d)
+				eventDispatcher.checkMissedMessages(client, message.d)
 			}
 
 		} else if (message.t === "GUILD_UPDATE") {
@@ -97,19 +87,9 @@ const utils = {
 				}
 			}
 
-		} else if (message.t === "GUILD_MEMBER_UPDATE") {
-			const guild = client.guilds.get(message.d.guild_id)
-			const member = guild?.members.find(m => m.user.id === message.d.user.id)
-			if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones
-				Object.assign(member, message.d)
-			}
-
 		} else if (message.t === "THREAD_CREATE") {
 			client.channels.set(message.d.id, message.d)
-			if (message.d["guild_id"]) {
-				const channels = client.guildChannelMap.get(message.d["guild_id"])
-				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
-			}
+
 
 		} else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") {
 			client.channels.set(message.d.id, message.d)
@@ -131,39 +111,71 @@ const utils = {
 			client.guildChannelMap.delete(message.d.id)
 
 
-		} else if (message.t === "CHANNEL_CREATE") {
-			client.channels.set(message.d.id, message.d)
-			if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
-				const channels = client.guildChannelMap.get(message.d["guild_id"])
-				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
-			}
-
-		} else if (message.t === "CHANNEL_DELETE") {
-			client.channels.delete(message.d.id)
-			if (message.d["guild_id"]) {
-				const channels = client.guildChannelMap.get(message.d["guild_id"])
-				if (channels) {
-					const previous = channels.indexOf(message.d.id)
-					if (previous !== -1) channels.splice(previous, 1)
+		} else if (message.t === "CHANNEL_CREATE" || message.t === "CHANNEL_DELETE") {
+			if (message.t === "CHANNEL_CREATE") {
+				client.channels.set(message.d.id, message.d)
+				if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
+					const channels = client.guildChannelMap.get(message.d["guild_id"])
+					if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
+				}
+			} else {
+				client.channels.delete(message.d.id)
+				if (message.d["guild_id"]) {
+					const channels = client.guildChannelMap.get(message.d["guild_id"])
+					if (channels) {
+						const previous = channels.indexOf(message.d.id)
+						if (previous !== -1) channels.splice(previous, 1)
+					}
 				}
 			}
 		}
 
 		// Event dispatcher for OOYE bridge operations
-		if (listen === "full" && message.t) {
+		if (listen === "full") {
 			try {
-				if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
+				if (message.t === "GUILD_UPDATE") {
+					await eventDispatcher.onGuildUpdate(client, message.d)
+
+				} else if (message.t === "GUILD_EMOJIS_UPDATE" || message.t === "GUILD_STICKERS_UPDATE") {
+					await eventDispatcher.onExpressionsUpdate(client, message.d)
+
+				} else if (message.t === "CHANNEL_UPDATE") {
+					await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false)
+
+				} else if (message.t === "CHANNEL_PINS_UPDATE") {
+					await eventDispatcher.onChannelPinsUpdate(client, message.d)
+
+				} else if (message.t === "THREAD_CREATE") {
+					// @ts-ignore
+					await eventDispatcher.onThreadCreate(client, message.d)
+
+				} else if (message.t === "THREAD_UPDATE") {
+					await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true)
+
+				} else if (message.t === "MESSAGE_CREATE") {
+					await eventDispatcher.onMessageCreate(client, message.d)
+
+				} else if (message.t === "MESSAGE_UPDATE") {
+					await eventDispatcher.onMessageUpdate(client, message.d)
+
+				} else if (message.t === "MESSAGE_DELETE") {
+					await eventDispatcher.onMessageDelete(client, message.d)
+
+				} else if (message.t === "MESSAGE_DELETE_BULK") {
+					await eventDispatcher.onMessageDeleteBulk(client, message.d)
+
+				} else if (message.t === "TYPING_START") {
+					await eventDispatcher.onTypingStart(client, message.d)
+
+				} else if (message.t === "MESSAGE_REACTION_ADD") {
+					await eventDispatcher.onReactionAdd(client, message.d)
+
+				} else if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
 					await eventDispatcher.onSomeReactionsRemoved(client, message.d)
-
-				} else if (message.t === "INTERACTION_CREATE") {
-					await interactions.dispatchInteraction(message.d)
-
-				} else if (message.t in eventDispatcher) {
-					await eventDispatcher[message.t](client, message.d)
 				}
 			} catch (e) {
 				// Let OOYE try to handle errors too
-				await eventDispatcher.onError(client, e, message)
+				eventDispatcher.onError(client, e, message)
 			}
 		}
 	}

d2m/event-dispatcher.js

@@ -2,6 +2,7 @@
 
 const assert = require("assert").strict
 const DiscordTypes = require("discord-api-types/v10")
+const util = require("util")
 const {sync, db, select, from} = require("../passthrough")
 
 /** @type {import("./actions/send-message")}) */
@@ -26,22 +27,19 @@ const updatePins = sync.require("./actions/update-pins")
 const api = sync.require("../matrix/api")
 /** @type {import("../discord/utils")} */
 const dUtils = sync.require("../discord/utils")
+/** @type {import("../discord/discord-command-handler")}) */
+const discordCommandHandler = sync.require("../discord/discord-command-handler")
+/** @type {import("../m2d/converters/utils")} */
+const mxUtils = require("../m2d/converters/utils")
 /** @type {import("./actions/speedbump")} */
 const speedbump = sync.require("./actions/speedbump")
-/** @type {import("./actions/retrigger")} */
-const retrigger = sync.require("./actions/retrigger")
-/** @type {import("./actions/set-presence")} */
-const setPresence = sync.require("./actions/set-presence")
-/** @type {import("./actions/poll-vote")} */
-const vote = sync.require("./actions/poll-vote")
-/** @type {import("../m2d/event-dispatcher")} */
-const matrixEventDispatcher = sync.require("../m2d/event-dispatcher")
-/** @type {import("../discord/interactions/matrix-info")} */
-const matrixInfoInteraction = sync.require("../discord/interactions/matrix-info")
 
-const {Semaphore} = require("@chriscdn/promise-semaphore")
+/** @type {any} */ // @ts-ignore bad types from semaphore
+const Semaphore = require("@chriscdn/promise-semaphore")
 const checkMissedPinsSema = new Semaphore()
 
+let lastReportedEvent = 0
+
 // Grab Discord events we care about for the bridge, check them, and pass them on
 
 module.exports = {
@@ -50,17 +48,49 @@ module.exports = {
 	 * @param {Error} e
 	 * @param {import("cloudstorm").IGatewayMessage} gatewayMessage
 	 */
-	async onError(client, e, gatewayMessage) {
+	onError(client, e, gatewayMessage) {
+		console.error("hit event-dispatcher's error handler with this exception:")
+		console.error(e) // TODO: also log errors into a file or into the database, maybe use a library for this? or just wing it? definitely need to be able to store the formatted event body to load back in later
+		console.error(`while handling this ${gatewayMessage.t} gateway event:`)
+		console.dir(gatewayMessage.d, {depth: null})
+
 		if (gatewayMessage.t === "TYPING_START") return
 
-		matrixEventDispatcher.printError(gatewayMessage.t, "Discord", e, gatewayMessage)
+		if (Date.now() - lastReportedEvent < 5000) return
+		lastReportedEvent = Date.now()
 
 		const channelID = gatewayMessage.d["channel_id"]
 		if (!channelID) return
 		const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
 		if (!roomID) return
 
-		await matrixEventDispatcher.sendError(roomID, "Discord", gatewayMessage.t, e, gatewayMessage)
+		let stackLines = null
+		if (e.stack) {
+			stackLines = e.stack.split("\n")
+			let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/"))
+			if (cloudstormLine !== -1) {
+				stackLines = stackLines.slice(0, cloudstormLine - 2)
+			}
+		}
+
+		const builder = new mxUtils.MatrixStringBuilder()
+		builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 <strong>Bridged event from Discord not delivered</strong>")
+		builder.addLine(`Gateway event: ${gatewayMessage.t}`)
+		builder.addLine(e.toString())
+		if (stackLines) {
+			builder.addLine(`Error trace:\n${stackLines.join("\n")}`, `<details><summary>Error trace</summary><pre>${stackLines.join("\n")}</pre></details>`)
+		}
+		builder.addLine("", `<details><summary>Original payload</summary><pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`)
+		api.sendEvent(roomID, "m.room.message", {
+			...builder.get(),
+			"moe.cadence.ooye.error": {
+				source: "discord",
+				payload: gatewayMessage
+			},
+			"m.mentions": {
+				user_ids: ["@cadence:cadence.moe"]
+			}
+		})
 	},
 
 	/**
@@ -73,28 +103,20 @@ module.exports = {
 	async checkMissedMessages(client, guild) {
 		if (guild.unavailable) return
 		const bridgedChannels = select("channel_room", "channel_id").pluck().all()
-		const preparedExists = from("message_room").join("historical_channel_room", "historical_room_index").pluck("message_id").and("WHERE reference_channel_id = ? LIMIT 1").prepare()
-		const preparedGet = select("event_message", "event_id", {}, "WHERE message_id = ?").pluck()
-		/** @type {(DiscordTypes.APIChannel & {type: DiscordTypes.GuildChannelType})[]} */
-		let channels = []
-		channels = channels.concat(guild.channels, guild.threads)
-		for (const channel of channels) {
+		const prepared = select("event_message", "event_id", {}, "WHERE message_id = ?").pluck()
+		for (const channel of guild.channels.concat(guild.threads)) {
 			if (!bridgedChannels.includes(channel.id)) continue
 			if (!("last_message_id" in channel) || !channel.last_message_id) continue
-
-			// Skip if channel is already up-to-date
-			const latestWasBridged = preparedGet.get(channel.last_message_id)
+			const latestWasBridged = prepared.get(channel.last_message_id)
 			if (latestWasBridged) continue
 
-			// Skip if channel was just added to the bridge (there's no place to resume from if it's brand new)
-			if (!preparedExists.get(channel.id)) continue
-
 			// Permissions check
 			const member = guild.members.find(m => m.user?.id === client.user.id)
 			if (!member) return
 			if (!("permission_overwrites" in channel)) continue
-			const permissions = dUtils.getPermissions(guild.id, member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look back in this channel
+			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
+			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
+			if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel
 
 			/** More recent messages come first. */
 			// console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
@@ -110,28 +132,17 @@ module.exports = {
 				}
 			}
 			let latestBridgedMessageIndex = messages.findIndex(m => {
-				return preparedGet.get(m.id)
+				return prepared.get(m.id)
 			})
 			// console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
 			if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date.
-
-			// We get member data so that we can accurately update any changes to nickname or permissions that have occurred in the meantime
-			// The rate limit is lax enough that the backlog will still be pretty quick (at time of writing, 5 per 1 second per guild)
-			/** @type {Map<string, DiscordTypes.APIGuildMember | undefined>} id -> member: cache members for the run because people talk to each other */
-			const members = new Map()
-
-			// Send in order
 			for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) {
-				const message = messages[i]
-
-				if (!members.has(message.author.id)) members.set(message.author.id, await client.snow.guild.getGuildMember(guild.id, message.author.id).catch(() => undefined))
-				await module.exports.MESSAGE_CREATE(client, {
+				const simulatedGatewayDispatchData = {
 					guild_id: guild.id,
-					member: members.get(message.author.id),
-					// @ts-ignore
 					backfill: true,
-					...message
-				})
+					...messages[i]
+				}
+				await module.exports.onMessageCreate(client, simulatedGatewayDispatchData)
 			}
 		}
 	},
@@ -152,8 +163,9 @@ module.exports = {
 			const lastPin = updatePins.convertTimestamp(channel.last_pin_timestamp)
 
 			// Permissions check
-			const permissions = dUtils.getPermissions(guild.id, member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look up the pins in this channel
+			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
+			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
+			if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel
 
 			const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get()
 			if (!row) continue // Only care about already bridged channels
@@ -169,7 +181,7 @@ module.exports = {
 	 */
 	async checkMissedExpressions(guild) {
 		const data = {guild_id: guild.id, ...guild}
-		await createSpace.syncSpaceExpressions(data, true)
+		createSpace.syncSpaceExpressions(data, true)
 	},
 
 	/**
@@ -178,10 +190,10 @@ module.exports = {
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.APIThreadChannel} thread
 	 */
-	async THREAD_CREATE(client, thread) {
+	async onThreadCreate(client, thread) {
 		const channelID = thread.parent_id || undefined
 		const parentRoomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
-		if (!parentRoomID) return // Not interested in a thread if we aren't interested in its wider channel (won't autocreate)
+		if (!parentRoomID) return // Not interested in a thread if we aren't interested in its wider channel
 		const threadRoomID = await createRoom.syncRoom(thread.id) // Create room (will share the same inflight as the initial message to the thread)
 		await announceThread.announceThread(parentRoomID, threadRoomID, thread)
 	},
@@ -190,78 +202,42 @@ module.exports = {
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayGuildUpdateDispatchData} guild
 	 */
-	async GUILD_UPDATE(client, guild) {
+	async onGuildUpdate(client, guild) {
 		const spaceID = select("guild_space", "space_id", {guild_id: guild.id}).pluck().get()
 		if (!spaceID) return
 		await createSpace.syncSpace(guild)
 	},
 
-	/**
-	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayGuildRoleUpdateDispatchData} data
-	 */
-	async GUILD_ROLE_UPDATE(client, data) {
-		const guild = client.guilds.get(data.guild_id)
-		if (!guild) return
-		const spaceID = select("guild_space", "space_id", {guild_id: data.guild_id}).pluck().get()
-		if (!spaceID) return
-
-		if (data.role.id === data.guild_id) { // @everyone role changed - find a way to do this more efficiently in the future to handle many role updates
-			await createSpace.syncSpaceFully(guild)
-		}
-	},
-
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayChannelUpdateDispatchData} channelOrThread
+	 * @param {boolean} isThread
 	 */
-	async CHANNEL_UPDATE(client, channelOrThread) {
+	async onChannelOrThreadUpdate(client, channelOrThread, isThread) {
 		const roomID = select("channel_room", "room_id", {channel_id: channelOrThread.id}).pluck().get()
 		if (!roomID) return // No target room to update the data on
 		await createRoom.syncRoom(channelOrThread.id)
 	},
 
-	/**
-	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayChannelUpdateDispatchData} thread
-	 */
-	async THREAD_UPDATE(client, thread) {
-		await module.exports.CHANNEL_UPDATE(client, thread)
-	},
-
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayChannelPinsUpdateDispatchData} data
 	 */
-	async CHANNEL_PINS_UPDATE(client, data) {
+	async onChannelPinsUpdate(client, data) {
 		const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
 		if (!roomID) return // No target room to update pins in
 		const convertedTimestamp = updatePins.convertTimestamp(data.last_pin_timestamp)
 		await updatePins.updatePins(data.channel_id, roomID, convertedTimestamp)
 	},
 
-	/**
-	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayChannelDeleteDispatchData} channel
-	 */
-	async CHANNEL_DELETE(client, channel) {
-		const guildID = channel["guild_id"]
-		if (!guildID) return // channel must have been a DM channel or something
-		const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get()
-		if (!roomID) return // channel wasn't being bridged in the first place
-		// @ts-ignore
-		await createRoom.unbridgeChannel(channel, guildID)
-	},
-
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
 	 */
-	async MESSAGE_CREATE(client, message) {
+	async onMessageCreate(client, message) {
 		if (message.author.username === "Deleted User") return // Nothing we can do for deleted users.
 		const channel = client.channels.get(message.channel_id)
 		if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
-
 		const guild = client.guilds.get(channel.guild_id)
 		assert(guild)
 
@@ -270,68 +246,52 @@ module.exports = {
 			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
 		}
 
-		if (dUtils.isEphemeralMessage(message)) return // Ephemeral messages are for the eyes of the receiver only!
-
-		if (!createRoom.existsOrAutocreatable(channel, guild.id)) return // Check that the sending-to room exists or is autocreatable
-
 		const {affected, row} = await speedbump.maybeDoSpeedbump(message.channel_id, message.id)
 		if (affected) return
 
 		// @ts-ignore
-		await sendMessage.sendMessage(message, channel, guild, row)
-
-		retrigger.messageFinishedBridging(message.id)
+		await sendMessage.sendMessage(message, guild, row),
+		await discordCommandHandler.execute(message, channel, guild)
 	},
 
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageUpdateDispatchData} data
 	 */
-	async MESSAGE_UPDATE(client, data) {
-		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
-		// If the message content is a string then it includes all interesting fields and is meaningful.
-		// Otherwise, if there are embeds, then the system generated URL preview embeds.
-		if (!(typeof data.content === "string" || "embeds" in data || "components" in data)) return
-
-		if (dUtils.isEphemeralMessage(data)) return // Ephemeral messages are for the eyes of the receiver only!
+	async onMessageUpdate(client, data) {
+		if (data.webhook_id) {
+			const row = select("webhook", "webhook_id", {webhook_id: data.webhook_id}).pluck().get()
+			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
+		}
 
 		// Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from.
 		const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id)
 		if (affected) return
 
-		if (!row) {
-			// Check that the sending-to room exists, and deal with Eventual Consistency(TM)
-			if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_UPDATE, client, data)) return
+		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
+		// If the message content is a string then it includes all interesting fields and is meaningful.
+		if (typeof data.content === "string") {
+			/** @type {DiscordTypes.GatewayMessageCreateDispatchData} */
+			// @ts-ignore
+			const message = data
+
+			const channel = client.channels.get(message.channel_id)
+			if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
+			const guild = client.guilds.get(channel.guild_id)
+			assert(guild)
+			// @ts-ignore
+			await editMessage.editMessage(message, guild, row)
 		}
-
-		/** @type {DiscordTypes.GatewayMessageCreateDispatchData} */
-		// @ts-ignore
-		const message = data
-		const channel = client.channels.get(message.channel_id)
-		if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
-		const guild = client.guilds.get(channel.guild_id)
-		assert(guild)
-
-		// @ts-ignore
-		await retrigger.pauseChanges(message.id, editMessage.editMessage(message, guild, row))
 	},
 
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageReactionAddDispatchData} data
 	 */
-	async MESSAGE_REACTION_ADD(client, data) {
+	async onReactionAdd(client, data) {
 		if (data.user_id === client.user.id) return // m2d reactions are added by the discord bot user - do not reflect them back to matrix.
-		if (data.emoji.name === "❓" && select("event_message", "message_id", {message_id: data.message_id, source: 0, part: 0}).get()) { // source 0 = matrix
-			const guild_id = data.guild_id ?? client.channels.get(data.channel_id)?.["guild_id"]
-			await Promise.all([
-				client.snow.channel.deleteReaction(data.channel_id, data.message_id, data.emoji.name).catch(() => {}),
-				// @ts-ignore - this is all you need for it to do a matrix-side lookup
-				matrixInfoInteraction.dm({guild_id, data: {target_id: data.message_id}, member: {user: {id: data.user_id}}})
-			])
-		} else {
-			await addReaction.addReaction(data)
-		}
+		discordCommandHandler.onReactionAdd(data)
+		await addReaction.addReaction(data)
 	},
 
 	/**
@@ -346,25 +306,24 @@ module.exports = {
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageDeleteDispatchData} data
 	 */
-	async MESSAGE_DELETE(client, data) {
+	async onMessageDelete(client, data) {
 		speedbump.onMessageDelete(data.id)
-		if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_DELETE, client, data)) return
 		await deleteMessage.deleteMessage(data)
 	},
 
-	/**
+		/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayMessageDeleteBulkDispatchData} data
 	 */
-	async MESSAGE_DELETE_BULK(client, data) {
-		await deleteMessage.deleteMessageBulk(data)
-	},
+		async onMessageDeleteBulk(client, data) {
+			await deleteMessage.deleteMessageBulk(data)
+		},
 
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayTypingStartDispatchData} data
 	 */
-	async TYPING_START(client, data) {
+	async onTypingStart(client, data) {
 		const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
 		if (!roomID) return
 		const mxid = from("sim").join("sim_member", "mxid").where({user_id: data.user_id, room_id: roomID}).pluck("mxid").get()
@@ -377,41 +336,9 @@ module.exports = {
 
 	/**
 	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayGuildEmojisUpdateDispatchData} data
+	 * @param {DiscordTypes.GatewayGuildEmojisUpdateDispatchData | DiscordTypes.GatewayGuildStickersUpdateDispatchData} data
 	 */
-	async GUILD_EMOJIS_UPDATE(client, data) {
+	async onExpressionsUpdate(client, data) {
 		await createSpace.syncSpaceExpressions(data, false)
-	},
-
-	/**
-	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayGuildStickersUpdateDispatchData} data
-	 */
-	async GUILD_STICKERS_UPDATE(client, data) {
-		await createSpace.syncSpaceExpressions(data, false)
-	},
-
-	/**
-	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayMessagePollVoteDispatchData} data
-	 */
-	async MESSAGE_POLL_VOTE_ADD(client, data) {
-		if (retrigger.eventNotFoundThenRetrigger(data.message_id, module.exports.MESSAGE_POLL_VOTE_ADD, client, data)) return
-		await vote.addVote(data)
-	},
-
-	async MESSAGE_POLL_VOTE_REMOVE(client, data) {
-		if (retrigger.eventNotFoundThenRetrigger(data.message_id, module.exports.MESSAGE_POLL_VOTE_REMOVE, client, data)) return
-		await vote.removeVote(data)
-	},
-
-	/**
-	 * @param {import("./discord-client")} client
-	 * @param {DiscordTypes.GatewayPresenceUpdateDispatchData} data
-	 */
-	PRESENCE_UPDATE(client, data) {
-		const status = data.status
-		if (!status) return
-		setPresence.presenceTracker.incomingPresence(data.user.id, data.guild_id, status)
 	}
 }

db/migrate.js

@@ -6,8 +6,7 @@ const {join} = require("path")
 async function migrate(db) {
 	let files = fs.readdirSync(join(__dirname, "migrations"))
 	files = files.sort()
-	db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL, PRIMARY KEY (filename)) WITHOUT ROWID").run()
-	/** @type {string} */
+	db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run()
 	let progress = db.prepare("SELECT * FROM migration").pluck().get()
 	if (!progress) {
 		progress = ""
@@ -38,8 +37,6 @@ async function migrate(db) {
 	if (migrationRan) {
 		console.log("Database migrations all done.")
 	}
-
-	db.pragma("foreign_keys = on")
 }
 
 module.exports.migrate = migrate

db/migrations/0002-optimise-profile-content.up.js

@@ -3,7 +3,6 @@ module.exports = async function(db) {
 	const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all()
 	const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?")
 	db.transaction(() => {
-		/* c8 ignore next 6 */
 		for (let s of contents) {
 			let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s))
 			const unsignedHash = hasher.h64Raw(b)

db/orm-defs.d.ts

@@ -1,9 +1,4 @@
 export type Models = {
-	auto_emoji: {
-		name: string
-		emoji_id: string
-	}
-
 	channel_room: {
 		channel_id: string
 		room_id: string
@@ -15,20 +10,6 @@ export type Models = {
 		speedbump_id: string | null
 		speedbump_webhook_id: string | null
 		speedbump_checked: number | null
-		guild_id: string | null
-		custom_topic: number
-	}
-
-	direct: {
-		mxid: string
-		room_id: string
-	}
-
-	emoji: {
-		emoji_id: string
-		name: string
-		animated: number
-		mxc_url: string
 	}
 
 	event_message: {
@@ -50,29 +31,6 @@ export type Models = {
 		guild_id: string
 		space_id: string
 		privacy_level: number
-		presence: 0 | 1
-		url_preview: 0 | 1
-		webhook_profile: 0 | 1
-	}
-
-	guild_active: {
-		guild_id: string
-		autocreate: 0 | 1
-	}
-
-	historical_channel_room: {
-		historical_room_index: number
-		reference_channel_id: string
-		room_id: string
-		upgraded_timestamp: number
-	}
-
-	invite: {
-		mxid: string
-		room_id: string
-		type: string | null
-		name: string | null
-		avatar: string | null
 	}
 
 	lottie: {
@@ -80,39 +38,23 @@ export type Models = {
 		mxc_url: string
 	}
 
-	media_proxy: {
-		permitted_hash: number
-	}
-
 	member_cache: {
 		room_id: string
 		mxid: string
 		displayname: string | null
-		avatar_url: string | null,
-		power_level: number
-		missing_profile: number | null
+		avatar_url: string | null
 	}
 
-	member_power: {
-		mxid: string
-		room_id: string
-		power_level: number
-	}
-
-	message_room: {
+	message_channel: {
 		message_id: string
-		historical_room_index: number
-	}
-
-	room_upgrade_pending: {
-		new_room_id: string
-		old_room_id: string
+		channel_id: string
 	}
 
+	// A sim is an account that is being simulated by the bridge to copy events from the other side. See d2m\actions\register-user.js for relevant behavior and comments.
 	sim: {
 		user_id: string
-		username: string
 		sim_name: string
+		localpart: string
 		mxid: string
 	}
 
@@ -134,32 +76,23 @@ export type Models = {
 		webhook_token: string
 	}
 
+	emoji: {
+		emoji_id: string
+		name: string
+		animated: number
+		mxc_url: string
+	}
+
 	reaction: {
 		hashed_event_id: number
 		message_id: string
 		encoded_emoji: string
-		original_encoding: string | null
 	}
 
-	poll: { // not actually in database yet
-		message_id: string
-		max_selections: number
-		question_text: string
-		is_closed: number
-	}
-
-	poll_option: {
-		message_id: string
-		matrix_option: string
-		discord_option: string | null
-		option_text: string // not actually in database yet
-		seq: number // not actually in database yet
-	}
-
-	poll_vote: {
-		message_id: string
-		matrix_option: string
-		discord_or_matrix_user_id: string
+	auto_emoji: {
+		name: string
+		emoji_id: string
+		guild_id: string
 	}
 }
 
@@ -168,12 +101,10 @@ export type Prepared<Row> = {
 	safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
 	raw: () => Prepared<Row[keyof Row][]>
 	all: (..._: any[]) => Row[]
-	get: (..._: any[]) => Row | null | undefined
+	get: (..._: any[]) => Row | null
 }
 
 export type AllKeys<U> = U extends any ? keyof U : never
 export type PickTypeOf<T, K extends AllKeys<T>> = T extends { [k in K]?: any } ? T[K] : never
 export type Merge<U> = {[x in AllKeys<U>]: PickTypeOf<U, x>}
 export type Nullable<T> = {[k in keyof T]: T[k] | null}
-export type Numberish<T> = {[k in keyof T]: T[k] extends number ? (number | bigint) : T[k]}
-export type ValueOrArray<T> = {[k in keyof T]: T[k][] | T[k]}

db/orm.js

@@ -8,20 +8,15 @@ const U = require("./orm-defs")
  * @template {keyof U.Models[Table]} Col
  * @param {Table} table
  * @param {Col[] | Col} cols
- * @param {Partial<U.ValueOrArray<U.Numberish<U.Models[Table]>>>} where
+ * @param {Partial<U.Models[Table]>} where
  * @param {string} [e]
  */
 function select(table, cols, where = {}, e = "") {
 	if (!Array.isArray(cols)) cols = [cols]
 	const parameters = []
 	const wheres = Object.entries(where).map(([col, value]) => {
-		if (Array.isArray(value)) {
-			parameters.push(...value)
-			return `"${col}" IN (` + Array(value.length).fill("?").join(", ") + ")"
-		} else {
-			parameters.push(value)
-			return `"${col}" = ?`
-		}
+		parameters.push(value)
+		return `"${col}" = ?`
 	})
 	const whereString = wheres.length ? " WHERE " + wheres.join(" AND ") : ""
 	/** @type {U.Prepared<Pick<U.Models[Table], Col>>} */
@@ -49,8 +44,6 @@ class From {
 		/** @private */
 		this.cols = []
 		/** @private */
-		this.makeColsSafe = true
-		/** @private */
 		this.using = []
 		/** @private */
 		this.isPluck = false
@@ -85,12 +78,6 @@ class From {
 		return r
 	}
 
-	selectUnsafe(...cols) {
-		this.cols = cols
-		this.makeColsSafe = false
-		return this
-	}
-
 	/**
 	 * @template {Col} Select
 	 * @param {Select} col
@@ -113,7 +100,7 @@ class From {
 	}
 
 	/**
-	 * @param {Partial<U.Numberish<U.Models[Table]>>} conditions
+	 * @param {Partial<U.Models[Table]>} conditions
 	 */
 	where(conditions) {
 		const wheres = Object.entries(conditions).map(([col, value]) => {
@@ -125,8 +112,7 @@ class From {
 	}
 
 	prepare() {
-		if (this.makeColsSafe) this.cols = this.cols.map(k => `"${k}"`)
-		let sql = `SELECT ${this.cols.join(", ")} FROM ${this.tables[0]} `
+		let sql = `SELECT ${this.cols.map(k => `"${k}"`).join(", ")} FROM ${this.tables[0]} `
 		for (let i = 1; i < this.tables.length; i++) {
 			const table = this.tables[i]
 			const col = this.using[i-1]

db/orm.test.js

@@ -1,28 +1,28 @@
 // @ts-check
 
 const {test} = require("supertape")
-const data = require("../../test/data")
+const data = require("../test/data")
 
 const {db, select, from} = require("../passthrough")
 
 test("orm: select: get works", t => {
-	const row = select("guild_space", "guild_id", {}, "WHERE space_id = ?").get("!jjmvBegULiLucuWEHU:cadence.moe")
+	const row = select("guild_space", "guild_id", {}, "WHERE space_id = ?").get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
 	t.equal(row?.guild_id, data.guild.general.id)
 })
 
 test("orm: from: get works", t => {
-	const row = from("guild_space").select("guild_id").and("WHERE space_id = ?").get("!jjmvBegULiLucuWEHU:cadence.moe")
+	const row = from("guild_space").select("guild_id").and("WHERE space_id = ?").get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
 	t.equal(row?.guild_id, data.guild.general.id)
 })
 
 test("orm: select: get pluck works", t => {
-	const guildID = select("guild_space", "guild_id", {}, "WHERE space_id = ?").pluck().get("!jjmvBegULiLucuWEHU:cadence.moe")
+	const guildID = select("guild_space", "guild_id", {}, "WHERE space_id = ?").pluck().get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
 	t.equal(guildID, data.guild.general.id)
 })
 
 test("orm: select: get, where and pluck works", t => {
-	const emojiName = select("emoji", "name", {emoji_id: "230201364309868544"}).pluck().get()
-	t.equal(emojiName, "hippo")
+	const channelID = select("message_channel", "channel_id", {message_id: "1128118177155526666"}).pluck().get()
+	t.equal(channelID, "112760669178241024")
 })
 
 test("orm: select: all, where and pluck works on multiple columns", t => {
@@ -30,13 +30,8 @@ test("orm: select: all, where and pluck works on multiple columns", t => {
 	t.deepEqual(names, ["cadence [they]"])
 })
 
-test("orm: select: in array works", t => {
-	const ids = select("emoji", "emoji_id", {name: ["online", "upstinky"]}).pluck().all()
-	t.deepEqual(ids, ["288858540888686602", "606664341298872324"])
-})
-
 test("orm: from: get pluck works", t => {
-	const guildID = from("guild_space").pluck("guild_id").and("WHERE space_id = ?").get("!jjmvBegULiLucuWEHU:cadence.moe")
+	const guildID = from("guild_space").pluck("guild_id").and("WHERE space_id = ?").get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
 	t.equal(guildID, data.guild.general.id)
 })
 
@@ -56,15 +51,5 @@ test("orm: from: join direction works", t => {
 	const hasNoOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
 	t.deepEqual(hasNoOwner, {user_id: "820865262526005258", proxy_owner_id: null})
 	const hasNoOwnerInner = from("sim").join("sim_proxy", "user_id", "inner").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
-	t.deepEqual(hasNoOwnerInner, undefined)
-})
-
-test("orm: select unsafe works (to select complex column names that can't be type verified)", t => {
-	const results = from("member_cache")
-		.join("member_power", "mxid")
-		.join("channel_room", "room_id") // only include rooms that are bridged
-		.and("where member_power.room_id = '*' and member_cache.power_level != member_power.power_level")
-		.selectUnsafe("mxid", "member_cache.room_id", "member_power.power_level")
-		.all()
-	t.equal(results[0].power_level, 150)
+	t.deepEqual(hasNoOwnerInner, null)
 })

discord/discord-command-handler.js

@@ -0,0 +1,273 @@
+// @ts-check
+
+const assert = require("assert").strict
+const util = require("util")
+const DiscordTypes = require("discord-api-types/v10")
+const reg = require("../matrix/read-registration")
+const {addbot} = require("../addbot")
+
+const {discord, sync, db, select} = require("../passthrough")
+/** @type {import("../matrix/api")}) */
+const api = sync.require("../matrix/api")
+/** @type {import("../matrix/file")} */
+const file = sync.require("../matrix/file")
+/** @type {import("../d2m/actions/create-space")} */
+const createSpace = sync.require("../d2m/actions/create-space")
+/** @type {import("./utils")} */
+const utils = sync.require("./utils")
+
+const PREFIX = "//"
+
+let buttons = []
+
+/**
+ * @param {string} channelID where to add the button
+ * @param {string} messageID where to add the button
+ * @param {string} emoji emoji to add as a button
+ * @param {string} userID only listen for responses from this user
+ * @returns {Promise<import("discord-api-types/v10").GatewayMessageReactionAddDispatchData>}
+ */
+async function addButton(channelID, messageID, emoji, userID) {
+	await discord.snow.channel.createReaction(channelID, messageID, emoji)
+	return new Promise(resolve => {
+		buttons.push({channelID, messageID, userID, resolve, created: Date.now()})
+	})
+}
+
+// Clear out old buttons every so often to free memory
+setInterval(() => {
+	const now = Date.now()
+	buttons = buttons.filter(b => now - b.created < 2*60*60*1000)
+}, 10*60*1000)
+
+/** @param {import("discord-api-types/v10").GatewayMessageReactionAddDispatchData} data */
+function onReactionAdd(data) {
+	const button = buttons.find(b => b.channelID === data.channel_id && b.messageID === data.message_id && b.userID === data.user_id)
+	if (button) {
+		buttons = buttons.filter(b => b !== button) // remove button data so it can't be clicked again
+		button.resolve(data)
+	}
+}
+
+/**
+ * @callback CommandExecute
+ * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
+ * @param {DiscordTypes.APIGuildTextChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {Partial<DiscordTypes.RESTPostAPIChannelMessageJSONBody>} [ctx]
+ */
+
+/**
+ * @typedef Command
+ * @property {string[]} aliases
+ * @property {(message: DiscordTypes.GatewayMessageCreateDispatchData, channel: DiscordTypes.APIGuildTextChannel, guild: DiscordTypes.APIGuild) => Promise<any>} execute
+ */
+
+/** @param {CommandExecute} execute */
+function replyctx(execute) {
+	/** @type {CommandExecute} */
+	return function(message, channel, guild, ctx = {}) {
+		ctx.message_reference = {
+			message_id: message.id,
+			channel_id: channel.id,
+			guild_id: guild.id,
+			fail_if_not_exists: false
+		}
+		return execute(message, channel, guild, ctx)
+	}
+}
+
+/** @type {Command[]} */
+const commands = [{
+	aliases: ["icon", "avatar", "roomicon", "roomavatar", "channelicon", "channelavatar"],
+	execute: replyctx(
+		async (message, channel, guild, ctx) => {
+			// Guard
+			const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get()
+			if (!roomID) return discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: "This channel isn't bridged to the other side."
+			})
+
+			// Current avatar
+			const avatarEvent = await api.getStateEvent(roomID, "m.room.avatar", "")
+			const avatarURLParts = avatarEvent?.url.match(/^mxc:\/\/([^/]+)\/(\w+)$/)
+			let currentAvatarMessage =
+				( avatarURLParts ? `Current room-specific avatar: ${reg.ooye.server_origin}/_matrix/media/r0/download/${avatarURLParts[1]}/${avatarURLParts[2]}`
+				: "No avatar. Now's your time to strike. Use `//icon` again with a link or upload to set the room-specific avatar.")
+
+			// Next potential avatar
+			const nextAvatarURL = message.attachments.find(a => a.content_type?.startsWith("image/"))?.url || message.content.match(/https?:\/\/[^ ]+\.[^ ]+\.(?:png|jpg|jpeg|webp)\b/)?.[0]
+			let nextAvatarMessage =
+				( nextAvatarURL ? `\nYou want to set it to: ${nextAvatarURL}\nHit ✅ to make it happen.`
+				: "")
+
+			const sent = await discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: currentAvatarMessage + nextAvatarMessage
+			})
+
+			if (nextAvatarURL) {
+				addButton(channel.id, sent.id, "✅", message.author.id).then(async data => {
+					const mxcUrl = await file.uploadDiscordFileToMxc(nextAvatarURL)
+					await api.sendState(roomID, "m.room.avatar", "", {
+						url: mxcUrl
+					})
+					db.prepare("UPDATE channel_room SET custom_avatar = ? WHERE channel_id = ?").run(mxcUrl, channel.id)
+					await discord.snow.channel.createMessage(channel.id, {
+						...ctx,
+						content: "Your creation is unleashed. Any complaints will be redirected to Grelbo."
+					})
+				})
+			}
+		}
+	)
+}, {
+	aliases: ["invite"],
+	execute: replyctx(
+		async (message, channel, guild, ctx) => {
+			// Check guild is bridged
+			const spaceID = select("guild_space", "space_id", {guild_id: guild.id}).pluck().get()
+			const roomID = select("channel_room", "room_id", {channel_id: channel.id}).pluck().get()
+			if (!spaceID || !roomID) return discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: "This server isn't bridged to Matrix, so you can't invite Matrix users."
+			})
+
+			// Check CREATE_INSTANT_INVITE permission
+			assert(message.member)
+			const guildPermissions = utils.getPermissions(message.member.roles, guild.roles)
+			if (!(guildPermissions & BigInt(1))) {
+				return discord.snow.channel.createMessage(channel.id, {
+					...ctx,
+					content: "You don't have permission to invite people to this Discord server."
+				})
+			}
+
+			// Guard against accidental mentions instead of the MXID
+			if (message.content.match(/<[@#:].*>/)) return discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: "You have to say the Matrix ID of the person you want to invite, but you mentioned a Discord user in your message.\nOne way to fix this is by writing `` ` `` backticks `` ` `` around the Matrix ID."
+			})
+
+			// Get named MXID
+			const mxid = message.content.match(/@([^:]+):([a-z0-9:-]+\.[a-z0-9.:-]+)/)?.[0]
+			if (!mxid) return discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: "You have to say the Matrix ID of the person you want to invite. Matrix IDs look like this: `@username:example.org`"
+			})
+
+			// Check for existing invite to the space
+			let spaceMember
+			try {
+				spaceMember = await api.getStateEvent(spaceID, "m.room.member", mxid)
+			} catch (e) {}
+			if (spaceMember && spaceMember.membership === "invite") {
+				return discord.snow.channel.createMessage(channel.id, {
+					...ctx,
+					content: `\`${mxid}\` already has an invite, which they haven't accepted yet.`
+				})
+			}
+
+			// Invite Matrix user if not in space
+			if (!spaceMember || spaceMember.membership !== "join") {
+				await api.inviteToRoom(spaceID, mxid)
+				return discord.snow.channel.createMessage(channel.id, {
+					...ctx,
+					content: `You invited \`${mxid}\` to the server.`
+				})
+			}
+
+			// The Matrix user *is* in the space, maybe we want to invite them to this channel?
+			let roomMember
+			try {
+				roomMember = await api.getStateEvent(roomID, "m.room.member", mxid)
+			} catch (e) {}
+			if (!roomMember || (roomMember.membership !== "join" && roomMember.membership !== "invite")) {
+				const sent = await discord.snow.channel.createMessage(channel.id, {
+					...ctx,
+					content: `\`${mxid}\` is already in this server. Would you like to additionally invite them to this specific channel?\nHit ✅ to make it happen.`
+				})
+				return addButton(channel.id, sent.id, "✅", message.author.id).then(async data => {
+					await api.inviteToRoom(roomID, mxid)
+					await discord.snow.channel.createMessage(channel.id, {
+						...ctx,
+						content: `You invited \`${mxid}\` to the channel.`
+					})
+				})
+			}
+
+			// The Matrix user *is* in the space and in the channel.
+			await discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: `\`${mxid}\` is already in this server and this channel.`
+			})
+		}
+	)
+}, {
+	aliases: ["addbot"],
+	execute: replyctx(
+		async (message, channel, guild, ctx) => {
+			return discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: addbot()
+			})
+		}
+	)
+}, {
+	aliases: ["privacy", "discoverable", "publish", "published"],
+	execute: replyctx(
+		async (message, channel, guild, ctx) => {
+			const current = select("guild_space", "privacy_level", {guild_id: guild.id}).pluck().get()
+			if (current == null) {
+				return discord.snow.channel.createMessage(channel.id, {
+					...ctx,
+					content: "This server isn't bridged to the other side."
+				})
+			}
+
+			const levels = ["invite", "link", "directory"]
+			const level = levels.findIndex(x => message.content.includes(x))
+			if (level === -1) {
+				return discord.snow.channel.createMessage(channel.id, {
+					...ctx,
+					content: "**Usage: `//privacy <level>`**. This will set who can join the space on Matrix-side. There are three levels:"
+						+ "\n`invite`: Can only join with a direct in-app invite from another Matrix user, or the //invite command."
+						+ "\n`link`: Matrix links can be created and shared like Discord's invite links. `invite` features also work."
+						+ "\n`directory`: Publishes to the Matrix in-app directory, like Server Discovery. Preview enabled. `invite` and `link` also work."
+						+ `\n**Current privacy level: \`${levels[current]}\`**`
+				})
+			}
+
+			assert(message.member)
+			const guildPermissions = utils.getPermissions(message.member.roles, guild.roles)
+			if (guild.owner_id !== message.author.id && !(guildPermissions & BigInt(0x28))) { // MANAGE_GUILD | ADMINISTRATOR
+				return discord.snow.channel.createMessage(channel.id, {
+					...ctx,
+					content: "You don't have permission to change the privacy level. You need Manage Server or Administrator."
+				})
+			}
+
+			db.prepare("UPDATE guild_space SET privacy_level = ? WHERE guild_id = ?").run(level, guild.id)
+			discord.snow.channel.createMessage(channel.id, {
+				...ctx,
+				content: `Privacy level updated to \`${levels[level]}\`. Changes will apply shortly.`
+			})
+			await createSpace.syncSpaceFully(guild.id)
+		}
+	)
+}]
+
+/** @type {CommandExecute} */
+async function execute(message, channel, guild) {
+	if (!message.content.startsWith(PREFIX)) return
+	const words = message.content.slice(PREFIX.length).split(" ")
+	const commandName = words[0]
+	const command = commands.find(c => c.aliases.includes(commandName))
+	if (!command) return
+
+	await command.execute(message, channel, guild)
+}
+
+module.exports.execute = execute
+module.exports.onReactionAdd = onReactionAdd

discord/utils.js

@@ -0,0 +1,74 @@
+// @ts-check
+
+const DiscordTypes = require("discord-api-types/v10")
+
+const EPOCH = 1420070400000
+
+/**
+ * @param {string[]} userRoles
+ * @param {DiscordTypes.APIGuild["roles"]} guildRoles
+ * @param {string} [userID]
+ * @param {DiscordTypes.APIGuildChannel["permission_overwrites"]} [channelOverwrites]
+ */
+function getPermissions(userRoles, guildRoles, userID, channelOverwrites) {
+	let allowed = BigInt(0)
+	let everyoneID
+	// Guild allows
+	for (const role of guildRoles) {
+		if (role.name === "@everyone") {
+			allowed |= BigInt(role.permissions)
+			everyoneID = role.id
+		}
+		if (userRoles.includes(role.id)) {
+			allowed |= BigInt(role.permissions)
+		}
+	}
+
+	if (channelOverwrites) {
+		/** @type {((overwrite: Required<DiscordTypes.APIGuildChannel>["permission_overwrites"][0]) => any)[]} */
+		const actions = [
+			// Channel @everyone deny
+			overwrite => overwrite.id === everyoneID && (allowed &= ~BigInt(overwrite.deny)),
+			// Channel @everyone allow
+			overwrite => overwrite.id === everyoneID && (allowed |= BigInt(overwrite.allow)),
+			// Role deny
+			overwrite => userRoles.includes(overwrite.id) && (allowed &= ~BigInt(overwrite.deny)),
+			// Role allow
+			overwrite => userRoles.includes(overwrite.id) && (allowed |= BigInt(overwrite.allow)),
+			// User deny
+			overwrite => overwrite.id === userID && (allowed &= ~BigInt(overwrite.deny)),
+			// User allow
+			overwrite => overwrite.id === userID && (allowed |= BigInt(overwrite.allow))
+		]
+		for (let i = 0; i < actions.length; i++) {
+			for (const overwrite of channelOverwrites) {
+				actions[i](overwrite)
+			}
+		}
+	}
+	return allowed
+}
+
+/**
+ * Command interaction responses have a webhook_id for some reason, but still have real author info of a real bot user in the server.
+ * @param {DiscordTypes.APIMessage} message
+ */
+function isWebhookMessage(message) {
+	const isInteractionResponse = message.type === 20
+	return message.webhook_id && !isInteractionResponse
+}
+
+/** @param {string} snowflake */
+function snowflakeToTimestampExact(snowflake) {
+	return Number(BigInt(snowflake) >> 22n) + EPOCH
+}
+
+/** @param {number} timestamp */
+function timestampToSnowflakeInexact(timestamp) {
+	return String((timestamp - EPOCH) * 2**22)
+}
+
+module.exports.getPermissions = getPermissions
+module.exports.isWebhookMessage = isWebhookMessage
+module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact
+module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact

discord/utils.test.js

@@ -0,0 +1,84 @@
+const {test} = require("supertape")
+const data = require("../test/data")
+const utils = require("./utils")
+
+test("is webhook message: identifies bot interaction response as not a message", t => {
+	t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction), false)
+})
+
+test("is webhook message: identifies webhook interaction response as not a message", t => {
+	t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction_without_bot_user), false)
+})
+
+test("is webhook message: identifies webhook message as a message", t => {
+	t.equal(utils.isWebhookMessage(data.special_message.bridge_echo_webhook), true)
+})
+
+test("discord utils: converts snowflake to timestamp", t => {
+	t.equal(utils.snowflakeToTimestampExact("86913608335773696"), 1440792219004)
+})
+
+test("discord utils: converts timestamp to snowflake", t => {
+	t.match(utils.timestampToSnowflakeInexact(1440792219004), /^869136083357.....$/)
+})
+
+test("getPermissions: channel overwrite to allow role works", t => {
+	const guildRoles = [
+		{
+			version: 1695412489043,
+			unicode_emoji: null,
+			tags: {},
+			position: 0,
+			permissions: "559623605571137",
+			name: "@everyone",
+			mentionable: false,
+			managed: false,
+			id: "1154868424724463687",
+			icon: null,
+			hoist: false,
+			flags: 0,
+			color: 0
+		},
+		{
+			version: 1695412604262,
+			unicode_emoji: null,
+			tags: { bot_id: "466378653216014359" },
+			position: 1,
+			permissions: "536995904",
+			name: "PluralKit",
+			mentionable: false,
+			managed: true,
+			id: "1154868908336099444",
+			icon: null,
+			hoist: false,
+			flags: 0,
+			color: 0
+		},
+		{
+			version: 1698778936921,
+			unicode_emoji: null,
+			tags: {},
+			position: 1,
+			permissions: "536870912",
+			name: "web hookers",
+			mentionable: false,
+			managed: false,
+			id: "1168988246680801360",
+			icon: null,
+			hoist: false,
+			flags: 0,
+			color: 0
+		}
+	]
+	const userRoles = [ "1168988246680801360" ]
+	const userID = "684280192553844747"
+	const overwrites = [
+		{ type: 0, id: "1154868908336099444", deny: "0", allow: "1024" },
+		{ type: 0, id: "1154868424724463687", deny: "1024", allow: "0" },
+		{ type: 0, id: "1168988246680801360", deny: "0", allow: "1024" },
+		{ type: 1, id: "353373325575323648", deny: "0", allow: "1024" }
+	]
+	const permissions = utils.getPermissions(userRoles, guildRoles, userID, overwrites)
+	const want = BigInt(1 << 10 | 1 << 16)
+	t.equal((permissions & want), want)
+})

docs/api.md

@@ -1,52 +0,0 @@
-# API
-
-There is a web API for getting information about things that are bridged with Out Of Your Element.
-
-The base URL is the URL of the particular OOYE instance, for example, https://bridge.cadence.moe.
-
-No authentication is required.
-
-I'm happy to add more endpoints, just ask for them.
-
-## Endpoint: GET /api/message
-
-|Query parameter|Type|Description|
-|---------------|----|-----------|
-|`message_id`|regexp `/^[0-9]+$/`|Discord message ID to look up information for|
-
-Response:
-
-```typescript
-{
-  source: "matrix" | "discord"                // Which platform the message originated on
-  matrix_author?: {                             // Only for Matrix messages; should be up-to-date rather than historical data
-    displayname: string,                        // Matrix user's current display name
-    avatar_url: string | null,                  // Absolute HTTP(S) URL to download the Matrix user's current avatar
-    mxid: string                                // Matrix user ID, can never change
-  },
-  events: [                                   // Data about each individual event
-    {
-      metadata: {                               // Data from OOYE's database about how bridging was performed
-        sender: string,                           // Same as matrix user ID
-        event_id: string,                         // Unique ID of the event on Matrix, can never change
-        event_type: "m.room.message" | string,    // Event type
-        event_subtype: "m.text" | string | null,  // For m.room.message events, this is the msgtype property
-        part: 0 | 1,                              // For multi-event messages, 0 if this is the first part
-        reaction_part: 0 | 1,                     // For multi-event messages, 0 if this is the last part
-        room_id: string,                          // Room ID that the event was sent in, linked to the Discord channel
-        source: number
-      },
-      raw: {                                    // Raw historical event data from the Matrix API. Contains at least these properties:
-        content: any,                             // The only non-metadata property, entirely client-generated
-        type: string,
-        room_id: string,
-        sender: string,
-        origin_server_ts: number,
-        unsigned?: any,
-        event_id: string,
-        user_id: string
-      }
-    }
-  ]
-}
-```

docs/developer-orientation.md

@@ -1,129 +0,0 @@
-# Development setup
-
-* Install development dependencies with `npm install --save-dev` so you can run the tests.
-* Most files you change, such as actions, converters, and web, will automatically be reloaded.
-* If developing on a different computer to the one running the homeserver, use SSH port forwarding so that Synapse can connect on its `localhost:6693` to reach the running bridge on your computer. Example: `ssh -T -v -R 6693:localhost:6693 me@matrix.cadence.moe`
-* I recommend developing in Visual Studio Code so that the JSDoc x TypeScript annotation comments just work automatically. I don't know which other editors or language servers support annotations and type inference.
-
-# Efficiency details
-
-Using WeatherStack as a thin layer between the bridge application and the Discord API lets us control exactly what data is cached in memory. Only necessary information is cached. For example, member data, user data, message content, and past edits are never stored in memory. This keeps the memory usage low and also prevents it ballooning in size over the bridge's runtime.
-
-The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs and URLs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 300,000 messages (representing a year of conversations) requires 40.6 MB of storage space in the SQLite database.
-
-Only necessary data and columns are queried from the database. We only contact the homeserver API if the database doesn't contain what we need.
-
-File uploads (like avatars from bridged members) are checked locally and deduplicated. Only brand new files are uploaded to the homeserver. This saves loads of space in the homeserver's media repo, especially for Synapse.
-
-Switching to [WAL mode](https://www.sqlite.org/wal.html) could improve your database access speed even more. Run `node scripts/wal.js` if you want to switch to WAL mode. (This will also enable `synchronous = NORMAL`.)
-
-
-# Repository structure
-
-    .
-    * Runtime configuration, like tokens and user info:
-    ├── registration.yaml
-    * You are here! :)
-    ├── readme.md
-    * The bridge's SQLite database is stored here:
-    ├── ooye.db*
-    * Source code
-    └── src
-        * Database schema:
-        ├── db
-        │   ├── orm.js, orm-defs.d.ts
-        │   * Migrations change the database schema when you update to a newer version of OOYE:
-        │   ├── migrate.js
-        │   └── migrations
-        │       └── *.sql, *.js
-        * Discord-to-Matrix bridging:
-        ├── d2m
-        │   * Execute actions through the whole flow, like sending a Discord message to Matrix:
-        │   ├── actions
-        │   │   └── *.js
-        │   * Convert data from one form to another without depending on bridge state. Called by actions:
-        │   ├── converters
-        │   │   └── *.js
-        │   * Making Discord work:
-        │   ├── discord-*.js
-        │   * Listening to events from Discord and dispatching them to the correct `action`:
-        │   └── event-dispatcher.js
-        * Discord bot commands and menus:
-        ├── discord
-        │   ├── interactions
-        │   │   └── *.js
-        │   └── discord-command-handler.js
-        * Matrix-to-Discord bridging:
-        ├── m2d
-        │   * Execute actions through the whole flow, like sending a Matrix message to Discord:
-        │   ├── actions
-        │   │   └── *.js
-        │   * Convert data from one form to another without depending on bridge state. Called by actions:
-        │   ├── converters
-        │   │   └── *.js
-        │   * Listening to events from Matrix and dispatching them to the correct `action`:
-        │   └── event-dispatcher.js
-        * We aren't using the matrix-js-sdk, so here are all the functions for the Matrix C-S and Appservice APIs:
-        ├── matrix
-        │   └── *.js
-        * Various files you can run once if you need them.
-        └── scripts
-            * First time running a new bridge? Run this file to set up prerequisites on the Matrix server:
-            ├── setup.js
-            * Hopefully you won't need the rest of these. Code quality varies wildly.
-            └── *.js
-
-# Read next
-
-If you haven't set up Out Of Your Element yet, you might find [Simplified homeserver setup](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/simplified-homeserver-setup.md) helpful.
-
-If you don't know what the Matrix event JSON generally looks like, turn on developer tools in your client (Element has pretty good ones). Right click a couple of messages and see what they look like on the inside.
-
-I recommend first reading [How to add a new event type](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/how-to-add-a-new-event-type.md) as this will fill you in on key information in how the codebase is organised, which data structures are important, and what level of abstraction we're working on.
-
-If you haven't seen the [Discord API documentation](https://discord.com/developers/docs/) before, have a quick look at one of the pages on there. Same with the [Matrix Client-Server APIs](https://spec.matrix.org/latest/client-server-api/). You don't need to know these inside out, they're primarily references, not stories. But it is useful to have an idea of what a couple of the API endpoints look like, the kind of data they tend to accept, and the kind of data they tend to return.
-
-Then you might like to peruse the other files in the docs folder. Most of these were written stream-of-thought style as I try to work through a problem and find the best way to implement it. You might enjoy getting inside my head and seeing me invent and evaluate ways to solve the problem.
-
-Whether you read those or not, I'm more than happy to help you 1-on-1 with coding your dream feature. Join the chatroom [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) or PM me [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) and ask away.
-
-# Dependency justification
-
-Total transitive production dependencies: 134
-
-### <font size="+2">🦕</font>
-
-* (31) better-sqlite3: SQLite is the best database, and this is the best library for it.
-* (27) @cloudrac3r/pug: Language for dynamic web pages. This is my fork. (I released code that hadn't made it to npm, and removed the heavy pug-filters feature.)
-* (16) stream-mime-type@1: This seems like the best option. Version 1 is used because version 2 is ESM-only.
-* (9) h3: Web server. OOYE needs this for the appservice listener, authmedia proxy, self-service, and more.
-* (11) sharp: Image resizing and compositing. OOYE needs this for the emoji sprite sheets.
-
-### <font size="-1">🪱</font>
-
-* (0) @chriscdn/promise-semaphore: It does what I want.
-* (1) @cloudrac3r/discord-markdown: This is my fork.
-* (0) @cloudrac3r/giframe: This is my fork.
-* (1) @cloudrac3r/html-template-tag: This is my fork.
-* (0) @cloudrac3r/in-your-element: This is my Matrix Appservice API library. It depends on h3 and zod, which are already pulled in by OOYE.
-* (0) @cloudrac3r/mixin-deep: This is my fork. (It fixes a bug in regular mixin-deep.)
-* (0) @cloudrac3r/pngjs: Lottie stickers are converted to bitmaps with the vendored Rlottie WASM build, then the bitmaps are converted to PNG with pngjs.
-* (0) @cloudrac3r/turndown: This HTML-to-Markdown converter looked the most suitable. I forked it to change the escaping logic to match the way Discord works.
-* (3) @stackoverflow/stacks: Stack Overflow design language and icons.
-* (0) ansi-colors: Helps with interactive prompting for the initial setup, and it's already pulled in by enquirer.
-* (1) chunk-text: It does what I want.
-* (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
-* (0) discord-api-types: Bitfields needed at runtime and types needed for development.
-* (0) domino: DOM implementation that's already pulled in by turndown.
-* (1) enquirer: Interactive prompting for the initial setup rather than forcing users to edit YAML non-interactively.
-* (0) entities: Looks fine. No dependencies.
-* (0) get-relative-path: Looks fine. No dependencies.
-* (1) heatsync: Module hot-reloader that I trust.
-* (0) lru-cache: For holding unused nonce in memory and letting them be overwritten later if never used.
-* (0) mime-type: File extension to mime type mapping that's already pulled in by stream-mime-type.
-* (0) prettier-bytes: It does what I want and has no dependencies.
-* (0) snowtransfer: Discord API library with bring-your-own-caching that I trust.
-* (0) try-to-catch: Not strictly necessary, but it's already pulled in by supertape, so I may as well.
-* (0) uqr: QR code SVG generator. Used on the website to scan in an invite link.
-* (0) xxhash-wasm: Used where cryptographically secure hashing is not required.
-* (0) zod: Input validation for the web server. It's popular and easy to use.

docs/docker.md

@@ -1,76 +0,0 @@
-# Docker policy
-
-**Out Of Your Element has no official support for Docker. There are no official files or images. If you choose to run Out Of Your Element in Docker, you must disclose this when asking for support. I may refuse to provide support/advice at any time. I may refuse to acknowledge issue reports.**
-
-This also goes for Podman, Nix, and other similar technology that upends a program's understanding of what it's running on.
-
-## What I recommend
-
-I recommend [following the official setup guide,](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md) which does not use Docker.
-
-Ultimately, though, do what makes you happy. I can't stop you from doing what you want. As long as you read this page and understand my perspective, that's good enough for me.
-
-## Why I advise against Docker
-
-When misconfigured, Docker has terrible impacts. It can cause messages to go missing or even permanent data loss. These have happened to people.
-
-Docker also makes it much harder for me to advise on debugging because it puts barriers between you and useful debugging tools, such as stdin, the database file, a shell, and the inspector. It's also not clear which version of the source code is running in the container, as there are many pieces of Docker (builder, container, image) that can cache old data, often making it so you didn't actually update when you thought you did. This has happened to people.
-
-## Why I don't provide a good configuration myself
-
-It is not possible for Docker to be correctly configured by default. The defaults are broken and will cause data loss.
-
-It is also not possible for me to provide a correct configuration for everyone. Even if I provided a correct image, the YAMLs and command-line arguments must be written by individual end users. Incorrect YAMLs and command-line arguments may cause connection issues or permanent data loss.
-
-## Why I don't provide assistance if you run OOYE in Docker
-
-Problems you encounter, especially with the initial setup, are much more likely to be caused by nuances in your Docker setup than problems in my code. Therefore, my code is not responsible for the problem. The cause of the problem is different code that I can't advise on.
-
-Also, if you reported an issue and I asked for additional information to help find the cause, you might be unable to provide it because of the debugging barriers discussed above.
-
-## Why I don't provide Docker resources
-
-I create OOYE unpaid in my spare time because I enjoy the process. I find great enjoyment in creating code and none at all in creating infrastructure.
-
-## Why you're probably fine without Docker
-
-### If you care about system footprint
-
-OOYE was designed to be simple and courteous:
-
-* It only creates files in its working directory
-* It does not require any other processes to be running (e.g., no dependency on a Postgres process)
-* It only requires node/npm executables in PATH, which you can store in any folder if you don't want to use your package manager
-
-### If you care about ease of setup
-
-In my opinion, the [official setup process](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md) is straightforward. After installing prerequisites (Node.js and the repo clone), the rest of the process interactively guides you through providing necessary information. Your input is checked for correctness so the bridge will definitely work when you run it.
-
-I find this easier than the usual Docker workflow of pasting values into a YAML and rolling the dice on whether it will start up or not.
-
-### If you care about security in the case of compromise/RCE
-
-There are no known vulnerabilities in dependencies. I [carefully selected simple, light dependencies](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/developer-orientation.md#dependency-justification) to reduce attack surface area.
-
-For defense in depth, I suggest running OOYE as a different user.
-
-### If you want to see all the processes when you run docker ps
-
-Well, you got me there.
-
-## Unofficial, independent, community-provided container setups
-
-I acknowledge the demand for using OOYE in a container, so I will still point you in the right direction.
-
-I had no hand in creating these and have not used or tested them whatsoever. I make no assurance that these will work reliably, or even at all. If you use these, you must do so with the understanding that if you run into any problems, **you must ask for support from the author of that setup, not from me, because you're running their code, not mine.**
-
-***The following list is distributed for your information, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.***
-
-- by melody: https://git.shork.ch/docker-images/out-of-your-element
-- by sim2kid: https://github.com/sim2kid/ooye-docker
-- by Katharos Technology: https://github.com/katharostech/docker_ooye
-- by Emma: https://cgit.rory.gay/nix/OOYE-module.git/tree
-
-## Making your own Docker setup
-
-If you decide to make your own, I may provide advice or indicate problems at my discretion. You acknowledge that I am not required to provide evidence of problems I indicate, nor solutions to them. You acknowledge that it is not possible for me to exhaustively indicate every problem, so I cannot indicate correctness. Even if I have provided advice to an unofficial, independent, community-provided setup, I do not endorse it.

docs/foreign-keys.md

@@ -1,98 +0,0 @@
-# Foreign keys in the Out Of Your Element database
-
-Historically, Out Of Your Element did not use foreign keys in the database, but since I found a need for them, I have decided to add them. Referential integrity is probably valuable as well.
-
-The need is that unlinking a channel and room using the web interface should clear up all related entries from `message_channel`, `event_message`, `reaction`, etc. Without foreign keys, this requires multiple DELETEs with tricky queries. With foreign keys and ON DELETE CASCADE, this just works.
-
-## Quirks
-
-* **REPLACE INTO** internally causes a DELETE followed by an INSERT, and the DELETE part **will trigger any ON DELETE CASCADE** foreign key conditions on the table, even when the primary key being replaced is the same.
-	* ```sql
-		CREATE TABLE discord_channel (channel_id TEXT NOT NULL, name TEXT NOT NULL, PRIMARY KEY (channel_id));
-		CREATE TABLE discord_message (message_id TEXT NOT NULL, channel_id TEXT NOT NULL, PRIMARY KEY (message_id),
-		  FOREIGN KEY (channel_id) REFERENCES discord_channel (channel_id) ON DELETE CASCADE);
-		INSERT INTO discord_channel (channel_id, name) VALUES ("c_1", "place");
-		INSERT INTO discord_message (message_id, channel_id) VALUES ("m_2", "c_1"); -- i love my message
-		REPLACE INTO discord_channel (channel_id, name) VALUES ("c_1", "new place"); -- replace into time
-		-- i love my message
-		SELECT * FROM discord_message; -- where is my message
-		```
-* In SQLite, `pragma foreign_keys = on` must be set **for each connection** after it's established. I've added this at the start of `migrate.js`, which is called by all database connections.
-  * Pragma? Pragma keys
-* Whenever a child row is inserted, SQLite will look up a row from the parent table to ensure referential integrity. This means **the parent table should be sufficiently keyed or indexed on columns referenced by foreign keys**, or SQLite won't let you do it, with a cryptic error message later on during DML. Due to normal forms, foreign keys naturally tend to reference the parent table's primary key, which is indexed, so that's okay. But still keep this in mind, since many of OOYE's tables effectively have two primary keys, for the Discord and Matrix IDs. A composite primary key doesn't count, even when it's the first column. A unique index counts.
-
-## Where keys
-
-Here are some tables that could potentially have foreign keys added between them, and my thought process of whether foreign keys would be a good idea:
-
-* `guild_active` <--(PK guild_id FK)-- `channel_room` ✅
-	* Could be good for referential integrity.
-	* Linking to guild_space would be pretty scary in case the guild was being relinked to a different space - since rooms aren't tied to a space, this wouldn't actually disturb anything. So I pick guild_active instead.
-* `channel_room` <--(PK channel_id FK)-- `message_channel` ✅
-	* Seems useful as we want message records to be deleted when a channel is unlinked.
-* `message_channel` <--(PK message_id PK)-- `event_message` ✅
-	* Seems useful as we want event information to be deleted when a channel is unlinked.
-* `guild_active` <--(PK guild_id PK)-- `guild_space` ✅
-	* All bridged guilds should have a corresponding guild_active entry, so referential integrity would be useful here to make sure we haven't got any weird states.
-* `channel_room` <--(**C** room_id PK)-- `member_cache` ✅
-	* Seems useful as we want to clear the member cache when a channel is unlinked.
-	* There is no index on `channel_room.room_id` right now. It would be good to create this index. Will just make it UNIQUE in the table definition.
-* `message_channel` <--(PK message_id FK)-- `reaction` ✅
-	* Seems useful as we want to clear the reactions cache when a channel is unlinked.
-* `sim` <--(**C** mxid FK)-- `sim_member`
-	* OOYE inner joins on this.
-	* Sims are never deleted so if this was added it would only be used for enforcing referential integrity.
-	* The storage cost of the additional index on `sim` would not be worth the benefits.
-* `channel_room` <--(**C** room_id PK)-- `sim_member`
-	* If a room is being permanently unlinked, it may be useful to see a populated member list. If it's about to be relinked to another channel, we want to keep the sims in the room for more speed and to avoid spamming state events into the timeline.
-	* Either way, the sims could remain in the room even after it's been unlinked. So no referential integrity is desirable here.
-* `sim` <--(PK user_id PK)-- `sim_proxy`
-	* OOYE left joins on this. In normal operation, this relationship might not exist.
-* `channel_room` <--(PK channel_id PK)-- `webhook` ✅
-	* Seems useful. Webhooks should be deleted from Discord just before the channel is unlinked. That should be mirrored in the database too.
-
-## Occurrences of REPLACE INTO/DELETE FROM
-
-* `edit-message.js` — `REPLACE INTO message_channel`
-	* Scary! Changed to INSERT OR IGNORE
-* `send-message.js` — `REPLACE INTO message_channel`
-	* Changed to INSERT OR IGNORE
-* `add-reaction.js` — `REPLACE INTO reaction`
-* `channel-webhook.js` — `REPLACE INTO webhook`
-* `send-event.js` — `REPLACE INTO message_channel`
-	* Seems incorrect? Maybe?? Originally added in fcbb045. Changed to INSERT
-* `event-to-message.js` — `REPLACE INTO member_cache`
-* `oauth.js` — `REPLACE INTO guild_active`
-	* Very scary!! Changed to INSERT .. ON CONFLICT DO UPDATE
-* `create-room.js` — `DELETE FROM channel_room`
-	* Please cascade
-* `delete-message.js`
-	* Removed redundant DELETEs
-* `edit-message.js` — `DELETE FROM event_message`
-* `register-pk-user.js` — `DELETE FROM sim`
-	* It's a failsafe during creation
-* `register-user.js` — `DELETE FROM sim`
-	* It's a failsafe during creation
-* `remove-reaction.js` — `DELETE FROM reaction`
-* `event-dispatcher.js` — `DELETE FROM member_cache`
-* `redact.js` — `DELETE FROM event_message`
-	* Removed this redundant DELETE
-* `send-event.js` — `DELETE FROM event_message`
-	* Removed this redundant DELETE
-
-## How keys
-
-SQLite does not have a complete ALTER TABLE command, so I have to DROP and CREATE. According to [the docs](https://www.sqlite.org/lang_altertable.html), the correct strategy is:
-
-1. (Not applicable) *If foreign key constraints are enabled, disable them using PRAGMA foreign_keys=OFF.*
-2. Start a transaction.
-3. (Not applicable) *Remember the format of all indexes, triggers, and views associated with table X. This information will be needed in step 8 below. One way to do this is to run a query like the following: SELECT type, sql FROM sqlite_schema WHERE tbl_name='X'.*
-4. Use CREATE TABLE to construct a new table "new_X" that is in the desired revised format of table X. Make sure that the name "new_X" does not collide with any existing table name, of course.
-5. Transfer content from X into new_X using a statement like: INSERT INTO new_X SELECT ... FROM X.
-6. Drop the old table X: DROP TABLE X.
-7. Change the name of new_X to X using: ALTER TABLE new_X RENAME TO X.
-8. (Not applicable) *Use CREATE INDEX, CREATE TRIGGER, and CREATE VIEW to reconstruct indexes, triggers, and views associated with table X. Perhaps use the old format of the triggers, indexes, and views saved from step 3 above as a guide, making changes as appropriate for the alteration.*
-9. (Not applicable) *If any views refer to table X in a way that is affected by the schema change, then drop those views using DROP VIEW and recreate them with whatever changes are necessary to accommodate the schema change using CREATE VIEW.*
-10. If foreign key constraints were originally enabled then run PRAGMA foreign_key_check to verify that the schema change did not break any foreign key constraints.
-11. Commit the transaction started in step 2.
-12. (Not applicable) *If foreign keys constraints were originally enabled, reenable them now.*

docs/get-started.md

@@ -1,116 +0,0 @@
-# Setup
-
-If you want Docker, [please read this first.](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/docker.md)
-
-If you get stuck, you're welcome to message [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) or [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) to ask for help setting up OOYE!
-
-You'll need:
-
-* Administrative access to a homeserver
-* Discord bot
-* Domain name for the bridge's website - [more info](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/why-does-the-bridge-have-a-website.md)
-* Reverse proxy for that domain - an interactive process will help you set this up in step 5!
-
-Follow these steps:
-
-1. [Get Node.js version 22 or later](https://nodejs.org/en/download/prebuilt-installer). If you're on Linux, you may prefer to install through system's package manager, though Debian and Ubuntu have hopelessly out of date packages.
-
-1. Switch to a normal user account. (i.e. do not run any of the following commands as root or sudo.)
-
-1. Clone this repo and checkout a specific tag. (Development happens on main. Stable versions are tagged.)
-	* The latest release tag is ![](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=flat-square&label=%20&color=black).
-
-1. Install dependencies: `npm install`
-
-1. Run `npm run setup` to check your setup and set the bot's initial state. You only need to run this once ever. This command will guide you precisely through the following steps:
-
-	* First, you'll be asked for information like your homeserver URL.
-
-	* Then you'll be prompted to set up a reverse proxy pointing from your domain to the bridge's web server. Sample configurations can be found at the end of this guide. It will check that the reverse proxy works before you continue.
-
-	* Then you'll need to provide information about your Discord bot, and you'll be asked to change some of its settings.
-
-	* Finally, a registration.yaml file will be generated, which you need to give to your homeserver. You'll be told how to do this. It will check that it's done properly.
-
-1. Start the bridge: `npm run start`
-
-## Update
-
-New versions are announced in [#updates](https://matrix.to/#/#ooye-updates:cadence.moe) and listed on [releases](https://gitdab.com/cadence/out-of-your-element/releases). Here's how to update:
-
-1. Fetch the repo and checkout the latest release tag.
-
-1. Install dependencies: `npm install`
-
-1. Restart the bridge: Stop the currently running process, and then start the new one with `npm run start`
-
-# Get Started
-
-Visit the website on the domain name you set up, and click the button to add the bot to your Discord server.
-
-* If you click the Easy Mode button, it will automatically create a Matrix room corresponding to each Discord channel. This happens next time a message is sent on Discord (so that your Matrix-side isn't immediately cluttered with lots of inactive rooms).
-
-* If you click the Self Service button, it won't create anything for you. You'll have to provide your own Matrix space and rooms. After you click, you'll be prompted through the process. Use this if you're migrating from another bridge!
-
-After that, to get into the rooms on your Matrix account, use the invite form on the website, or the `/invite [your mxid here]` command on Discord.
-
-I hope you enjoy Out Of Your Element!
-
-----
-<br><br><br><br><br>
-
-# Appendix
-
-## Example reverse proxy for nginx, dedicated domain name
-
-Replace `bridge.cadence.moe` with the hostname you're using.
-
-```nix
-server {
-	listen 80;
-	listen [::]:80;
-	server_name bridge.cadence.moe;
-
-	return 301 https://bridge.cadence.moe$request_uri;
-}
-
-server {
-	listen 443 ssl http2;
-	listen [::]:443 ssl http2;
-	server_name bridge.cadence.moe;
-
-	# ssl parameters here...
-	client_max_body_size 5M;
-
-	location / {
-		add_header Strict-Transport-Security "max-age=63072000; includeSubDomains" always;
-		proxy_pass http://127.0.0.1:6693;
-	}
-}
-```
-
-## Example reverse proxy for nginx, sharing a domain name
-
-Same as above, but change the following:
-
-- `location / {` -> `location /ooye/ {` (any sub-path you want; you MUST use a trailing slash or it won't work)
-- `proxy_pass http://127.0.0.1:6693;` -> `proxy_pass http://127.0.0.1:6693/;` (you MUST use a trailing slash on this too or it won't work)
-
-## Example reverse proxy for Caddy, dedicated domain name
-
-```nix
-bridge.cadence.moe {
-	log {
-		output file /var/log/caddy/access.log
-		format console
-	}
-	encode gzip
-	reverse_proxy 127.0.0.1:6693
-}
-```
-
-## Example reverse proxy for traefik
-
-Note: Out Of Your Element has no official Docker support. This guide is for using traefik when OOYE is ***not*** in a container.
-
-See [third-party/reverse-proxy-traefik.md](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/third-party/reverse-proxy-traefik.md)

docs/room-upgrades.md

@@ -1,80 +0,0 @@
-# Room upgrades
-
-"Upgrading" a room is supposed to create a new room and then tries to set it up exactly like the old one. So it copies name, topic, power levels, space membership, etc. The old room is marked as old with an `m.room.tombstone` event, its power levels are adjusted to make it harder to send messages, and a hyperlink to the new room is added.
-
-## What happens?
-
-A room upgrade is triggered by a POST request to `/_matrix/client/v3/rooms/{roomId}/upgrade`. The upgrade process is done by the server, and involves multiple events across multiple rooms. Since this is server-specific, what will _actually_ happen depends on the server's implementation, but the spec says it does this:
-
-1. Checks that the user has permission to send `m.room.tombstone` events in the room.
-2. Creates a replacement room with a `m.room.create` event containing a predecessor field, the applicable `room_version`, and a `type` field which is copied from the predecessor room. If no `type` is set on the previous room, no `type` is specified on the new room’s create event either.
-3. Replicates transferable state events to the new room. The exact details for what is transferred is left as an implementation detail, however the recommended state events to transfer are:
-	* `m.room.server_acl`
-	* `m.room.encryption`
-	* `m.room.name`
-	* `m.room.avatar`
-	* `m.room.topic`
-	* `m.room.guest_access`
-	* `m.room.history_visibility`
-	* `m.room.join_rules`
-	* `m.room.power_levels`
-
-	(Membership can't be transferred by the server.)
-
-4. Moves any local aliases to the new room.
-5. Sends a `m.room.tombstone` event to the old room to indicate that it is not intended to be used any further.
-6. If possible, the power levels in the old room should also be modified to prevent sending of events and inviting new users. For example, setting `events_default` and `invite` to the greater of `50` and `users_default + 1`.
-
-### Synapse additionally:
-
-1. Copies its `m.space.child` events (if it was a space).
-	* This is good for OOYE, because it automatically tries to join new rooms when they're added to a registered space.
-2. Copies bans.
-3. Un/publishes to the public room directory as applicable.
-4. Copies user tags and push rules.
-
-Conduwuit does not do those!
-
-### Element additionally:
-
-1. May invite all users from the old room to the new room, depending on if the checkbox is checked in the dialog.
-2. Update parent spaces to remove the old room and add the new room.
-
-Cinny does not do those! The new room is totally detached! The hyperlink from the old room (and the moved alias by server) is the only way to find it!
-
-* This is probably still okay for OOYE? Since the join rules are preserved, and if they were `restricted`, OOYE is able to join via the tombstone hyperlink. Then, after it joins, it's already PL 100 since the power levels are preserved. It's very bad if the join rules were `invite`, but OOYE never sets this join rule - it's either `restricted` or `public`.
-
-### Other clients
-
-Nheko doesn't support room upgrades at all. Cinyy, NeoChat and FluffyChat just call the API and don't do anything. FluffyChat invites all joined/invited users to the new room if the join rule is restricted.
-
-### Notable things that don't happen at all:
-
-* Add `m.space.parent` pointing to the space it was in (if it was a room in a space).
-
-## What should OOYE do?
-
-### Ideal case (Element, Synapse)
-
-The new room is added to the space and OOYE autojoins it. It already has the correct power levels and join rules.
-
-OOYE still needs to do this:
-
-1. Un/set `m.room.parent` in the rooms.
-2. Update `channel_room` and `historical_channel_room` tables.
-
-### Not ideal case (everyone else)
-
-OOYE should:
-
-1. Join the room by following the hyperlink from the tombstone, if able
-	* If not able, somebody messed with the join rules. Send a PM to the user who upgraded - the new room's creator - asking for an invite.
-2. Wait for join.
-3. Un/set `m.space.child` events on the space.
-4. Un/set `m.room.parent` in the rooms.
-5. Update `channel_room` and `historical_channel_room` tables.
-6. Un/publish to the room directory.
-
-### It's actually fine to do all the steps always
-
-Even by blindly following the entire list, each step is a no-op or atomic, so it doesn't matter if Element is also trying to do them.

docs/self-service-room-creation-rules.md

@@ -1,77 +0,0 @@
-# Self-service room creation rules
-
-Before version 3 of Out Of Your Element, new Matrix rooms would be created on-demand when a Discord channel is spoken in for the first time. This has worked pretty well.
-
-This is done through functions like ensureRoom and ensureSpace in actions:
-
-```js
-async function sendMessage(message, channel, guild, row) {
-	const roomID = await createRoom.ensureRoom(message.channel_id)
-	...
-}
-
-/**
- * Ensures the room exists. If it doesn't, creates the room with an accurate initial state.
- * @param {string} channelID
- * @returns {Promise<string>} Matrix room ID
- */
-function ensureRoom(channelID) {
-	return _syncRoom(channelID, /* shouldActuallySync */ false) /* calls ensureSpace */
-}
-
-/**
- * Ensures the space exists. If it doesn't, creates the space with an accurate initial state.
- * @param {DiscordTypes.APIGuild} guild
- * @returns {Promise<string>} Matrix space ID
- */
-function ensureSpace(guild) {
-	return _syncSpace(guild, /* shouldActuallySync */ false)
-}
-```
-
-With the introduction of self-service mode, we still want to retain this as a possible mode of operation, since some people prefer to have OOYE handle this administrative work. However, other people prefer to manage the links between channels and rooms themselves, and have control over what new rooms get linked up to.
-
-Visibly, this is managed through the web interface. The web interface lets moderators enable/disable auto-creation of new rooms, as well as set which channels and rooms are linked together.
-
-There is a small complication. Not only are Matrix rooms created automatically, their Matrix spaces are also created automatically during room sync: ensureRoom calls ensureSpace. If a user opts in to self-service mode by clicking the specific button in the web portal, we must ensure the _space is not created automatically either,_ because the Matrix user will provide a space to link to.
-
-To solve this, we need a way to suppress specific guilds from having auto-created spaces. The natural way to represent this is a column on guild_space, but that doesn't work, because each guild_space row requires a guild and space to be linked, and we _don't want_ them to be linked.
-
-So, internally, OOYE keeps track of this through a new table:
-
-```sql
-CREATE TABLE "guild_active" (
-	"guild_id"	TEXT NOT NULL, -- only guilds that are bridged are present in this table
-	"autocreate"	INTEGER NOT NULL, -- 0 or 1
-	PRIMARY KEY("guild_id")
-) WITHOUT ROWID;
-```
-
-There is one more complication. When adding a Discord bot through web oauth with a redirect_uri, Discord adds the bot to the server normally, _then_ redirects back to OOYE, and only then does OOYE know which guild the bot was just added to. So, for a short time between the bot being added and the user being redirected, OOYE might receive Discord events in the server before it has the chance to create the guild_active database row.
-
-So to prevent this, self-service behaviour needs to be an implicit default, and users must firmly choose one system or another to begin using OOYE. It is important for me to design this in a way that doesn't force users to do any extra work or make a choice they don't understand to keep the pre-v3 behaviour.
-
-So there will be 3 states of whether a guild is self-service or not. At first, it could be absent from the table, in which case events for it will be dropped. Or it could be in the table with autocomplete = 0, in which case only rooms that already exist in channel_room will have messages bridged. Or it could have autocomplete = 1, in which case Matrix rooms will be created as needed, as per the pre-v3 behaviour.
-
-| Auto-create | Meaning                  |
-| --          | ------------             |
-| 😶‍🌫️         | Unbridged - waiting      |
-| ❌         | Bridged - self-service   |
-| ✅         | Bridged - auto-create    |
-
-Pressing buttons on web or using the /invite command on a guild will insert a row into guild_active, allowing it to be bridged.
-
-So here's all the technical changes needed to support self-service in v3:
-
-- New guild_active table showing whether, and how, a guild is bridged.
-- When /invite command is used, INSERT OR IGNORE INTO state 1 and ensureRoom + ensureSpace.
-- When bot is added through "easy mode" web button, REPLACE INTO state 1 and ensureSpace.
-- When bot is added through "self-service" web button, REPLACE INTO state 0.
-- Event dispatcher will only ensureRoom if the guild_active state is 1.
-- createRoom will only create other dependencies if the guild is autocreate.
-
-## Enough with your theory. How do rooms actually get bridged now?
-
-After clicking the easy mode button on web and adding the bot to a server, it will create new Matrix rooms on-demand when any invite features are used (web or command) OR just when any message is sent on Discord.
-
-Alternatively, pressing the self-service mode button and adding the bot to a server will prompt the web user to link it with a space. After doing so, they'll be on the standard guild management page where they can invite to the space and manually link rooms. Nothing will be autocreated.

docs/third-party/reverse-proxy-traefik.md

@@ -1,113 +0,0 @@
-> This guide was written by @bgtlover:stealthy.club, a community contributor. The author of Out Of Your Element hopes it will be useful, but cannot say whether the information is accurate or complete.
-
-## Example reverse proxy configuration with traefik
-
-Note: This guide describes setting up the reverse proxy configuration when OOYE is ***not*** in a Docker container.
-
-Because traefik is generally used in Docker, this guide assumes the user already has it configured properly. However, given that Docker is very complex and the smallest mistakes can cascade in catastrophic, not immediately observable, and unpredictable ways, a fairly complete setup will be reproduced. Therefore, system administrators are advised to diff this sample setup against theirs rather than copy it wholesale.
-
-### Note on variable substitution
-
-Variables will be denoted as `{{var}}`. This syntax has been chosen because that's also how YAML substitution works. The values that fit each variable will be explained after the code block containing the placeholder.
-
-### Base compose configuration for traefik
-
-This file defines the traefik service stack. It's responsible for mounting volumes correctly, declaring ports that should be opened on the host side, and the external traefik network (created manually).
-
-In compose.yml, put the following:
-
-```yaml
-services:
-  traefik:
-    image: "traefik:latest"
-    restart: always
-    command:
-      - "--configFile=/etc/traefik/static_config.yml"
-    ports:
-      - "80:80"   #http
-      - "443:443" #https
-    networks:
-      - traefik
-    volumes:
-      - ./letsencrypt:/letsencrypt
-      - /etc/localtime:/etc/localtime:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - ./static_config.yml:/etc/traefik/static_config.yml
-      - ./config:/etc/traefik/config
-networks:
-  traefik:
-    external: true
-```
-
-### Static traefik configuration
-
-The static traefik configuration is used to define base traefik behavior, for example entry points, access and runtime logs, a file or directory for per-service configuration, etc.
-
-In static_config.yml, put the following:
-
-```yaml
-api:
-  dashboard: true
-
-providers:
-  docker:
-    endpoint: "unix:///var/run/docker.sock"
-    exposedByDefault: false
-    network: "traefik"
-  file:
-    directory: /etc/traefik/config/
-    watch: true
-
-entryPoints:
-  web-secure:
-    address: ":443"
-    asDefault: true
-    http3: {}
-    http:
-      tls:
-        certResolver: default
-  web:
-    address: ":80"
-    http:
-      redirections:
-        entryPoint:
-          to: web-secure
-
-certificatesResolvers:
-  default:
-    acme:
-      email: {{email}}
-      storage: "/letsencrypt/acme.json"
-      tlsChallenge: {}
-
-```
-
-Replace `{{email}}` with a valid email address.
-
-### Out of your element traefik dynamic configuration
-
-Traefik's dynamic configuration files configure proxy behaviors on a per-application level.
-
-In config/out-of-your-element.yml, put the following:
-
-```yaml
-http:
-  routers:
-    out-of-your-element:
-      rule: Host(`bridge.stealthy.club`)
-      service: out-of-your-element-service
-  services:
-    out-of-your-element-service:
-      loadBalancer:
-        servers:
-          - url: "http://{{ip}}:{{port}}"
-
-```
-
-The `{{port}}` is 6693 unless you changed it during Out Of Your Element's first time setup.
-
-Replace `{{ip}}` with the ***external*** IP of your server.
-
-Make sure the port is allowed through your firewall if applicable.
-
-For context, the external IP is required because of Docker networking. Because Docker modifies the host-side iptables firewall and creates virtual interfaces for its networks, and because the networking inside containers is configured such that localhost points to the IP of the container instead of the actual host, placing localhost in the url field above would make the traefik container establish an HTTP connection to itself, which would cause a bad gateway error.

docs/why-does-the-bridge-have-a-website.md

@@ -1,55 +0,0 @@
-# Why does the bridge have a website?
-
-## It's essential for making images work
-
-Matrix has a feature called [Authenticated Media](https://matrix.org/blog/2024/06/26/sunsetting-unauthenticated-media/), where uploaded media (like user avatars and uploaded files) is restricted to Matrix users only. This means Discord users wouldn't be able to see important parts of the conversation.
-
-To keep things working for Discord users, OOYE's web server can act as a proxy files that were uploaded on Matrix-side. This will automatically take effect when needed, so Discord users shouldn't notice any issues.
-
-## Why now?
-
-I knew a web interface had a lot of potential, but I was reluctant to add one because it would make the initial setup more complicated. However, when authenticated media forced my hand, I saw an opportunity to introduce new useful features. Hopefully you'll agree that it's worth it!
-
-# What else does it do?
-
-## Makes it easy to invite the bot
-
-The home page of the website has buttons to add the bridge bot to a Discord server. If you are primarly a Matrix user and you want somebody else (who may be less technical) to add the bot to their own Discord server, this should make it a lot more intuitive for them.
-
-## Makes it easy to invite yourself or others
-
-After your hypothetical less-technical friend adds the bot to their Discord server, you need to generate an invite for your Matrix account on Matrix-side. Without the website, you might need to guide them through running a /invite command with your user ID. With the website, they don't have to do anything extra. You can use your phone to scan the QR code on their screen, which lets you invite your user ID in your own time.
-
-You can also set the person's permissions when you invite them, so you can easily bootstrap the Matrix side with your trusted ones as moderators.
-
-## To link channels and rooms
-
-Without a website, to link Discord channels to existing Matrix rooms, you'd need to run a /link command with the internal IDs of each room. This is tedious and error-prone, especially if you want to set up a lot of channels. With the web interface, you can see a list of all the available rooms and click them to link them together.
-
-## To change settings
-
-Important settings, like whether the Matrix rooms should be private or publicly accessible, can be configured by clicking buttons rather than memorising commands. Changes take effect immediately.
-
-# Permissions
-
-## Bot invites
-
-Anybody who can access the home page can use the buttons to add your bot - but even without the website, they can already do this by manually constructing a URL. If you want to make it so _only you_ can add your bot to servers, you need to edit [your Discord application](https://discord.com/developers/applications), go to the Bot section, and turn off the switch that says Public Bot.
-
-## Server settings
-
-If you have either the Administrator or Manage Server permissions in a Discord server, you can use the website to manage that server, including linking channels and changing settings.
-
-# Initial setup
-
-The website is built in to OOYE and is always running as part of the bridge. For authenticated media proxy to work, you'll need to make the web server accessible on the public internet over HTTPS, presumably using a reverse proxy.
-
-When you use `npm run setup` as part of OOYE's initial setup, it will guide you through this process, and it will do a thorough self-test to make sure it's configured correctly. If you get stuck or want a configuration template, check the notes below.
-
-## Reverse proxy
-
-When OOYE is running, the web server runs on port 6693. (To use a different port or a UNIX socket, edit registration.yaml's `socket` setting and restart.)
-
-It doesn't have to have its own dedicated domain name, you can also use a sub-path on an existing domain, like the domain of your Matrix homeserver. You are likely already using a reverse proxy to run your homeserver, so this should just be a configuration change.
-
-[See here for sample configurations!](https://gitdab.com/cadence/out-of-your-element/src/branch/main/docs/get-started.md#appendix)

jsconfig.json

@@ -1,10 +0,0 @@
-{
-	"compilerOptions": {
-		"target": "es2024",
-		"module": "nodenext",
-		"lib": ["ESNext"],
-		"strict": true,
-		"noImplicitAny": false,
-		"useUnknownInCatchVariables": false
-	}
-}

m2d/actions/add-reaction.js

@@ -0,0 +1,31 @@
+// @ts-check
+
+const assert = require("assert").strict
+const Ty = require("../../types")
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select} = passthrough
+/** @type {import("../converters/utils")} */
+const utils = sync.require("../converters/utils")
+/** @type {import("../converters/emoji")} */
+const emoji = sync.require("../converters/emoji")
+
+/**
+ * @param {Ty.Event.Outer<Ty.Event.M_Reaction>} event
+ */
+async function addReaction(event) {
+	const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get()
+	if (!channelID) return // We just assume the bridge has already been created
+	const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id}, "ORDER BY reaction_part").pluck().get()
+	if (!messageID) return // Nothing can be done if the parent message was never bridged.
+
+	const key = event.content["m.relates_to"].key
+	const discordPreferredEncoding = emoji.encodeEmoji(key, event.content.shortcode)
+	if (!discordPreferredEncoding) return
+
+	await discord.snow.channel.createReaction(channelID, messageID, discordPreferredEncoding) // acting as the discord bot itself
+
+	db.prepare("REPLACE INTO reaction (hashed_event_id, message_id, encoded_emoji) VALUES (?, ?, ?)").run(utils.getEventIDHash(event.event_id), messageID, discordPreferredEncoding)
+}
+
+module.exports.addReaction = addReaction

m2d/actions/channel-webhook.js

@@ -2,7 +2,7 @@
 
 const assert = require("assert").strict
 const DiscordTypes = require("discord-api-types/v10")
-const stream = require("stream")
+const {Readable} = require("stream")
 const passthrough = require("../../passthrough")
 const {discord, db, select} = passthrough
 
@@ -44,25 +44,20 @@ async function ensureWebhook(channelID, forceCreate = false) {
  */
 async function withWebhook(channelID, callback) {
 	const webhook = await ensureWebhook(channelID, false)
-	return callback(webhook).catch(async e => {
-		if (e.message === `{"message": "Unknown Webhook", "code": 10015}`) { // pathetic error handling from SnowTransfer
-			// Our webhook is gone. Maybe somebody deleted it, or removed and re-added OOYE from the guild.
-			const newWebhook = await ensureWebhook(channelID, true)
-			return callback(newWebhook) // not caught; if the error happens again just throw it instead of looping
-		}
-
+	return callback(webhook).catch(e => {
+		// TODO: check if the error was webhook-related and if webhook.created === false, then: const webhook = ensureWebhook(channelID, true); return callback(webhook)
 		throw e
 	})
 }
 
 /**
  * @param {string} channelID
- * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]}} data
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
  * @param {string} [threadID]
  */
 async function sendMessageWithWebhook(channelID, data, threadID) {
 	const result = await withWebhook(channelID, async webhook => {
-		return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID})
+		return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID, disableEveryone: true})
 	})
 	return result
 }
@@ -70,7 +65,7 @@ async function sendMessageWithWebhook(channelID, data, threadID) {
 /**
  * @param {string} channelID
  * @param {string} messageID
- * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]}} data
+ * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
  * @param {string} [threadID]
  */
 async function editMessageWithWebhook(channelID, messageID, data, threadID) {

m2d/actions/emoji-sheet.js

@@ -1,14 +1,13 @@
 // @ts-check
 
-const stream = require("stream")
+const assert = require("assert")
+const fetch = require("node-fetch").default
+
+const utils = require("../converters/utils")
 const {sync} = require("../../passthrough")
 
 /** @type {import("../converters/emoji-sheet")} */
 const emojiSheetConverter = sync.require("../converters/emoji-sheet")
-/** @type {import("../../matrix/api")} */
-const api = sync.require("../../matrix/api")
-/** @type {import("../../matrix/mreq")} */
-const mreq = sync.require("../../matrix/mreq")
 
 /**
  * Downloads the emoji from the web and converts to uncompressed PNG data.
@@ -17,19 +16,20 @@ const mreq = sync.require("../../matrix/mreq")
  */
 async function getAndConvertEmoji(mxc) {
 	const abortController = new AbortController()
+
+	const url = utils.getPublicUrlForMxc(mxc)
+	assert(url)
+
+	/** @type {import("node-fetch").Response} */
 	// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
 	// If we were using connection pooling, we would be forced to download the entire GIF.
 	// So we set no agent to ensure we are not connection pooling.
-	const res = await api.getMedia(mxc, {signal: abortController.signal})
-	if (res.status !== 200) {
-		const root = await res.json()
-		throw new mreq.MatrixServerError(root, {mxc})
-	}
-	const readable = stream.Readable.fromWeb(res.body)
-	return emojiSheetConverter.convertImageStream(readable, () => {
+	// @ts-ignore the signal is slightly different from the type it wants (still works fine)
+	const res = await fetch(url, {agent: false, signal: abortController.signal})
+	return emojiSheetConverter.convertImageStream(res.body, () => {
 		abortController.abort()
-		readable.emit("end")
-		readable.on("error", () => {}) // DOMException [AbortError]: This operation was aborted
+		res.body.pause()
+		res.body.emit("end")
 	})
 }
 

m2d/actions/redact.js

@@ -0,0 +1,44 @@
+// @ts-check
+
+const assert = require("assert").strict
+const Ty = require("../../types")
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select, from} = passthrough
+/** @type {import("../converters/utils")} */
+const utils = sync.require("../converters/utils")
+
+/**
+ * @param {Ty.Event.Outer_M_Room_Redaction} event
+ */
+async function deleteMessage(event) {
+	const rows = from("event_message").join("message_channel", "message_id").select("channel_id", "message_id").where({event_id: event.redacts}).all()
+	db.prepare("DELETE FROM event_message WHERE event_id = ?").run(event.event_id)
+	for (const row of rows) {
+		db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(row.message_id)
+		discord.snow.channel.deleteMessage(row.channel_id, row.message_id, event.content.reason)
+	}
+}
+
+/**
+ * @param {Ty.Event.Outer_M_Room_Redaction} event
+ */
+async function removeReaction(event) {
+	const hash = utils.getEventIDHash(event.redacts)
+	// TODO: this works but fix the type
+	const row = from("reaction").join("message_channel", "message_id").select("channel_id", "message_id", "encoded_emoji").where({hashed_event_id: hash}).get()
+	if (!row) return
+	await discord.snow.channel.deleteReactionSelf(row.channel_id, row.message_id, row.encoded_emoji)
+	db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(hash)
+}
+
+/**
+ * Try everything that could possibly be redacted.
+ * @param {Ty.Event.Outer_M_Room_Redaction} event
+ */
+async function handle(event) {
+	await deleteMessage(event)
+	await removeReaction(event)
+}
+
+module.exports.handle = handle

m2d/actions/send-event.js

@@ -0,0 +1,123 @@
+// @ts-check
+
+const Ty = require("../../types")
+const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
+const assert = require("assert").strict
+const crypto = require("crypto")
+const fetch = require("node-fetch").default
+const passthrough = require("../../passthrough")
+const {sync, discord, db, select} = passthrough
+
+/** @type {import("./channel-webhook")} */
+const channelWebhook = sync.require("./channel-webhook")
+/** @type {import("../converters/event-to-message")} */
+const eventToMessage = sync.require("../converters/event-to-message")
+/** @type {import("../../matrix/api")}) */
+const api = sync.require("../../matrix/api")
+/** @type {import("../../d2m/actions/register-user")} */
+const registerUser = sync.require("../../d2m/actions/register-user")
+/** @type {import("../actions/emoji-sheet")} */
+const emojiSheet = sync.require("../actions/emoji-sheet")
+
+/**
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message
+ * @returns {Promise<DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}>}
+ */
+async function resolvePendingFiles(message) {
+	if (!message.pendingFiles) return message
+	const files = await Promise.all(message.pendingFiles.map(async p => {
+		if ("buffer" in p) {
+			return {
+				name: p.name,
+				file: p.buffer
+			}
+		}
+		if ("key" in p) {
+			// Encrypted file
+			const d = crypto.createDecipheriv("aes-256-ctr", Buffer.from(p.key, "base64url"), Buffer.from(p.iv, "base64url"))
+			// @ts-ignore
+			fetch(p.url).then(res => res.body.pipe(d))
+			return {
+				name: p.name,
+				file: d
+			}
+		} else {
+			// Unencrypted file
+			/** @type {Readable} */ // @ts-ignore
+			const body = await fetch(p.url).then(res => res.body)
+			return {
+				name: p.name,
+				file: body
+			}
+		}
+	}))
+	const newMessage = {
+		...message,
+		files: files.concat(message.files || [])
+	}
+	delete newMessage.pendingFiles
+	return newMessage
+}
+
+/** @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker} event */
+async function sendEvent(event) {
+	const row = select("channel_room", ["channel_id", "thread_parent"], {room_id: event.room_id}).get()
+	if (!row) return // allow the bot to exist in unbridged rooms, just don't do anything with it
+	let channelID = row.channel_id
+	let threadID = undefined
+	if (row.thread_parent) {
+		threadID = channelID
+		channelID = row.thread_parent // it's the thread's parent... get with the times...
+	}
+	// @ts-ignore
+	const guildID = discord.channels.get(channelID).guild_id
+	const guild = discord.guilds.get(guildID)
+	assert(guild)
+
+	// no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it
+
+	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch, mxcDownloader: emojiSheet.getAndConvertEmoji})
+
+	messagesToEdit = await Promise.all(messagesToEdit.map(async e => {
+		e.message = await resolvePendingFiles(e.message)
+		return e
+	}))
+	messagesToSend = await Promise.all(messagesToSend.map(message => {
+		return resolvePendingFiles(message)
+	}))
+
+	let eventPart = 0 // 0 is primary, 1 is supporting
+
+	/** @type {DiscordTypes.APIMessage[]} */
+	const messageResponses = []
+	for (const data of messagesToEdit) {
+		const messageResponse = await channelWebhook.editMessageWithWebhook(channelID, data.id, data.message, threadID)
+		eventPart = 1
+		messageResponses.push(messageResponse)
+	}
+
+	for (const id of messagesToDelete) {
+		db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(id)
+		db.prepare("DELETE FROM event_message WHERE message_id = ?").run(id)
+		await channelWebhook.deleteMessageWithWebhook(channelID, id, threadID)
+	}
+
+	for (const message of messagesToSend) {
+		const reactionPart = messagesToEdit.length === 0 && message === messagesToSend[messagesToSend.length - 1] ? 0 : 1
+		const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID)
+		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, threadID || channelID)
+		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart, reactionPart) // source 0 = matrix
+
+		eventPart = 1
+		messageResponses.push(messageResponse)
+	}
+
+	for (const user of ensureJoined) {
+		registerUser.ensureSimJoined(user, event.room_id)
+	}
+
+	return messageResponses
+}
+
+module.exports.sendEvent = sendEvent

m2d/converters/emoji-sheet.js

@@ -1,11 +1,10 @@
 // @ts-check
 
 const assert = require("assert").strict
-const stream = require("stream")
 const {pipeline} = require("stream").promises
 const sharp = require("sharp")
-const {GIFrame} = require("@cloudrac3r/giframe")
-const {PNG} = require("@cloudrac3r/pngjs")
+const {GIFrame} = require("giframe")
+const {PNG} = require("pngjs")
 const streamMimeType = require("stream-mime-type")
 
 const SIZE = 48
@@ -49,7 +48,7 @@ async function compositeMatrixEmojis(mxcs, mxcDownloader) {
 }
 
 /**
- * @param {stream.Readable} streamIn
+ * @param {import("node-fetch").Response["body"]} streamIn
  * @param {() => any} stopStream
  * @returns {Promise<Buffer | undefined>} Uncompressed PNG image
  */
@@ -82,10 +81,9 @@ async function convertImageStream(streamIn, stopStream) {
 				giframe.feed(chunk)
 			})
 			const frame = await giframe.getFrame()
-			const pixels = Uint8Array.from(frame.pixels)
 			stopStream()
 
-			const buffer = await sharp(pixels, {raw: {width: frame.width, height: frame.height, channels: 4}})
+			const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}})
 				.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
 				.png({compressionLevel: 0})
 				.toBuffer({resolveWithObject: true})

m2d/converters/emoji.js

@@ -0,0 +1,57 @@
+// @ts-check
+
+const assert = require("assert").strict
+const Ty = require("../../types")
+
+const passthrough = require("../../passthrough")
+const {sync, select} = passthrough
+
+/**
+ * @param {string} input
+ * @param {string | null | undefined} shortcode
+ * @returns {string?}
+ */
+function encodeEmoji(input, shortcode) {
+	let discordPreferredEncoding
+	if (input.startsWith("mxc://")) {
+		// Custom emoji
+		let row = select("emoji", ["emoji_id", "name"], {mxc_url: input}).get()
+		if (!row && shortcode) {
+			// Use the name to try to find a known emoji with the same name.
+			const name = shortcode.replace(/^:|:$/g, "")
+			row = select("emoji", ["emoji_id", "name"], {name: name}).get()
+		}
+		if (!row) {
+			// We don't have this emoji and there's no realistic way to just-in-time upload a new emoji somewhere.
+			// Sucks!
+			return null
+		}
+		// Cool, we got an exact or a candidate emoji.
+		discordPreferredEncoding = encodeURIComponent(`${row.name}:${row.emoji_id}`)
+	} else {
+		// Default emoji
+		// https://github.com/discord/discord-api-docs/issues/2723#issuecomment-807022205 ????????????
+		const encoded = encodeURIComponent(input)
+		const encodedTrimmed = encoded.replace(/%EF%B8%8F/g, "")
+
+		const forceTrimmedList = [
+			"%F0%9F%91%8D", // 👍
+			"%F0%9F%91%8E", // 👎️
+			"%E2%AD%90", // ⭐
+			"%F0%9F%90%88", // 🐈
+			"%E2%9D%93", // ❓
+			"%F0%9F%8F%86", // 🏆️
+			"%F0%9F%93%9A", // 📚️
+		]
+
+		discordPreferredEncoding =
+			( forceTrimmedList.includes(encodedTrimmed) ? encodedTrimmed
+			: encodedTrimmed !== encoded && [...input].length === 2 ? encoded
+			: encodedTrimmed)
+
+		console.log("add reaction from matrix:", input, encoded, encodedTrimmed, "chosen:", discordPreferredEncoding)
+	}
+	return discordPreferredEncoding
+}
+
+module.exports.encodeEmoji = encodeEmoji

m2d/converters/event-to-message.js

@@ -0,0 +1,833 @@
+// @ts-check
+
+const Ty = require("../../types")
+const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
+const chunk = require("chunk-text")
+const TurndownService = require("turndown")
+const domino = require("domino")
+const assert = require("assert").strict
+const entities = require("entities")
+
+const passthrough = require("../../passthrough")
+const {sync, db, discord, select, from} = passthrough
+/** @type {import("../converters/utils")} */
+const mxUtils = sync.require("../converters/utils")
+/** @type {import("../../discord/utils")} */
+const dUtils = sync.require("../../discord/utils")
+/** @type {import("./emoji-sheet")} */
+const emojiSheet = sync.require("./emoji-sheet")
+
+/** @type {[RegExp, string][]} */
+const markdownEscapes = [
+	[/\\/g, '\\\\'],
+	[/\*/g, '\\*'],
+	[/^-/g, '\\-'],
+	[/^\+ /g, '\\+ '],
+	[/^(=+)/g, '\\$1'],
+	[/^(#{1,6}) /g, '\\$1 '],
+	[/`/g, '\\`'],
+	[/^~~~/g, '\\~~~'],
+	[/\[/g, '\\['],
+	[/\]/g, '\\]'],
+	[/^>/g, '\\>'],
+	[/_/g, '\\_'],
+	[/^(\d+)\. /g, '$1\\. ']
+	/*
+		Strikethrough is deliberately not escaped. Usually when Matrix users type ~~ it's not because they wanted to send ~~,
+		it's because they wanted strikethrough and it didn't work because their client doesn't support it.
+		As bridge developers, we can choose between "messages should look as similar as possible" vs "it was most likely intended to be strikethrough".
+		I went with the latter. Even though the appearance doesn't match, I'd rather it displayed as originally intended for 80% of the readers than for 0%.
+	*/
+]
+
+const turndownService = new TurndownService({
+	hr: "----",
+	headingStyle: "atx",
+	preformattedCode: true,
+	codeBlockStyle: "fenced"
+})
+
+/**
+ * Markdown characters in the HTML content need to be escaped, though take care not to escape the middle of bare links
+ * @param {string} string
+ */
+// @ts-ignore bad type from turndown
+turndownService.escape = function (string) {
+	const escapedWords = string.split(" ").map(word => {
+		if (word.match(/^https?:\/\//)) {
+			return word
+		} else {
+			return markdownEscapes.reduce(function (accumulator, escape) {
+				return accumulator.replace(escape[0], escape[1])
+		 	}, word)
+		}
+	})
+	return escapedWords.join(" ")
+}
+
+turndownService.remove("mx-reply")
+
+turndownService.addRule("strikethrough", {
+	filter: ["del", "s"],
+	replacement: function (content) {
+		return "~~" + content + "~~"
+	}
+})
+
+turndownService.addRule("underline", {
+	filter: ["u"],
+	replacement: function (content) {
+		return "__" + content + "__"
+	}
+})
+
+turndownService.addRule("blockquote", {
+	filter: "blockquote",
+	replacement: function (content) {
+		content = content.replace(/^\n+|\n+$/g, "")
+		content = content.replace(/^/gm, "> ")
+		return content
+	}
+})
+
+turndownService.addRule("spoiler", {
+	filter: function (node, options) {
+		return node.tagName === "SPAN" && node.hasAttribute("data-mx-spoiler")
+	},
+
+	replacement: function (content, node) {
+		if (node.getAttribute("data-mx-spoiler")) {
+			// escape parentheses so it can't become a link
+			return `\\(${node.getAttribute("data-mx-spoiler")}\\) ||${content}||`
+		}
+		return `||${content}||`
+	}
+})
+
+turndownService.addRule("inlineLink", {
+	filter: function (node, options) {
+		return (
+			node.nodeName === "A" &&
+			node.getAttribute("href")
+		)
+	},
+
+	replacement: function (content, node) {
+		if (node.getAttribute("data-user-id")) {
+			const user_id = node.getAttribute("data-user-id")
+			const row = select("sim_proxy", ["displayname", "proxy_owner_id"], {user_id}).get()
+			if (row) {
+				return `**@${row.displayname}** (<@${row.proxy_owner_id}>)`
+			} else {
+				return `<@${user_id}>`
+			}
+		}
+		if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}`
+		if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>`
+		const href = node.getAttribute("href")
+		content = content.replace(/ @.*/, "")
+		if (href === content) return href
+		if (href.startsWith("https://matrix.to/#/@") && content[0] !== "@") content = "@" + content
+		return "[" + content + "](" + href + ")"
+	}
+})
+
+turndownService.addRule("listItem", {
+	filter: "li",
+	replacement: function (content, node, options) {
+		content = content
+		.replace(/^\n+/, "") // remove leading newlines
+		.replace(/\n+$/, "\n") // replace trailing newlines with just a single one
+		.replace(/\n/gm, "\n  ") // indent
+		var prefix = options.bulletListMarker + " "
+		var parent = node.parentNode
+		if (parent.nodeName === "OL") {
+			var start = parent.getAttribute("start")
+			var index = Array.prototype.indexOf.call(parent.children, node)
+			prefix = (start ? Number(start) + index : index + 1) + ". "
+		}
+		return prefix + content + (node.nextSibling && !/\n$/.test(content) ? "\n" : "")
+	}
+})
+
+/** @type {string[]} SPRITE SHEET EMOJIS FEATURE: mxc urls for the currently processing message */
+let endOfMessageEmojis = []
+turndownService.addRule("emoji", {
+	filter: function (node, options) {
+		if (node.nodeName !== "IMG" || !node.hasAttribute("data-mx-emoticon") || !node.getAttribute("src") || !node.getAttribute("title")) return false
+		return true
+	},
+
+	replacement: function (content, node) {
+		const mxcUrl = node.getAttribute("src")
+		const guessedName = node.getAttribute("title").replace(/^:|:$/g, "")
+		return convertEmoji(mxcUrl, guessedName, true, true)
+	}
+})
+
+turndownService.addRule("fencedCodeBlock", {
+	filter: function (node, options) {
+		return (
+			options.codeBlockStyle === "fenced" &&
+			node.nodeName === "PRE" &&
+			node.firstChild &&
+			node.firstChild.nodeName === "CODE"
+		)
+	},
+	replacement: function (content, node, options) {
+		const className = node.firstChild.getAttribute("class") || ""
+		const language = (className.match(/language-(\S+)/) || [null, ""])[1]
+		const code = node.firstChild
+		const visibleCode = getCodeContent(code)
+
+		var fence = "```"
+
+		return (
+			fence + language + "\n" +
+			visibleCode +
+			"\n" + fence
+		)
+	}
+})
+
+/** @param {{ childNodes: Node[]; }} preCode the <code> directly inside the <pre> */
+function getCodeContent(preCode) {
+	return preCode.childNodes.map(c => c.nodeName === "BR" ? "\n" : c.textContent).join("").replace(/\n*$/g, "")
+}
+
+/**
+ * @param {string | null} mxcUrl
+ * @param {string | null} nameForGuess without colons
+ * @param {boolean} allowSpriteSheetIndicator
+ * @param {boolean} allowLink
+ * @returns {string} discord markdown that represents the custom emoji in some form
+ */
+function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink) {
+	// Get the known emoji from the database.
+	let row
+	if (mxcUrl) row = select("emoji", ["emoji_id", "name", "animated"], {mxc_url: mxcUrl}).get()
+	// Now we have to search all servers to see if we're able to send this emoji.
+	if (row) {
+		const found = [...discord.guilds.values()].find(g => g.emojis.find(e => e.id === row.id))
+		if (!found) row = null
+	}
+	// Or, if we don't have an emoji right now, we search for the name instead.
+	if (!row && nameForGuess) {
+		const nameForGuessLower = nameForGuess.toLowerCase()
+		for (const guild of discord.guilds.values()) {
+			/** @type {{name: string, id: string, animated: number}[]} */
+			// @ts-ignore
+			const emojis = guild.emojis
+			const found = emojis.find(e => e.id === row?.id || e.name?.toLowerCase() === nameForGuessLower)
+			if (found) {
+				row = {
+					animated: found.animated,
+					emoji_id: found.id,
+					name: found.name
+				}
+				break
+			}
+		}
+	}
+	if (row) {
+		// We know an emoji, and we can use it
+		const animatedChar = row.animated ? "a" : ""
+		return `<${animatedChar}:${row.name}:${row.emoji_id}>`
+	} else if (allowSpriteSheetIndicator && mxcUrl && endOfMessageEmojis.includes(mxcUrl)) {
+		// We can't locate or use a suitable emoji. After control returns, it will rewind over this, delete this section, and upload the emojis as a sprite sheet.
+		return `<::>`
+	} else if (allowLink && mxcUrl && nameForGuess) {
+		// We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle.
+		return `[:${nameForGuess}:](${mxUtils.getPublicUrlForMxc(mxcUrl)})`
+	} else if (nameForGuess) {
+		return `:${nameForGuess}:`
+	} else {
+		return ""
+	}
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} mxid
+ * @returns {Promise<{displayname?: string?, avatar_url?: string?}>}
+ */
+async function getMemberFromCacheOrHomeserver(roomID, mxid, api) {
+	const row = select("member_cache", ["displayname", "avatar_url"], {room_id: roomID, mxid}).get()
+	if (row) return row
+	return api.getStateEvent(roomID, "m.room.member", mxid).then(event => {
+		db.prepare("REPLACE INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(roomID, mxid, event?.displayname || null, event?.avatar_url || null)
+		return event
+	}).catch(() => {
+		return {displayname: null, avatar_url: null}
+	})
+}
+
+/**
+ * Splits a display name into one chunk containing <=80 characters (80 being how many characters Discord allows for the name of a webhook),
+ * and another chunk containing the rest of the characters. Splits on whitespace if possible.
+ * These chunks, respectively, go in the display name, and at the top of the message.
+ * If the second part isn't empty, it'll also contain boldening markdown and a line break at the end, so that regardless of its value it
+ * can be prepended to the message content as-is.
+ * @summary Splits too-long Matrix names into a display name chunk and a message content chunk.
+ * @param  {string} displayName - The Matrix side display name to chop up.
+ * @returns {[string, string]} [shortened display name, display name runoff]
+ */
+function splitDisplayName(displayName) {
+	/** @type {string[]} */
+	let displayNameChunks = chunk(displayName, 80)
+
+	if (displayNameChunks.length === 1) {
+		return [displayName, ""]
+	} else {
+		const displayNamePreRunoff = displayNameChunks[0]
+		// displayNameRunoff is a slice of the original rather than a concatenation of the rest of the chunks in order to preserve whatever whitespace it was broken on.
+		const displayNameRunoff = `**${displayName.slice(displayNamePreRunoff.length + 1)}**\n`
+
+		return [displayNamePreRunoff, displayNameRunoff]
+	}
+}
+
+/**
+ * Convert a Matrix user ID into a Discord user ID for mentioning, where if the user is a PK proxy, it will mention the proxy owner.
+ * @param {string} mxid
+ */
+function getUserOrProxyOwnerID(mxid) {
+	const row = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({mxid}).get()
+	if (!row) return null
+	return row.proxy_owner_id || row.user_id
+}
+
+/**
+ * At the time of this executing, we know what the end of message emojis are, and we know that at least one of them is unknown.
+ * This function will strip them from the content and generate the correct pending file of the sprite sheet.
+ * @param {string} content
+ * @param {{id: string, name: string}[]} attachments
+ * @param {({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} pendingFiles
+ * @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
+ */
+async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, mxcDownloader) {
+	if (!content.includes("<::>")) return content // No unknown emojis, nothing to do
+	// Remove known and unknown emojis from the end of the message
+	const r = /<a?:[a-zA-Z0-9_]*:[0-9]*>\s*$/
+	while (content.match(r)) {
+		content = content.replace(r, "")
+	}
+	// Create a sprite sheet of known and unknown emojis from the end of the message
+	const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis, mxcDownloader)
+	// Attach it
+	const name = "emojis.png"
+	attachments.push({id: String(attachments.length), name})
+	pendingFiles.push({name, buffer})
+	return content
+}
+
+/**
+ * @param {string} input
+ * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API
+ */
+async function handleRoomOrMessageLinks(input, di) {
+	let offset = 0
+	for (const match of [...input.matchAll(/("?https:\/\/matrix.to\/#\/(![^"/, ?)]+)(?:\/(\$[^"/ ?)]+))?(?:\?[^",:!? )]*)?)(">|[, )]|$)/g)]) {
+		assert(typeof match.index === "number")
+		const [_, attributeValue, roomID, eventID, endMarker] = match
+		let result
+
+		const resultType = endMarker === '">' ? "html" : "plain"
+		const MAKE_RESULT = {
+			ROOM_LINK: {
+				html: channelID => `${attributeValue}" data-channel-id="${channelID}">`,
+				plain: channelID => `<#${channelID}>${endMarker}`
+			},
+			MESSAGE_LINK: {
+				html: (guildID, channelID, messageID) => `${attributeValue}" data-channel-id="${channelID}" data-guild-id="${guildID}" data-message-id="${messageID}">`,
+				plain: (guildID, channelID, messageID) => `https://discord.com/channels/${guildID}/${channelID}/${messageID}${endMarker}`
+			}
+		}
+
+		// Don't process links that are part of the reply fallback, they'll be removed entirely by turndown
+		if (input.slice(match.index + match[0].length + offset).startsWith("In reply to")) continue
+
+		const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
+		if (!channelID) continue
+		if (!eventID) {
+			// 1: It's a room link, so <#link> to the channel
+			result = MAKE_RESULT.ROOM_LINK[resultType](channelID)
+		} else {
+			// Linking to a particular event with a discord.com/channels/guildID/channelID/messageID link
+			// Need to know the guildID and messageID
+			const guildID = discord.channels.get(channelID)?.["guild_id"]
+			if (!guildID) continue
+			const messageID = select("event_message", "message_id", {event_id: eventID}).pluck().get()
+			if (messageID) {
+				// 2: Linking to a known event
+				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, messageID)
+			} else {
+				// 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp
+				let originalEvent
+				try {
+					originalEvent = await di.api.getEvent(roomID, eventID)
+				} catch (e) {
+					continue // Our homeserver doesn't know about the event, so can't resolve it to a Discord link
+				}
+				const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts)
+				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, guessedMessageID)
+			}
+		}
+
+		input = input.slice(0, match.index + offset) + result + input.slice(match.index + match[0].length + offset)
+		offset += result.length - match[0].length
+	}
+
+	return input
+}
+
+/**
+ * @param {string} content
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di
+ */
+async function checkWrittenMentions(content, guild, di) {
+	let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+
+	if (writtenMentionMatch) {
+		const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]})
+		if (results[0]) {
+			assert(results[0].user)
+			return {
+				// @ts-ignore - typescript doesn't know about indices yet
+				content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]),
+				ensureJoined: results[0].user
+			}
+		}
+	}
+}
+
+/**
+ * @param {Element} node
+ * @param {string[]} tagNames allcaps tag names
+ * @returns {any | undefined} the node you were checking for, or undefined
+ */
+function nodeIsChildOf(node, tagNames) {
+	// @ts-ignore
+	for (; node; node = node.parentNode) if (tagNames.includes(node.tagName)) return node
+}
+
+const attachmentEmojis = new Map([
+	["m.image", "🖼️"],
+	["m.video", "🎞️"],
+	["m.audio", "🎶"],
+	["m.file", "📄"]
+])
+
+/**
+ * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event
+ * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"], mxcDownloader: (mxc: string) => Promise<Buffer | undefined>}} di simple-as-nails dependency injection for the matrix API
+ */
+async function eventToMessage(event, guild, di) {
+	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
+	let messages = []
+
+	let displayName = event.sender
+	let avatarURL = undefined
+	/** @type {string[]} */
+	let messageIDsToEdit = []
+	let replyLine = ""
+	// Extract a basic display name from the sender
+	const match = event.sender.match(/^@(.*?):/)
+	if (match) displayName = match[1]
+	// Try to extract an accurate display name and avatar URL from the member event
+	const member = await getMemberFromCacheOrHomeserver(event.room_id, event.sender, di?.api)
+	if (member.displayname) displayName = member.displayname
+	if (member.avatar_url) avatarURL = mxUtils.getPublicUrlForMxc(member.avatar_url) || undefined
+	// If the display name is too long to be put into the webhook (80 characters is the maximum),
+	// put the excess characters into displayNameRunoff, later to be put at the top of the message
+	let [displayNameShortened, displayNameRunoff] = splitDisplayName(displayName)
+	// If the message type is m.emote, the full name is already included at the start of the message, so remove any runoff
+	if (event.type === "m.room.message" && event.content.msgtype === "m.emote") {
+		displayNameRunoff = ""
+	}
+
+	let content = event.content.body // ultimate fallback
+	const attachments = []
+	/** @type {({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} */
+	const pendingFiles = []
+	/** @type {DiscordTypes.APIUser[]} */
+	const ensureJoined = []
+
+	// Convert content depending on what the message is
+	if (event.type === "m.room.message" && (event.content.msgtype === "m.text" || event.content.msgtype === "m.emote")) {
+		// Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events.
+		// this event ---is an edit of--> original event ---is a reply to--> past event
+		await (async () => {
+			// Check if there is an edit
+			const relatesTo = event.content["m.relates_to"]
+			if (!event.content["m.new_content"] || !relatesTo || relatesTo.rel_type !== "m.replace") return
+			// Check if we have a pointer to what was edited
+			const originalEventId = relatesTo.event_id
+			if (!originalEventId) return
+			messageIDsToEdit = select("event_message", "message_id", {event_id: originalEventId}, "ORDER BY part").pluck().all()
+			if (!messageIDsToEdit.length) return
+
+			// Ok, it's an edit.
+			event.content = event.content["m.new_content"]
+
+			// Is it editing a reply? We need special handling if it is.
+			// Get the original event, then check if it was a reply
+			const originalEvent = await di.api.getEvent(event.room_id, originalEventId)
+			if (!originalEvent) return
+			const repliedToEventId = originalEvent.content["m.relates_to"]?.["m.in_reply_to"]?.event_id
+			if (!repliedToEventId) return
+
+			// After all that, it's an edit of a reply.
+			// We'll be sneaky and prepare the message data so that the next steps can handle it just like original messages.
+			Object.assign(event.content, {
+				"m.relates_to": {
+					"m.in_reply_to": {
+						event_id: repliedToEventId
+					}
+				}
+			})
+		})()
+
+		// Handling replies. We'll look up the data of the replied-to event from the Matrix homeserver.
+		// Note that an <mx-reply> element is not guaranteed because this might be m.new_content.
+		await (async () => {
+			const repliedToEventId = event.content["m.relates_to"]?.["m.in_reply_to"]?.event_id
+			if (!repliedToEventId) return
+			let repliedToEvent
+			try {
+				repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId)
+			} catch (e) {
+				// Original event isn't on our homeserver, so we'll *partially* trust the client's reply fallback.
+				// We'll trust the fallback's quoted content and put it in the reply preview, but we won't trust the authorship info on it.
+
+				// (But if the fallback's quoted content doesn't exist, we give up. There's nothing for us to quote.)
+				if (event.content["format"] !== "org.matrix.custom.html" || typeof event.content["formatted_body"] !== "string") {
+					const lines = event.content.body.split("\n")
+					let stage = 0
+					for (let i = 0; i < lines.length; i++) {
+						if (stage >= 0 && lines[i][0] === ">") stage = 1
+						if (stage >= 1 && lines[i].trim() === "") stage = 2
+						if (stage === 2 && lines[i].trim() !== "") {
+							event.content.body = lines.slice(i).join("\n")
+							break
+						}
+					}
+					return
+				}
+				const mxReply = event.content["formatted_body"]
+				const quoted = mxReply.match(/^<mx-reply><blockquote>.*?In reply to.*?<br>(.*)<\/blockquote><\/mx-reply>/)?.[1]
+				if (!quoted) return
+				const contentPreviewChunks = chunk(
+					entities.decodeHTML5Strict( // Remove entities like &amp; &quot;
+						quoted.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
+							.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+							.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+							.replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting.
+					), 50)
+				replyLine = "> " + contentPreviewChunks[0]
+				if (contentPreviewChunks.length > 1) replyLine = replyLine.replace(/[,.']$/, "") + "..."
+				replyLine += "\n"
+				return
+			}
+
+			// @ts-ignore
+			const autoEmoji = new Map(select("auto_emoji", ["name", "emoji_id"], {}, "WHERE name = 'L1' OR name = 'L2'").raw().all())
+			replyLine = `<:L1:${autoEmoji.get("L1")}><:L2:${autoEmoji.get("L2")}>`
+			const row = from("event_message").join("message_channel", "message_id").select("channel_id", "message_id").where({event_id: repliedToEventId}).and("ORDER BY part").get()
+			if (row) {
+				replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} `
+			}
+			const sender = repliedToEvent.sender
+			const authorID = getUserOrProxyOwnerID(sender)
+			if (authorID) {
+				replyLine += `<@${authorID}>`
+			} else {
+				let senderName = select("member_cache", "displayname", {mxid: sender}).pluck().get()
+				if (!senderName) {
+					const match = sender.match(/@([^:]*)/)
+					assert(match)
+					senderName = match[1]
+				}
+				replyLine += `Ⓜ️**${senderName}**`
+			}
+			// If the event has been edited, the homeserver will include the relation in `unsigned`.
+			if (repliedToEvent.unsigned?.["m.relations"]?.["m.replace"]?.content?.["m.new_content"]) {
+				repliedToEvent = repliedToEvent.unsigned["m.relations"]["m.replace"] // Note: this changes which event_id is in repliedToEvent.
+				repliedToEvent.content = repliedToEvent.content["m.new_content"]
+			}
+			let contentPreview
+			const fileReplyContentAlternative = attachmentEmojis.get(repliedToEvent.content.msgtype)
+			if (fileReplyContentAlternative) {
+				contentPreview = " " + fileReplyContentAlternative
+			} else if (repliedToEvent.unsigned?.redacted_because) {
+				contentPreview = " (in reply to a deleted message)"
+			} else {
+				// Generate a reply preview for a standard message
+				/** @type {string} */
+				let repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body
+				repliedToContent = repliedToContent.replace(/.*<\/mx-reply>/s, "") // Remove everything before replies, so just use the actual message body
+				repliedToContent = repliedToContent.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
+				repliedToContent = repliedToContent.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+				repliedToContent = repliedToContent.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+				repliedToContent = repliedToContent.replace(/<img([^>]*)>/g, (_, att) => { // Convert Matrix emoji images into Discord emoji markdown
+					const mxcUrlMatch = att.match(/\bsrc="(mxc:\/\/[^"]+)"/)
+					const titleTextMatch = att.match(/\btitle=":?([^:"]+)/)
+					return convertEmoji(mxcUrlMatch?.[1], titleTextMatch?.[1], false, false)
+				})
+				repliedToContent = repliedToContent.replace(/<[^:>][^>]*>/g, "") // Completely strip all HTML tags and formatting.
+				repliedToContent = entities.decodeHTML5Strict(repliedToContent) // Remove entities like &amp; &quot;
+				const contentPreviewChunks = chunk(repliedToContent, 50)
+				if (contentPreviewChunks.length) {
+					contentPreview = ":\n> " + contentPreviewChunks[0]
+					if (contentPreviewChunks.length > 1) contentPreview = contentPreview.replace(/[,.']$/, "") + "..."
+				} else {
+					console.log("Unable to generate reply preview for this replied-to event because we stripped all of it:", repliedToEvent)
+					contentPreview = ""
+				}
+			}
+			replyLine = `> ${replyLine}${contentPreview}\n`
+		})()
+
+		if (event.content.format === "org.matrix.custom.html" && event.content.formatted_body) {
+			let input = event.content.formatted_body
+			if (event.content.msgtype === "m.emote") {
+				input = `* ${displayName} ${input}`
+			}
+
+			// Handling mentions of Discord users
+			input = input.replace(/("https:\/\/matrix.to\/#\/((?:@|%40)[^"]+)")>/g, (whole, attributeValue, mxid) => {
+				mxid = decodeURIComponent(mxid)
+				if (mxUtils.eventSenderIsFromDiscord(mxid)) {
+					// Handle mention of an OOYE sim user by their mxid
+					const id = select("sim", "user_id", {mxid}).pluck().get()
+					if (!id) return whole
+					return `${attributeValue} data-user-id="${id}">`
+				} else {
+					// Handle mention of a Matrix user by their mxid
+					// Check if this Matrix user is actually the sim user from another old bridge in the room?
+					const match = mxid.match(/[^:]*discord[^:]*_([0-9]{6,}):/) // try to match @_discord_123456, @_discordpuppet_123456, etc.
+					if (match) return `${attributeValue} data-user-id="${match[1]}">`
+					// Nope, just a real Matrix user.
+					return whole
+				}
+			})
+
+			// Handling mentions of rooms and room-messages
+			input = await handleRoomOrMessageLinks(input, di)
+
+			// Stripping colons after mentions
+			input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1")
+			input = input.replace(/("https:\/\/matrix.to.*?<\/a>):?/g, "$1")
+
+			// Element adds a bunch of <br> before </blockquote> but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those.
+			input = input.replace(/(?:\n|<br ?\/?>\s*)*<\/blockquote>/g, "</blockquote>")
+
+			// The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason.
+			// But I should not count it if it's between block elements.
+			input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => {
+				// console.error(beforeContext, beforeTag, afterContext, afterTag)
+				if (typeof beforeTag !== "string" && typeof afterTag !== "string") {
+					return "<br>"
+				}
+				beforeContext = beforeContext || ""
+				beforeTag = beforeTag || ""
+				afterContext = afterContext || ""
+				afterTag = afterTag || ""
+				if (!mxUtils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !mxUtils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) {
+					return beforeContext + "<br>" + afterContext
+				} else {
+					return whole
+				}
+			})
+
+			// Note: Element's renderers on Web and Android currently collapse whitespace, like the browser does. Turndown also collapses whitespace which is good for me.
+			// If later I'm using a client that doesn't collapse whitespace and I want turndown to follow suit, uncomment the following line of code, and it Just Works:
+			// input = input.replace(/ /g, "&nbsp;")
+			// There is also a corresponding test to uncomment, named "event2message: whitespace is retained"
+
+			// Handling written @mentions: we need to look for candidate Discord members to join to the room
+			// This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here.
+			// We're using the domino parser because Turndown uses the same and can reuse this tree.
+			const doc = domino.createDocument(
+				// DOM parsers arrange elements in the <head> and <body>. Wrapping in a custom element ensures elements are reliably arranged in a single element.
+				'<x-turndown id="turndown-root">' + input + '</x-turndown>'
+			);
+			const root = doc.getElementById("turndown-root");
+			async function forEachNode(node) {
+				for (; node; node = node.nextSibling) {
+					// Check written mentions
+					if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) {
+						const result = await checkWrittenMentions(node.nodeValue, guild, di)
+						if (result) {
+							node.nodeValue = result.content
+							ensureJoined.push(result.ensureJoined)
+						}
+					}
+					// Check for incompatible backticks in code blocks
+					let preNode
+					if (node.nodeType === 3 && node.nodeValue.includes("```") && (preNode = nodeIsChildOf(node, ["PRE"]))) {
+						if (preNode.firstChild?.nodeName === "CODE") {
+							const ext = (preNode.firstChild.className.match(/language-(\S+)/) || [null, "txt"])[1]
+							const filename = `inline_code.${ext}`
+							// Build the replacement <code> node
+							const replacementCode = doc.createElement("code")
+							replacementCode.textContent = `[${filename}]`
+							// Build its containing <span> node
+							const replacement = doc.createElement("span")
+							replacement.appendChild(doc.createTextNode(" "))
+							replacement.appendChild(replacementCode)
+							replacement.appendChild(doc.createTextNode(" "))
+							// Replace the code block with the <span>
+							preNode.replaceWith(replacement)
+							// Upload the code as an attachment
+							const content = getCodeContent(preNode.firstChild)
+							attachments.push({id: String(attachments.length), filename})
+							pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")})
+						}
+					}
+					await forEachNode(node.firstChild)
+				}
+			}
+			await forEachNode(root)
+
+			// SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet.
+			// First we need to determine which emojis are at the end.
+			endOfMessageEmojis = []
+			let match
+			let last = input.length
+			while ((match = input.slice(0, last).match(/<img [^>]*>\s*$/))) {
+				if (!match[0].includes("data-mx-emoticon")) break
+				const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/)
+				if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1])
+				assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec")
+				last = match.index
+			}
+
+			// @ts-ignore bad type from turndown
+			content = turndownService.turndown(root)
+
+			// Put < > around any surviving matrix.to links to hide the URL previews
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^ )]*/, "<$&>")
+
+			// It's designed for commonmark, we need to replace the space-space-newline with just newline
+			content = content.replace(/  \n/g, "\n")
+
+			// If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated
+			if (replyLine && content.startsWith("> ")) content = "\n" + content
+
+			// SPRITE SHEET EMOJIS FEATURE:
+			content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader)
+		} else {
+			// Looks like we're using the plaintext body!
+			content = event.content.body
+
+			if (event.content.msgtype === "m.emote") {
+				content = `* ${displayName} ${content}`
+			}
+
+			content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^ )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews
+
+			const result = await checkWrittenMentions(content, guild, di)
+			if (result) {
+				content = result.content
+				ensureJoined.push(result.ensureJoined)
+			}
+
+			// Markdown needs to be escaped, though take care not to escape the middle of links
+			// @ts-ignore bad type from turndown
+			content = turndownService.escape(content)
+		}
+	} else if (event.type === "m.room.message" && (event.content.msgtype === "m.file" || event.content.msgtype === "m.video" || event.content.msgtype === "m.audio" || event.content.msgtype === "m.image")) {
+		content = ""
+		const filename = event.content.filename || event.content.body
+		// A written `event.content.body` will be bridged to Discord's image `description` which is like alt text.
+		// Bridging as description rather than message content in order to match Matrix clients (Element, Neochat) which treat this as alt text or title text.
+		const description = (event.content.body !== event.content.filename && event.content.filename && event.content.body) || undefined
+		if ("url" in event.content) {
+			// Unencrypted
+			const url = mxUtils.getPublicUrlForMxc(event.content.url)
+			assert(url)
+			attachments.push({id: "0", description, filename})
+			pendingFiles.push({name: filename, url})
+		} else {
+			// Encrypted
+			const url = mxUtils.getPublicUrlForMxc(event.content.file.url)
+			assert(url)
+			assert.equal(event.content.file.key.alg, "A256CTR")
+			attachments.push({id: "0", description, filename})
+			pendingFiles.push({name: filename, url, key: event.content.file.key.k, iv: event.content.file.iv})
+		}
+	} else if (event.type === "m.sticker") {
+		content = ""
+		const url = mxUtils.getPublicUrlForMxc(event.content.url)
+		assert(url)
+		let filename = event.content.body
+		if (event.type === "m.sticker") {
+			let mimetype
+			if (event.content.info?.mimetype?.includes("/")) {
+				mimetype = event.content.info.mimetype
+			} else {
+				const res = await di.fetch(url, {method: "HEAD"})
+				if (res.status === 200) {
+					mimetype = res.headers.get("content-type")
+				}
+				if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`)
+			}
+			filename += "." + mimetype.split("/")[1]
+		}
+		attachments.push({id: "0", filename})
+		pendingFiles.push({name: filename, url})
+	}
+
+	content = displayNameRunoff + replyLine + content
+
+	// Split into 2000 character chunks
+	const chunks = chunk(content, 2000)
+	messages = messages.concat(chunks.map(content => ({
+		content,
+		username: displayNameShortened,
+		avatar_url: avatarURL
+	})))
+
+	if (attachments.length) {
+		// If content is empty (should be the case when uploading a file) then chunk-text will create 0 messages.
+		// There needs to be a message to add attachments to.
+		if (!messages.length) messages.push({
+			content,
+			username: displayNameShortened,
+			avatar_url: avatarURL
+		})
+		messages[0].attachments = attachments
+		// @ts-ignore these will be converted to real files when the message is about to be sent
+		messages[0].pendingFiles = pendingFiles
+	}
+
+	const messagesToEdit = []
+	const messagesToSend = []
+	for (let i = 0; i < messages.length; i++) {
+		const next = messageIDsToEdit[0]
+		if (next) {
+			messagesToEdit.push({id: next, message: messages[i]})
+			messageIDsToEdit.shift()
+		} else {
+			messagesToSend.push(messages[i])
+		}
+	}
+
+	// Ensure there is code coverage for adding, editing, and deleting
+	if (messagesToSend.length) void 0
+	if (messagesToEdit.length) void 0
+	if (messageIDsToEdit.length) void 0
+
+	return {
+		messagesToEdit,
+		messagesToSend,
+		messagesToDelete: messageIDsToEdit,
+		ensureJoined
+	}
+}
+
+module.exports.eventToMessage = eventToMessage

m2d/converters/utils.js

@@ -0,0 +1,222 @@
+// @ts-check
+
+const reg = require("../../matrix/read-registration")
+const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
+const assert = require("assert").strict
+/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
+let hasher = null
+// @ts-ignore
+require("xxhash-wasm")().then(h => hasher = h)
+
+const BLOCK_ELEMENTS = [
+	"ADDRESS", "ARTICLE", "ASIDE", "AUDIO", "BLOCKQUOTE", "BODY", "CANVAS",
+	"CENTER", "DD", "DETAILS", "DIR", "DIV", "DL", "DT", "FIELDSET", "FIGCAPTION", "FIGURE",
+	"FOOTER", "FORM", "FRAMESET", "H1", "H2", "H3", "H4", "H5", "H6", "HEADER",
+	"HGROUP", "HR", "HTML", "ISINDEX", "LI", "MAIN", "MENU", "NAV", "NOFRAMES",
+	"NOSCRIPT", "OL", "OUTPUT", "P", "PRE", "SECTION", "SUMMARY", "TABLE", "TBODY", "TD",
+	"TFOOT", "TH", "THEAD", "TR", "UL"
+]
+const NEWLINE_ELEMENTS = BLOCK_ELEMENTS.concat(["BR"])
+
+/**
+ * Determine whether an event is the bridged representation of a discord message.
+ * Such messages shouldn't be bridged again.
+ * @param {string} sender
+ */
+function eventSenderIsFromDiscord(sender) {
+	// If it's from a user in the bridge's namespace, then it originated from discord
+	// This includes messages sent by the appservice's bot user, because that is what's used for webhooks
+	// TODO: It would be nice if bridge system messages wouldn't trigger this check and could be bridged from matrix to discord, while webhook reflections would remain ignored...
+	// TODO that only applies to the above todo: But you'd have to watch out for the /icon command, where the bridge bot would set the room avatar, and that shouldn't be reflected into the room a second time.
+	if (userRegex.some(x => sender.match(x))) {
+		return true
+	}
+
+	return false
+}
+
+/**
+ * @param {string} mxc
+ * @returns {string?}
+ */
+function getPublicUrlForMxc(mxc) {
+	const avatarURLParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/)
+	if (avatarURLParts) return `${reg.ooye.server_origin}/_matrix/media/r0/download/${avatarURLParts[1]}/${avatarURLParts[2]}`
+	else return null
+}
+
+/**
+ * Event IDs are really big and have more entropy than we need.
+ * If we want to store the event ID in the database, we can store a more compact version by hashing it with this.
+ * I choose a 64-bit non-cryptographic hash as only a 32-bit hash will see birthday collisions unreasonably frequently: https://en.wikipedia.org/wiki/Birthday_attack#Mathematics
+ * xxhash outputs an unsigned 64-bit integer.
+ * Converting to a signed 64-bit integer with no bit loss so that it can be stored in an SQLite integer field as-is: https://www.sqlite.org/fileformat2.html#record_format
+ * This should give very efficient storage with sufficient entropy.
+ * @param {string} eventID
+ */
+function getEventIDHash(eventID) {
+	assert(hasher, "xxhash is not ready yet")
+	if (eventID[0] === "$" && eventID.length >= 13) {
+		eventID = eventID.slice(1) // increase entropy per character to potentially help xxhash
+	}
+	const unsignedHash = hasher.h64(eventID)
+	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
+	return signedHash
+}
+
+class MatrixStringBuilder {
+	constructor() {
+		this.body = ""
+		this.formattedBody = ""
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	add(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			this.body += body
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	addLine(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n"
+			this.body += body
+			const match = this.formattedBody.match(/<\/?([a-zA-Z]+[a-zA-Z0-9]*)[^>]*>\s*$/)
+			if (this.formattedBody.length && (!match || !NEWLINE_ELEMENTS.includes(match[1].toUpperCase()))) this.formattedBody += "<br>"
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	addParagraph(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n\n"
+			this.body += body
+			formattedBody = `<p>${formattedBody}</p>`
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	get() {
+		return {
+			msgtype: "m.text",
+			body: this.body,
+			format: "org.matrix.custom.html",
+			formatted_body: this.formattedBody
+		}
+	}
+}
+
+/**
+ * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers.
+ * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation.
+ * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged.
+ * https://spec.matrix.org/v1.9/appendices/#routing
+ * https://gitdab.com/cadence/out-of-your-element/issues/11
+ * @param {string} roomID
+ * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api
+ */
+async function getViaServers(roomID, api) {
+	const candidates = []
+	const {joined} = await api.getJoinedMembers(roomID)
+	// Candidate 0: The bot's own server name
+	candidates.push(reg.ooye.server_name)
+	// Candidate 1: Highest joined non-sim non-bot power level user in the room
+	// https://github.com/matrix-org/matrix-react-sdk/blob/552c65db98b59406fb49562e537a2721c8505517/src/utils/permalinks/Permalinks.ts#L172
+	try {
+		/** @type {{users?: {[mxid: string]: number}}} */
+		const powerLevels = await api.getStateEvent(roomID, "m.room.power_levels", "")
+		if (powerLevels.users) {
+			const sorted = Object.entries(powerLevels.users).sort((a, b) => b[1] - a[1]) // Highest...
+			for (const power of sorted) {
+				const mxid = power[0]
+				if (!(mxid in joined)) continue // joined...
+				if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot...
+				const match = mxid.match(/:(.*)/)
+				assert(match)
+				if (!candidates.includes(match[1])) {
+					candidates.push(match[1])
+					break
+				}
+			}
+		}
+	} catch (e) {
+		// power levels event not found
+	}
+	// Candidates 2-3: Most popular servers in the room
+	/** @type {Map<string, number>} */
+	const servers = new Map()
+	// We can get the most popular servers if we know the members, so let's process those...
+	Object.keys(joined)
+		.filter(mxid => !mxid.startsWith("@_")) // Quick check
+		.filter(mxid => !userRegex.some(r => mxid.match(r))) // Full check
+		.slice(0, 1000) // Just sample the first thousand real members
+		.map(mxid => {
+			const match = mxid.match(/:(.*)/)
+			assert(match)
+			return match[1]
+		})
+		.filter(server => !server.match(/([a-f0-9:]+:+)+[a-f0-9]+/)) // No IPv6 servers
+		.filter(server => !server.match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)) // No IPv4 servers
+		// I don't care enough to check ACLs
+		.forEach(server => {
+			const existing = servers.get(server)
+			if (!existing) servers.set(server, 1)
+			else servers.set(server, existing + 1)
+		})
+	const serverList = [...servers.entries()].sort((a, b) => b[1] - a[1])
+	for (const server of serverList) {
+		if (!candidates.includes(server[0])) {
+			candidates.push(server[0])
+			if (candidates.length >= 4) break // Can have at most 4 candidate via servers
+		}
+	}
+	return candidates
+}
+
+/**
+ * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers.
+ * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation.
+ * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged.
+ * https://spec.matrix.org/v1.9/appendices/#routing
+ * https://gitdab.com/cadence/out-of-your-element/issues/11
+ * @param {string} roomID
+ * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api
+ * @returns {Promise<URLSearchParams>}
+ */
+async function getViaServersQuery(roomID, api) {
+	const list = await getViaServers(roomID, api)
+	const qs = new URLSearchParams()
+	for (const server of list) {
+		qs.append("via", server)
+	}
+	return qs
+}
+
+module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS
+module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord
+module.exports.getPublicUrlForMxc = getPublicUrlForMxc
+module.exports.getEventIDHash = getEventIDHash
+module.exports.MatrixStringBuilder = MatrixStringBuilder
+module.exports.getViaServers = getViaServers
+module.exports.getViaServersQuery = getViaServersQuery

m2d/converters/utils.test.js

@@ -0,0 +1,178 @@
+// @ts-check
+
+const e = new Error("Custom error")
+
+const {test} = require("supertape")
+const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers} = require("./utils")
+const util = require("util")
+
+/** @param {string[]} mxids */
+function joinedList(mxids) {
+	/** @type {{[mxid: string]: {display_name: null, avatar_url: null}}} */
+	const joined = {}
+	for (const mxid of mxids) {
+		joined[mxid] = {
+			display_name: null,
+			avatar_url: null
+		}
+	}
+	return {joined}
+}
+
+test("sender type: matrix user", t => {
+	t.notOk(eventSenderIsFromDiscord("@cadence:cadence.moe"))
+})
+
+test("sender type: ooye bot", t => {
+	t.ok(eventSenderIsFromDiscord("@_ooye_bot:cadence.moe"))
+})
+
+test("sender type: ooye puppet", t => {
+	t.ok(eventSenderIsFromDiscord("@_ooye_sheep:cadence.moe"))
+})
+
+test("event hash: hash is the same each time", t => {
+	const eventID = "$example"
+	t.equal(getEventIDHash(eventID), getEventIDHash(eventID))
+})
+
+test("event hash: hash is different for different inputs", t => {
+	t.notEqual(getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe1"), getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe2"))
+})
+
+test("MatrixStringBuilder: add, addLine, add same text", t => {
+	const gatewayMessage = {t: "MY_MESSAGE", d: {display: "Custom message data"}}
+	let stackLines = e.stack?.split("\n")
+
+	const builder = new MatrixStringBuilder()
+	builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 <strong>Bridged event from Discord not delivered</strong>")
+	builder.addLine(`Gateway event: ${gatewayMessage.t}`)
+	builder.addLine(e.toString())
+	if (stackLines) {
+		stackLines = stackLines.slice(0, 2)
+		stackLines[1] = stackLines[1].replace(/\\/g, "/").replace(/(\s*at ).*(\/m2d\/)/, "$1.$2")
+		builder.addLine(`Error trace:`, `<details><summary>Error trace</summary>`)
+		builder.add(`\n${stackLines.join("\n")}`, `<pre>${stackLines.join("\n")}</pre></details>`)
+	}
+	builder.addLine("", `<details><summary>Original payload</summary><pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`)
+
+	t.deepEqual(builder.get(), {
+		msgtype: "m.text",
+		body: "\u26a0 Bridged event from Discord not delivered"
+			+ "\nGateway event: MY_MESSAGE"
+			+ "\nError: Custom error"
+			+ "\nError trace:"
+			+ "\nError: Custom error"
+			+ "\n    at ./m2d/converters/utils.test.js:3:11)\n",
+		format: "org.matrix.custom.html",
+		formatted_body: "\u26a0 <strong>Bridged event from Discord not delivered</strong>"
+			+ "<br>Gateway event: MY_MESSAGE"
+			+ "<br>Error: Custom error"
+			+ "<br><details><summary>Error trace</summary><pre>Error: Custom error\n    at ./m2d/converters/utils.test.js:3:11)</pre></details>"
+			+ `<details><summary>Original payload</summary><pre>{ display: 'Custom message data' }</pre></details>`
+	})
+})
+
+test("MatrixStringBuilder: complete code coverage", t => {
+	const builder = new MatrixStringBuilder()
+	builder.add("Line 1")
+	builder.addParagraph("Line 2")
+	builder.add("Line 3")
+	builder.addParagraph("Line 4")
+
+	t.deepEqual(builder.get(), {
+		msgtype: "m.text",
+		body: "Line 1\n\nLine 2Line 3\n\nLine 4",
+		format: "org.matrix.custom.html",
+		formatted_body: "Line 1<p>Line 2</p>Line 3<p>Line 4</p>"
+	})
+})
+
+test("getViaServers: returns the server name if the room only has sim users", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe"])
+	})
+	t.deepEqual(result, ["cadence.moe"])
+})
+
+test("getViaServers: also returns the most popular servers in order", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "thecollective.invalid", "selfhosted.invalid"])
+})
+
+test("getViaServers: does not return IP address servers", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:45.77.232.172:8443", "@cadence:[::1]:8443", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "123example.456example.invalid"])
+})
+
+test("getViaServers: also returns the highest power level user (100)", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@singleuser:selfhosted.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@moderator:tractor.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "selfhosted.invalid", "thecollective.invalid", "tractor.invalid"])
+})
+
+test("getViaServers: also returns the highest power level user (50)", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"])
+})
+
+test("getViaServers: returns at most 4 results", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@singleuser:selfhosted.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result.length, 4)
+})
+
+test("getViaServers: returns results even when power levels can't be fetched", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => {
+			throw new Error("event not found or something")
+		},
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result.length, 4)
+})
+
+test("getViaServers: only considers power levels of currently joined members", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@former_moderator:missing.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"])
+})

m2d/event-dispatcher.js

@@ -0,0 +1,152 @@
+// @ts-check
+
+/*
+ * Grab Matrix events we care about, check them, and bridge them.
+ */
+
+const util = require("util")
+const Ty = require("../types")
+const {discord, db, sync, as} = require("../passthrough")
+
+/** @type {import("./actions/send-event")} */
+const sendEvent = sync.require("./actions/send-event")
+/** @type {import("./actions/add-reaction")} */
+const addReaction = sync.require("./actions/add-reaction")
+/** @type {import("./actions/redact")} */
+const redact = sync.require("./actions/redact")
+/** @type {import("../matrix/matrix-command-handler")} */
+const matrixCommandHandler = sync.require("../matrix/matrix-command-handler")
+/** @type {import("./converters/utils")} */
+const utils = sync.require("./converters/utils")
+/** @type {import("../matrix/api")}) */
+const api = sync.require("../matrix/api")
+/** @type {import("../matrix/read-registration")}) */
+const reg = sync.require("../matrix/read-registration")
+
+let lastReportedEvent = 0
+
+function guard(type, fn) {
+	return async function(event, ...args) {
+		try {
+			return await fn(event, ...args)
+		} catch (e) {
+			console.error("hit event-dispatcher's error handler with this exception:")
+			console.error(e) // TODO: also log errors into a file or into the database, maybe use a library for this? or just wing it?
+			console.error(`while handling this ${type} gateway event:`)
+			console.dir(event, {depth: null})
+
+			if (Date.now() - lastReportedEvent < 5000) return
+			lastReportedEvent = Date.now()
+
+			let stackLines = e.stack.split("\n")
+			api.sendEvent(event.room_id, "m.room.message", {
+				msgtype: "m.text",
+				body: "\u26a0 Matrix event not delivered to Discord. See formatted content for full details.",
+				format: "org.matrix.custom.html",
+				formatted_body: "\u26a0 <strong>Matrix event not delivered to Discord</strong>"
+					+ `<br>Event type: ${type}`
+					+ `<br>${e.toString()}`
+					+ `<br><details><summary>Error trace</summary>`
+					+ `<pre>${stackLines.join("\n")}</pre></details>`
+					+ `<details><summary>Original payload</summary>`
+					+ `<pre>${util.inspect(event, false, 4, false)}</pre></details>`,
+				"moe.cadence.ooye.error": {
+					source: "matrix",
+					payload: event
+				},
+				"m.mentions": {
+					user_ids: ["@cadence:cadence.moe"]
+				}
+			})
+		}
+	}
+}
+
+async function retry(roomID, eventID) {
+	const event = await api.getEvent(roomID, eventID)
+	const error = event.content["moe.cadence.ooye.error"]
+	if (event.sender !== `@${reg.sender_localpart}:${reg.ooye.server_name}` || !error) return
+	if (error.source === "matrix") {
+		as.emit("type:" + error.payload.type, error.payload)
+	} else if (error.source === "discord") {
+		discord.cloud.emit("event", error.payload)
+	}
+}
+
+sync.addTemporaryListener(as, "type:m.room.message", guard("m.room.message",
+/**
+ * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File} event it is a m.room.message because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const messageResponses = await sendEvent.sendEvent(event)
+	if (event.type === "m.room.message" && event.content.msgtype === "m.text") {
+		// @ts-ignore
+		await matrixCommandHandler.execute(event)
+	}
+}))
+
+sync.addTemporaryListener(as, "type:m.sticker", guard("m.sticker",
+/**
+ * @param {Ty.Event.Outer_M_Sticker} event it is a m.sticker because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const messageResponses = await sendEvent.sendEvent(event)
+}))
+
+sync.addTemporaryListener(as, "type:m.reaction", guard("m.reaction",
+/**
+ * @param {Ty.Event.Outer<Ty.Event.M_Reaction>} event it is a m.reaction because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	if (event.content["m.relates_to"].key === "🔁") {
+		// Try to bridge a failed event again?
+		await retry(event.room_id, event.content["m.relates_to"].event_id)
+	} else {
+		matrixCommandHandler.onReactionAdd(event)
+		await addReaction.addReaction(event)
+	}
+}))
+
+sync.addTemporaryListener(as, "type:m.room.redaction", guard("m.room.redaction",
+/**
+ * @param {Ty.Event.Outer_M_Room_Redaction} event it is a m.room.redaction because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	await redact.handle(event)
+}))
+
+sync.addTemporaryListener(as, "type:m.room.avatar", guard("m.room.avatar",
+/**
+ * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Avatar>} event
+ */
+async event => {
+	if (event.state_key !== "") return
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const url = event.content.url || null
+	db.prepare("UPDATE channel_room SET custom_avatar = ? WHERE room_id = ?").run(url, event.room_id)
+}))
+
+sync.addTemporaryListener(as, "type:m.room.name", guard("m.room.name",
+/**
+ * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Name>} event
+ */
+async event => {
+	if (event.state_key !== "") return
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const name = event.content.name || null
+	db.prepare("UPDATE channel_room SET nick = ? WHERE room_id = ?").run(name, event.room_id)
+}))
+
+sync.addTemporaryListener(as, "type:m.room.member", guard("m.room.member",
+/**
+ * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Member>} event
+ */
+async event => {
+	if (event.state_key[0] !== "@") return
+	if (utils.eventSenderIsFromDiscord(event.state_key)) return
+	db.prepare("REPLACE INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(event.room_id, event.state_key, event.content.displayname || null, event.content.avatar_url || null)
+}))

matrix/api.js

@@ -0,0 +1,249 @@
+// @ts-check
+
+const Ty = require("../types")
+const assert = require("assert").strict
+
+const passthrough = require("../passthrough")
+const { discord, sync, db } = passthrough
+/** @type {import("./mreq")} */
+const mreq = sync.require("./mreq")
+/** @type {import("./file")} */
+const file = sync.require("./file")
+/** @type {import("./txnid")} */
+const makeTxnId = sync.require("./txnid")
+
+/**
+ * @param {string} p endpoint to access
+ * @param {string?} [mxid] optional: user to act as, for the ?user_id parameter
+ * @param {{[x: string]: any}} [otherParams] optional: any other query parameters to add
+ * @returns {string} the new endpoint
+ */
+function path(p, mxid, otherParams = {}) {
+	const u = new URL(p, "http://localhost")
+	if (mxid) u.searchParams.set("user_id", mxid)
+	for (const entry of Object.entries(otherParams)) {
+		if (entry[1] != undefined) {
+			u.searchParams.set(entry[0], entry[1])
+		}
+	}
+	let result = u.pathname
+	const str = u.searchParams.toString()
+	if (str) result += "?" + str
+	return result
+}
+
+/**
+ * @param {string} username
+ * @returns {Promise<Ty.R.Registered>}
+ */
+function register(username) {
+	console.log(`[api] register: ${username}`)
+	return mreq.mreq("POST", "/client/v3/register", {
+		type: "m.login.application_service",
+		username
+	})
+}
+
+/**
+ * @returns {Promise<string>} room ID
+ */
+async function createRoom(content) {
+	console.log(`[api] create room:`, content)
+	/** @type {Ty.R.RoomCreated} */
+	const root = await mreq.mreq("POST", "/client/v3/createRoom", content)
+	return root.room_id
+}
+
+/**
+ * @returns {Promise<string>} room ID
+ */
+async function joinRoom(roomIDOrAlias, mxid) {
+	/** @type {Ty.R.RoomJoined} */
+	const root = await mreq.mreq("POST", path(`/client/v3/join/${roomIDOrAlias}`, mxid))
+	return root.room_id
+}
+
+async function inviteToRoom(roomID, mxidToInvite, mxid) {
+	await mreq.mreq("POST", path(`/client/v3/rooms/${roomID}/invite`, mxid), {
+		user_id: mxidToInvite
+	})
+}
+
+async function leaveRoom(roomID, mxid) {
+	await mreq.mreq("POST", path(`/client/v3/rooms/${roomID}/leave`, mxid), {})
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} eventID
+ * @template T
+ */
+async function getEvent(roomID, eventID) {
+	/** @type {Ty.Event.Outer<T>} */
+	const root = await mreq.mreq("GET", `/client/v3/rooms/${roomID}/event/${eventID}`)
+	return root
+}
+
+/**
+ * @param {string} roomID
+ * @param {number} ts unix silliseconds
+ */
+async function getEventForTimestamp(roomID, ts) {
+	/** @type {{event_id: string, origin_server_ts: number}} */
+	const root = await mreq.mreq("GET", path(`/client/v1/rooms/${roomID}/timestamp_to_event`, null, {ts}))
+	return root
+}
+
+/**
+ * @param {string} roomID
+ * @returns {Promise<Ty.Event.BaseStateEvent[]>}
+ */
+function getAllState(roomID) {
+	return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state`)
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} type
+ * @param {string} key
+ * @returns the *content* of the state event
+ */
+function getStateEvent(roomID, type, key) {
+	return mreq.mreq("GET", `/client/v3/rooms/${roomID}/state/${type}/${key}`)
+}
+
+/**
+ * "Any of the AS's users must be in the room. This API is primarily for Application Services and should be faster to respond than /members as it can be implemented more efficiently on the server."
+ * @param {string} roomID
+ * @returns {Promise<{joined: {[mxid: string]: {avatar_url: string?, display_name: string?}}}>}
+ */
+function getJoinedMembers(roomID) {
+	return mreq.mreq("GET", `/client/v3/rooms/${roomID}/joined_members`)
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} eventID
+ * @param {{from?: string, limit?: any}} pagination
+ * @param {string?} [relType]
+ * @returns {Promise<Ty.Pagination<Ty.Event.Outer<any>>>}
+ */
+function getRelations(roomID, eventID, pagination, relType) {
+	let path = `/client/v1/rooms/${roomID}/relations/${eventID}`
+	if (relType) path += `/${relType}`
+	if (!pagination.from) delete pagination.from
+	if (!pagination.limit) pagination.limit = 50 // get a little more consistency between homeservers
+	path += `?${new URLSearchParams(pagination)}`
+	return mreq.mreq("GET", path)
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} type
+ * @param {string} stateKey
+ * @param {string} [mxid]
+ * @returns {Promise<string>} event ID
+ */
+async function sendState(roomID, type, stateKey, content, mxid) {
+	console.log(`[api] state: ${roomID}: ${type}/${stateKey}`)
+	assert.ok(type)
+	assert.ok(typeof stateKey === "string")
+	/** @type {Ty.R.EventSent} */
+	// encodeURIComponent is necessary because state key can contain some special characters like / but you must encode them so they fit in a single component of the URI
+	const root = await mreq.mreq("PUT", path(`/client/v3/rooms/${roomID}/state/${type}/${encodeURIComponent(stateKey)}`, mxid), content)
+	return root.event_id
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} type
+ * @param {any} content
+ * @param {string?} [mxid]
+ * @param {number} [timestamp] timestamp of the newly created event, in unix milliseconds
+ */
+async function sendEvent(roomID, type, content, mxid, timestamp) {
+	if (!["m.room.message", "m.reaction", "m.sticker"].includes(type)) {
+		console.log(`[api] event ${type} to ${roomID} as ${mxid || "default sim"}`)
+	}
+	/** @type {Ty.R.EventSent} */
+	const root = await mreq.mreq("PUT", path(`/client/v3/rooms/${roomID}/send/${type}/${makeTxnId.makeTxnId()}`, mxid, {ts: timestamp}), content)
+	return root.event_id
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} eventID
+ * @param {string?} [mxid]
+ * @returns {Promise<string>} event ID
+ */
+async function redactEvent(roomID, eventID, mxid) {
+	/** @type {Ty.R.EventRedacted} */
+	const root = await mreq.mreq("PUT", path(`/client/v3/rooms/${roomID}/redact/${eventID}/${makeTxnId.makeTxnId()}`, mxid), {})
+	return root.event_id
+}
+
+/**
+ * @param {string} roomID
+ * @param {boolean} isTyping
+ * @param {string} mxid
+ * @param {number} [duration] milliseconds
+ */
+async function sendTyping(roomID, isTyping, mxid, duration) {
+	await mreq.mreq("PUT", path(`/client/v3/rooms/${roomID}/typing/${mxid}`, mxid), {
+		typing: isTyping,
+		timeout: duration
+	})
+}
+
+async function profileSetDisplayname(mxid, displayname) {
+	await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/displayname`, mxid), {
+		displayname
+	})
+}
+
+async function profileSetAvatarUrl(mxid, avatar_url) {
+	await mreq.mreq("PUT", path(`/client/v3/profile/${mxid}/avatar_url`, mxid), {
+		avatar_url
+	})
+}
+
+/**
+ * Set a user's power level within a room.
+ * @param {string} roomID
+ * @param {string} mxid
+ * @param {number} power
+ */
+async function setUserPower(roomID, mxid, power) {
+	assert(roomID[0] === "!")
+	assert(mxid[0] === "@")
+	// Yes there's no shortcut https://github.com/matrix-org/matrix-appservice-bridge/blob/2334b0bae28a285a767fe7244dad59f5a5963037/src/components/intent.ts#L352
+	const powerLevels = await getStateEvent(roomID, "m.room.power_levels", "")
+	powerLevels.users = powerLevels.users || {}
+	if (power != null) {
+		powerLevels.users[mxid] = power
+	} else {
+		delete powerLevels.users[mxid]
+	}
+	await sendState(roomID, "m.room.power_levels", "", powerLevels)
+	return powerLevels
+}
+
+module.exports.path = path
+module.exports.register = register
+module.exports.createRoom = createRoom
+module.exports.joinRoom = joinRoom
+module.exports.inviteToRoom = inviteToRoom
+module.exports.leaveRoom = leaveRoom
+module.exports.getEvent = getEvent
+module.exports.getEventForTimestamp = getEventForTimestamp
+module.exports.getAllState = getAllState
+module.exports.getStateEvent = getStateEvent
+module.exports.getJoinedMembers = getJoinedMembers
+module.exports.getRelations = getRelations
+module.exports.sendState = sendState
+module.exports.sendEvent = sendEvent
+module.exports.redactEvent = redactEvent
+module.exports.sendTyping = sendTyping
+module.exports.profileSetDisplayname = profileSetDisplayname
+module.exports.profileSetAvatarUrl = profileSetAvatarUrl
+module.exports.setUserPower = setUserPower

matrix/api.test.js

@@ -24,7 +24,3 @@ test("api path: real world mxid", t => {
 test("api path: extras number works", t => {
 	t.equal(path(`/client/v3/rooms/!example/timestamp_to_event`, null, {ts: 1687324651120}), "/client/v3/rooms/!example/timestamp_to_event?ts=1687324651120")
 })
-
-test("api path: multiple via params", t => {
-	t.equal(path(`/client/v3/rooms/!example/join`, null, {via: ["cadence.moe", "matrix.org"], ts: 1687324651120}), "/client/v3/rooms/!example/join?via=cadence.moe&via=matrix.org&ts=1687324651120")
-})

matrix/appservice.js

@@ -0,0 +1,8 @@
+const reg = require("../matrix/read-registration")
+const AppService = require("matrix-appservice").AppService
+const as = new AppService({
+	homeserverToken: reg.hs_token
+})
+as.listen(+(new URL(reg.url).port))
+
+module.exports = as

matrix/file.js

@@ -1,9 +1,8 @@
 // @ts-check
 
-const passthrough = require("../passthrough")
-const {reg, writeRegistration} = require("./read-registration.js")
-const Ty = require("../types")
+const fetch = require("node-fetch").default
 
+const passthrough = require("../passthrough")
 const {sync, db, select} = passthrough
 /** @type {import("./mreq")} */
 const mreq = sync.require("./mreq")
@@ -47,8 +46,11 @@ async function uploadDiscordFileToMxc(path) {
 		return existingFromDb
 	}
 
-	// Download from Discord and upload to Matrix
-	const promise = module.exports._actuallyUploadDiscordFileToMxc(url).then(root => {
+	// Download from Discord
+	const promise = fetch(url, {}).then(/** @param {import("node-fetch").Response} res */ async res => {
+		// Upload to Matrix
+		const root = await module.exports._actuallyUploadDiscordFileToMxc(urlNoExpiry, res)
+
 		// Store relationship in database
 		db.prepare("INSERT INTO file (discord_url, mxc_url) VALUES (?, ?)").run(urlNoExpiry, root.content_uri)
 		inflight.delete(urlNoExpiry)
@@ -60,33 +62,15 @@ async function uploadDiscordFileToMxc(path) {
 	return promise
 }
 
-/**
- * @param {string} url
- * @returns {Promise<Ty.R.FileUploaded>}
- */
-async function _actuallyUploadDiscordFileToMxc(url) {
-	const res = await fetch(url, {})
-	try {
-		/** @type {Ty.R.FileUploaded} */
-		const root = await mreq.mreq("POST", "/media/v3/upload", res.body, {
-			headers: {
-				"Content-Type": res.headers.get("content-type")
-			}
-		})
-		return root
-	} catch (e) {
-		if (e instanceof mreq.MatrixServerError && e.data.error?.includes("Content-Length") && !reg.ooye.content_length_workaround) {
-			reg.ooye.content_length_workaround = true
-			const root = await _actuallyUploadDiscordFileToMxc(url)
-			console.error("OOYE cannot stream uploads to Synapse. The `content_length_workaround` option"
-				+ "\nhas been activated in registration.yaml, which works around the problem, but"
-				+ "\nhalves the speed of bridging d->m files. A better way to resolve this problem"
-				+ "\nis to run an nginx reverse proxy to Synapse and re-run OOYE setup.")
-			writeRegistration(reg)
-			return root
+async function _actuallyUploadDiscordFileToMxc(url, res) {
+	const body = res.body
+	/** @type {import("../types").R.FileUploaded} */
+	const root = await mreq.mreq("POST", "/media/v3/upload", body, {
+		headers: {
+			"Content-Type": res.headers.get("content-type")
 		}
-		throw e
-	}
+	})
+	return root
 }
 
 function guildIcon(guild) {
@@ -98,28 +82,29 @@ function userAvatar(user) {
 }
 
 function memberAvatar(guildID, user, member) {
-	if (!member?.avatar) return userAvatar(user)
-	return `/guilds/${guildID}/users/${user.id}/avatars/${member?.avatar}.png?size=${IMAGE_SIZE}`
+	if (!member.avatar) return userAvatar(user)
+	return `/guilds/${guildID}/users/${user.id}/avatars/${member.avatar}.png?size=${IMAGE_SIZE}`
 }
 
 function emoji(emojiID, animated) {
-	const base = `/emojis/${emojiID}.webp`
-	if (animated) return base + "?animated=true"
-	else return base
+	const base = `/emojis/${emojiID}`
+	if (animated) return base + ".gif"
+	else return base + ".png"
 }
 
 const stickerFormat = new Map([
-	[1, {label: "PNG", ext: "png", mime: "image/png", endpoint: "/stickers/"}],
-	[2, {label: "APNG", ext: "png", mime: "image/apng", endpoint: "/stickers/"}],
-	[3, {label: "LOTTIE", ext: "json", mime: "lottie", endpoint: "/stickers/"}],
-	[4, {label: "GIF", ext: "gif", mime: "image/gif", endpoint: "https://media.discordapp.net/stickers/"}]
+	[1, {label: "PNG", ext: "png", mime: "image/png"}],
+	[2, {label: "APNG", ext: "png", mime: "image/apng"}],
+	[3, {label: "LOTTIE", ext: "json", mime: "lottie"}],
+	[4, {label: "GIF", ext: "gif", mime: "image/gif"}]
 ])
 
 /** @param {{id: string, format_type: number}} sticker */
 function sticker(sticker) {
 	const format = stickerFormat.get(sticker.format_type)
 	if (!format) throw new Error(`No such format ${sticker.format_type} for sticker ${JSON.stringify(sticker)}`)
-	return `${format.endpoint}${sticker.id}.${format.ext}`
+	const ext = format.ext
+	return `/stickers/${sticker.id}.${ext}`
 }
 
 module.exports.DISCORD_IMAGES_BASE = DISCORD_IMAGES_BASE

matrix/kstate.js

@@ -0,0 +1,79 @@
+// @ts-check
+
+const assert = require("assert").strict
+const mixin = require("mixin-deep")
+const deepEqual = require("deep-equal")
+
+/** Mutates the input. */
+function kstateStripConditionals(kstate) {
+	for (const [k, content] of Object.entries(kstate)) {
+		// conditional for whether a key is even part of the kstate (doing this declaratively on json is hard, so represent it as a property instead.)
+		if ("$if" in content) {
+			if (content.$if) delete content.$if
+			else delete kstate[k]
+		}
+	}
+	return kstate
+}
+
+function kstateToState(kstate) {
+	const events = []
+	kstateStripConditionals(kstate)
+	for (const [k, content] of Object.entries(kstate)) {
+		const slashIndex = k.indexOf("/")
+		assert(slashIndex > 0)
+		const type = k.slice(0, slashIndex)
+		const state_key = k.slice(slashIndex + 1)
+		events.push({type, state_key, content})
+	}
+	return events
+}
+
+/**
+ * @param {import("../types").Event.BaseStateEvent[]} events
+ * @returns {any}
+ */
+function stateToKState(events) {
+	const kstate = {}
+	for (const event of events) {
+		kstate[event.type + "/" + event.state_key] = event.content
+	}
+	return kstate
+}
+
+function diffKState(actual, target) {
+	const diff = {}
+	// go through each key that it should have
+	for (const key of Object.keys(target)) {
+		if (!key.includes("/")) throw new Error(`target kstate's key "${key}" does not contain a slash separator; if a blank state_key was intended, add a trailing slash to the kstate key.\ncontext: ${JSON.stringify(target)}`)
+
+		if (key === "m.room.power_levels/") {
+			// Special handling for power levels, we want to deep merge the actual and target into the final state.
+			if (!(key in actual)) throw new Error(`want to apply a power levels diff, but original power level data is missing\nstarted with:  ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`)
+			const temp = mixin({}, actual[key], target[key])
+			if (!deepEqual(actual[key], temp, {strict: true})) {
+				// they differ. use the newly prepared object as the diff.
+				diff[key] = temp
+			}
+
+		} else if (key in actual) {
+			// diff
+			if (!deepEqual(actual[key], target[key], {strict: true})) {
+				// they differ. use the target as the diff.
+				diff[key] = target[key]
+			}
+
+		} else {
+			// not present, needs to be added
+			diff[key] = target[key]
+		}
+
+		// keys that are missing in "actual" will not be deleted on "target" (no action)
+	}
+	return diff
+}
+
+module.exports.kstateStripConditionals = kstateStripConditionals
+module.exports.kstateToState = kstateToState
+module.exports.stateToKState = stateToKState
+module.exports.diffKState = diffKState

matrix/kstate.test.js

@@ -0,0 +1,191 @@
+const assert = require("assert")
+const {kstateToState, stateToKState, diffKState, kstateStripConditionals} = require("./kstate")
+const {test} = require("supertape")
+
+test("kstate strip: strips false conditions", t => {
+	t.deepEqual(kstateStripConditionals({
+		a: {$if: false, value: 2},
+		b: {value: 4}
+	}), {
+		b: {value: 4}
+	})
+})
+
+test("kstate strip: keeps true conditions while removing $if", t => {
+	t.deepEqual(kstateStripConditionals({
+		a: {$if: true, value: 2},
+		b: {value: 4}
+	}), {
+		a: {value: 2},
+		b: {value: 4}
+	})
+})
+
+test("kstate2state: general", t => {
+	t.deepEqual(kstateToState({
+		"m.room.name/": {name: "test name"},
+		"m.room.member/@cadence:cadence.moe": {membership: "join"},
+		"uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"}
+	}), [
+		{
+			type: "m.room.name",
+			state_key: "",
+			content: {
+				name: "test name"
+			}
+		},
+		{
+			type: "m.room.member",
+			state_key: "@cadence:cadence.moe",
+			content: {
+				membership: "join"
+			}
+		},
+		{
+			type: "uk.half-shot.bridge",
+			state_key: "org.matrix.appservice-irc://irc/epicord.net/#general",
+			content: {
+				creator: "@cadence:cadence.moe"
+			}
+		}
+	])
+})
+
+test("state2kstate: general", t => {
+	t.deepEqual(stateToKState([
+		{
+			type: "m.room.name",
+			state_key: "",
+			content: {
+				name: "test name"
+			}
+		},
+		{
+			type: "m.room.member",
+			state_key: "@cadence:cadence.moe",
+			content: {
+				membership: "join"
+			}
+		},
+		{
+			type: "uk.half-shot.bridge",
+			state_key: "org.matrix.appservice-irc://irc/epicord.net/#general",
+			content: {
+				creator: "@cadence:cadence.moe"
+			}
+		}
+	]), {
+		"m.room.name/": {name: "test name"},
+		"m.room.member/@cadence:cadence.moe": {membership: "join"},
+		"uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"}
+	})
+})
+
+test("diffKState: detects edits", t => {
+	t.deepEqual(
+		diffKState({
+			"m.room.name/": {name: "test name"},
+			"same/": {a: 2}
+		}, {
+			"m.room.name/": {name: "edited name"},
+			"same/": {a: 2}
+		}),
+		{
+			"m.room.name/": {name: "edited name"}
+		}
+	)
+})
+
+test("diffKState: detects new properties", t => {
+	t.deepEqual(
+		diffKState({
+			"m.room.name/": {name: "test name"},
+		}, {
+			"m.room.name/": {name: "test name"},
+			"new/": {a: 2}
+		}),
+		{
+			"new/": {a: 2}
+		}
+	)
+})
+
+test("diffKState: power levels are mixed together", t => {
+	const original = {
+		"m.room.power_levels/": {
+			"ban": 50,
+			"events": {
+				"m.room.name": 100,
+				"m.room.power_levels": 100
+			},
+			"events_default": 0,
+			"invite": 50,
+			"kick": 50,
+			"notifications": {
+				"room": 20
+			},
+			"redact": 50,
+			"state_default": 50,
+			"users": {
+				"@example:localhost": 100
+			},
+			"users_default": 0
+		}
+	}
+	const result = diffKState(original, {
+		"m.room.power_levels/": {
+			"events": {
+				"m.room.avatar": 0
+			}
+		}
+	})
+	t.deepEqual(result, {
+		"m.room.power_levels/": {
+			"ban": 50,
+			"events": {
+				"m.room.name": 100,
+				"m.room.power_levels": 100,
+				"m.room.avatar": 0
+			},
+			"events_default": 0,
+			"invite": 50,
+			"kick": 50,
+			"notifications": {
+				"room": 20
+			},
+			"redact": 50,
+			"state_default": 50,
+			"users": {
+				"@example:localhost": 100
+			},
+			"users_default": 0
+		}
+	})
+	t.notDeepEqual(original, result)
+})
+
+test("diffKState: cannot merge power levels if original power levels are missing", t => {
+	const original = {}
+	assert.throws(() =>
+		diffKState(original, {
+			"m.room.power_levels/": {
+				"events": {
+					"m.room.avatar": 0
+				}
+			}
+		})
+	, /original power level data is missing/)
+	t.pass()
+})
+
+test("diffKState: kstate keys must contain a slash separator", t => {
+	assert.throws(() =>
+		diffKState({
+			"m.room.name/": {name: "test name"},
+		}, {
+			"m.room.name/": {name: "test name"},
+			"new": {a: 2}
+		})
+	, /does not contain a slash separator/)
+	t.pass()
+})

matrix/matrix-command-handler.js

@@ -8,13 +8,13 @@ const sharp = require("sharp")
 const {discord, sync, db, select} = require("../passthrough")
 /** @type {import("./api")}) */
 const api = sync.require("./api")
-/** @type {import("./utils")} */
-const mxUtils = sync.require("./utils")
+/** @type {import("../m2d/converters/utils")} */
+const mxUtils = sync.require("../m2d/converters/utils")
 /** @type {import("../discord/utils")} */
 const dUtils = sync.require("../discord/utils")
 /** @type {import("./kstate")} */
 const ks = sync.require("./kstate")
-const {reg} = require("./read-registration")
+const reg = require("./read-registration")
 
 const PREFIXES = ["//", "/"]
 
@@ -58,7 +58,7 @@ async function addButton(roomID, eventID, key, mxid) {
 setInterval(() => {
 	const now = Date.now()
 	buttons = buttons.filter(b => now - b.created < 2*60*60*1000)
-}, 10*60*1000).unref()
+}, 10*60*1000)
 
 /** @param {Ty.Event.Outer<Ty.Event.M_Reaction>} event */
 function onReactionAdd(event) {
@@ -114,7 +114,7 @@ const commands = [{
 				const guild = discord.guilds.get(guildID)
 				assert(guild)
 				const slots = getSlotCount(guild.premium_tier)
-				const permissions = dUtils.getPermissions(guild.id, [], guild.roles)
+				const permissions = dUtils.getPermissions([], guild.roles)
 				if (guild.emojis.length >= slots) {
 					matrixOnlyReason = "CAPACITY"
 				} else if (!(permissions & 0x40000000n)) { // MANAGE_GUILD_EXPRESSIONS (apparently CREATE_GUILD_EXPRESSIONS isn't good enough...)
@@ -123,9 +123,12 @@ const commands = [{
 			}
 			if (matrixOnlyReason) {
 				// If uploading to Matrix, check if we have permission
-				const {powerLevels, powers: {[mxUtils.bot]: botPower}} = await mxUtils.getEffectivePower(event.room_id, [mxUtils.bot], api)
-				const requiredPower = powerLevels.events?.["im.ponies.room_emotes"] ?? powerLevels.state_default ?? 50
-				if (botPower < requiredPower) {
+				const state = await api.getAllState(event.room_id)
+				const kstate = ks.stateToKState(state)
+				const powerLevels = kstate["m.room.power_levels/"]
+				const required = powerLevels.events["im.ponies.room_emotes"] ?? powerLevels.state_default ?? 50
+				const have = powerLevels.users[`@${reg.sender_localpart}:${reg.ooye.server_name}`] ?? powerLevels.users_default ?? 0
+				if (have < required) {
 					return api.sendEvent(event.room_id, "m.room.message", {
 						...ctx,
 						msgtype: "m.text",
@@ -174,7 +177,7 @@ const commands = [{
 				.addLine(`Ⓜ️ *If you were a Discord user, you wouldn't have permission to create emojis. ${matrixOnlyConclusion}`, `Ⓜ️ <em>If you were a Discord user, you wouldn't have permission to create emojis. ${matrixOnlyConclusion}</em>`, matrixOnlyReason === "CAPACITY")
 				.addLine("[Preview not available in plain text.]", "Preview:")
 			for (const e of toUpload) {
-				b.add("", `:${e.name}: <img data-mx-emoticon height="48" src="${e.url}" title=":${e.name}:" alt=":${e.name}:">`)
+				b.add("", `<img data-mx-emoticon height="48" src="${e.url}" title=":${e.name}:" alt=":${e.name}:">`)
 			}
 			b.addLine("Hit ✅ to add it.")
 			const sent = await api.sendEvent(event.room_id, "m.room.message", {
@@ -214,14 +217,15 @@ const commands = [{
 				} else {
 					// Upload it to Discord and have the bridge sync it back to Matrix again
 					for (const e of toUpload) {
+						const publicUrl = mxUtils.getPublicUrlForMxc(e.url)
 						// @ts-ignore
-						const resizeInput = await api.getMedia(e.url, {agent: false}).then(res => res.arrayBuffer())
+						const resizeInput = await fetch(publicUrl, {agent: false}).then(res => res.arrayBuffer())
 						const resizeOutput = await sharp(resizeInput)
 							.resize(EMOJI_SIZE, EMOJI_SIZE, {fit: "inside", withoutEnlargement: true, background: {r: 0, g: 0, b: 0, alpha: 0}})
 							.png()
 							.toBuffer({resolveWithObject: true})
 						console.log(`uploading emoji ${resizeOutput.data.length} bytes to :${e.name}:`)
-						await discord.snow.assets.createGuildEmoji(guildID, {name: e.name, image: "data:image/png;base64," + resizeOutput.data.toString("base64")})
+						const emoji = await discord.snow.guildAssets.createEmoji(guildID, {name: e.name, image: "data:image/png;base64," + resizeOutput.data.toString("base64")})
 					}
 					api.sendEvent(event.room_id, "m.room.message", {
 						...ctx,
@@ -250,7 +254,7 @@ const commands = [{
 
 			const guild = discord.guilds.get(guildID)
 			assert(guild)
-			const permissions = dUtils.getPermissions(guild.id, [], guild.roles)
+			const permissions = dUtils.getPermissions([], guild.roles)
 			if (!(permissions & 0x800000000n)) { // CREATE_PUBLIC_THREADS
 				return api.sendEvent(event.room_id, "m.room.message", {
 					...ctx,

matrix/mreq.js

@@ -0,0 +1,85 @@
+// @ts-check
+
+const fetch = require("node-fetch").default
+const mixin = require("mixin-deep")
+const stream = require("stream")
+const getStream = require("get-stream")
+
+const passthrough = require("../passthrough")
+const { sync } = passthrough
+/** @type {import("./read-registration")} */
+const reg = sync.require("./read-registration.js")
+
+const baseUrl = `${reg.ooye.server_origin}/_matrix`
+
+class MatrixServerError extends Error {
+	constructor(data, opts) {
+		super(data.error || data.errcode)
+		this.data = data
+		/** @type {string} */
+		this.errcode = data.errcode
+		this.opts = opts
+	}
+}
+
+/**
+ * @param {string} method
+ * @param {string} url
+ * @param {any} [body]
+ * @param {any} [extra]
+ */
+async function mreq(method, url, body, extra = {}) {
+	if (body == undefined || Object.is(body.constructor, Object)) {
+		body = JSON.stringify(body)
+	} else if (body instanceof stream.Readable && reg.ooye.content_length_workaround) {
+		body = await getStream.buffer(body)
+	}
+
+	const opts = mixin({
+		method,
+		body,
+		headers: {
+			Authorization: `Bearer ${reg.as_token}`
+		}
+	}, extra)
+
+	// console.log(baseUrl + url, opts)
+	const res = await fetch(baseUrl + url, opts)
+	const root = await res.json()
+
+	if (!res.ok || root.errcode) {
+		if (root.error?.includes("Content-Length")) {
+			console.error(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:`
+				+ `\n  * Run an nginx reverse proxy to Synapse, and point registration.yaml's`
+				+ `\n    \`server_origin\` to nginx`
+				+ `\n  * Set \`content_length_workaround: true\` in registration.yaml (this will`
+				+ `\n    halve the speed of bridging d->m files)`)
+			throw new Error("Synapse is not accepting stream uploads, see the message above.")
+		}
+		delete opts.headers.Authorization
+		throw new MatrixServerError(root, {baseUrl, url, ...opts})
+	}
+	return root
+}
+
+/**
+ * JavaScript doesn't have Racket-like parameters with dynamic scoping, so
+ * do NOT do anything else at the same time as this.
+ * @template T
+ * @param {string} token
+ * @param {(...arg: any[]) => Promise<T>} callback
+ * @returns {Promise<T>}
+ */
+async function withAccessToken(token, callback) {
+	const prevToken = reg.as_token
+	reg.as_token = token
+	try {
+		return await callback()
+	} finally {
+		reg.as_token = prevToken
+	}
+}
+
+module.exports.MatrixServerError = MatrixServerError
+module.exports.mreq = mreq
+module.exports.withAccessToken = withAccessToken

matrix/read-registration.js

@@ -0,0 +1,14 @@
+// @ts-check
+
+const fs = require("fs")
+const assert = require("assert").strict
+const yaml = require("js-yaml")
+
+/** @ts-ignore @type {import("../types").AppServiceRegistrationConfig} */
+const reg = yaml.load(fs.readFileSync("registration.yaml", "utf8"))
+reg["ooye"].invite = (reg.ooye.invite || []).filter(mxid => mxid.endsWith(`:${reg.ooye.server_name}`)) // one day I will understand why typescript disagrees with dot notation on this line
+assert(reg.ooye.max_file_size)
+assert(reg.ooye.namespace_prefix)
+assert(reg.ooye.server_name)
+
+module.exports = reg

matrix/read-registration.test.js

@@ -0,0 +1,10 @@
+const {test} = require("supertape")
+const reg = require("./read-registration")
+
+test("reg: has necessary parameters", t => {
+	const propertiesToCheck = ["sender_localpart", "id", "as_token", "ooye"]
+	t.deepEqual(
+		propertiesToCheck.filter(p => p in reg),
+		propertiesToCheck
+	)
+})