Bridge forums as spaces

I should probably change this to use kstate.

.c8rc.json

@@ -0,0 +1,8 @@
+{
+	"watermarks": {
+		"statements": [60, 100],
+		"lines": [60, 100],
+		"functions": [60, 100],
+		"branches": [60, 100]
+	}
+}

.gitignore

@@ -3,3 +3,5 @@ config.js
 registration.yaml
 coverage
 db/ooye.db*
+test/res/*
+!test/res/lottie*

d2m/actions/announce-thread.js

@@ -8,6 +8,8 @@ const {discord, sync, db, select} = passthrough
 const threadToAnnouncement = sync.require("../converters/thread-to-announcement")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
+/** @type {import("./register-user")} */
+const registerUser = sync.require("./register-user")
 
 /**
  * @param {string} parentRoomID
@@ -15,10 +17,10 @@ const api = sync.require("../../matrix/api")
  * @param {import("discord-api-types/v10").APIThreadChannel} thread
  */
 async function announceThread(parentRoomID, threadRoomID, thread) {
-	const creatorMxid = select("sim", "mxid", {user_id: thread.owner_id}).pluck().get()
-
-   const content = await threadToAnnouncement.threadToAnnouncement(parentRoomID, threadRoomID, creatorMxid, thread, {api})
-
+	assert(thread.owner_id)
+	// @ts-ignore
+	const creatorMxid = await registerUser.ensureSimJoined({id: thread.owner_id}, parentRoomID)
+	const content = await threadToAnnouncement.threadToAnnouncement(parentRoomID, threadRoomID, creatorMxid, thread, {api})
 	await api.sendEvent(parentRoomID, "m.room.message", content, creatorMxid)
 }
 

d2m/actions/create-room.js

@@ -12,6 +12,8 @@ const file = sync.require("../../matrix/file")
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/kstate")} */
 const ks = sync.require("../../matrix/kstate")
+/** @type {import("../../discord/utils")} */
+const utils = sync.require("../../discord/utils")
 /** @type {import("./create-space")}) */
 const createSpace = sync.require("./create-space") // watch out for the require loop
 
@@ -57,13 +59,16 @@ function applyKStateDiffToRoom(roomID, kstate) {
 }
 
 /**
- * @param {{id: string, name: string, topic?: string?, type: number}} channel
+ * @param {{id: string, name: string, topic?: string?, type: number, parent_id?: string?}} channel
  * @param {{id: string}} guild
  * @param {string | null | undefined} customName
  */
 function convertNameAndTopic(channel, guild, customName) {
+	// @ts-ignore
+	const parentChannel = discord.channels.get(channel.parent_id)
 	let channelPrefix =
-		( channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] "
+		( parentChannel?.type === DiscordTypes.ChannelType.GuildForum ? ""
+		: channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] "
 		: channel.type === DiscordTypes.ChannelType.PrivateThread ? "[🔒⛓️] "
 		: channel.type === DiscordTypes.ChannelType.GuildVoice ? "[🔊] "
 		: "")
@@ -86,9 +91,24 @@ function convertNameAndTopic(channel, guild, customName) {
  * @param {DiscordTypes.APIGuild} guild
  */
 async function channelToKState(channel, guild) {
-	const spaceID = await createSpace.ensureSpace(guild)
-	assert(typeof spaceID === "string")
-	const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get()
+	// @ts-ignore
+	const parentChannel = discord.channels.get(channel.parent_id)
+	/** Used for membership/permission checks. */
+	let guildSpaceID
+	/** Used as the literal parent on Matrix, for categorisation. Will be the same as `guildSpaceID` unless it's a forum channel's thread, in which case a different space is used to group those threads. */
+	let parentSpaceID
+	let privacyLevel
+	if (parentChannel?.type === DiscordTypes.ChannelType.GuildForum) { // it's a forum channel's thread, so use a different space to group those threads
+		guildSpaceID = await createSpace.ensureSpace(guild)
+		parentSpaceID = await ensureRoom(channel.parent_id)
+		privacyLevel = select("guild_space", "privacy_level", {space_id: guildSpaceID}).pluck().get()
+	} else { // otherwise use the guild's space like usual
+		parentSpaceID = await createSpace.ensureSpace(guild)
+		guildSpaceID = parentSpaceID
+		privacyLevel = select("guild_space", "privacy_level", {space_id: parentSpaceID}).pluck().get()
+	}
+	assert(typeof parentSpaceID === "string")
+	assert(typeof guildSpaceID === "string")
 	assert(typeof privacyLevel === "number")
 
 	const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get()
@@ -112,20 +132,23 @@ async function channelToKState(channel, guild) {
 		join_rule: "restricted",
 		allow: [{
 			type: "m.room_membership",
-			room_id: spaceID
+			room_id: guildSpaceID
 		}]
 	}
 	if (PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel] !== "restricted") {
 		join_rules = {join_rule: PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel]}
 	}
 
+	const everyonePermissions = utils.getPermissions([], guild.roles, undefined, channel.permission_overwrites)
+	const everyoneCanMentionEveryone = utils.hasAllPermissions(everyonePermissions, ["MentionEveryone"])
+
 	const channelKState = {
 		"m.room.name/": {name: convertedName},
 		"m.room.topic/": {topic: convertedTopic},
 		"m.room.avatar/": avatarEventContent,
 		"m.room.guest_access/": {guest_access: PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
 		"m.room.history_visibility/": {history_visibility},
-		[`m.space.parent/${spaceID}`]: {
+		[`m.space.parent/${parentSpaceID}`]: {
 			via: [reg.ooye.server_name],
 			canonical: true
 		},
@@ -135,6 +158,9 @@ async function channelToKState(channel, guild) {
 			events: {
 				"m.room.avatar": 0
 			},
+			notifications: {
+				room: everyoneCanMentionEveryone ? 0 : 20
+			},
 			users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})
 		},
 		"chat.schildi.hide_ui/read_receipts": {
@@ -159,7 +185,7 @@ async function channelToKState(channel, guild) {
 		}
 	}
 
-	return {spaceID, privacyLevel, channelKState}
+	return {spaceID: parentSpaceID, privacyLevel, channelKState}
 }
 
 /**
@@ -175,6 +201,9 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 	let threadParent = null
 	if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id
 
+	let spaceCreationContent = {}
+	if (channel.type === DiscordTypes.ChannelType.GuildForum) spaceCreationContent = {creation_content: {type: "m.space"}}
+
 	// Name and topic can be done earlier in room creation rather than in initial_state
 	// https://spec.matrix.org/latest/client-server-api/#creation
 	const name = kstate["m.room.name/"].name
@@ -191,7 +220,8 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 			preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
 			visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
 			invite: [],
-			initial_state: ks.kstateToState(kstate)
+			initial_state: ks.kstateToState(kstate),
+			...spaceCreationContent
 		})
 
 		db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent)

d2m/actions/create-room.test.js

@@ -1,5 +1,6 @@
 // @ts-check
 
+const mixin = require("mixin-deep")
 const {channelToKState, _convertNameAndTopic} = require("./create-room")
 const {kstateStripConditionals} = require("../../matrix/kstate")
 const {test} = require("supertape")
@@ -39,6 +40,16 @@ test("channel2room: invite-only privacy room", async t => {
 	)
 })
 
+test("channel2room: room where limited people can mention everyone", async t => {
+	const limitedGuild = mixin({}, testData.guild.general)
+	limitedGuild.roles[0].permissions = (BigInt(limitedGuild.roles[0].permissions) - 131072n).toString()
+	const limitedRoom = mixin({}, testData.room.general, {"m.room.power_levels/": {notifications: {room: 20}}})
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, limitedGuild).then(x => x.channelKState)),
+		limitedRoom
+	)
+})
+
 test("convertNameAndTopic: custom name and topic", t => {
 	t.deepEqual(
 		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, "hauntings"),

d2m/actions/create-space.js

@@ -1,8 +1,9 @@
 // @ts-check
 
 const assert = require("assert").strict
+const {isDeepStrictEqual} = require("util")
 const DiscordTypes = require("discord-api-types/v10")
-const deepEqual = require("deep-equal")
+const Ty = require("../../types")
 const reg = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
@@ -13,8 +14,8 @@ const api = sync.require("../../matrix/api")
 const file = sync.require("../../matrix/file")
 /** @type {import("./create-room")} */
 const createRoom = sync.require("./create-room")
-/** @type {import("../converters/expression")} */
-const expression = sync.require("../converters/expression")
+/** @type {import("./expression")} */
+const expression = sync.require("./expression")
 /** @type {import("../../matrix/kstate")} */
 const ks = sync.require("../../matrix/kstate")
 
@@ -181,9 +182,16 @@ async function syncSpaceFully(guildID) {
 	const spaceDiff = ks.diffKState(spaceKState, guildKState)
 	await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff)
 
-	const childRooms = ks.kstateToState(spaceKState).filter(({type, content}) => {
-		return type === "m.space.child" && "via" in content
-	}).map(({state_key}) => state_key)
+	/** @type {string[]} room IDs */
+	let childRooms = []
+	/** @type {string | undefined} */
+	let nextBatch = undefined
+	do {
+		/** @type {Ty.HierarchyPagination<Ty.R.Hierarchy>} */
+		const res = await api.getHierarchy(spaceID, {from: nextBatch})
+		childRooms.push(...res.rooms.map(room => room.room_id))
+		nextBatch = res.next_batch
+	} while (nextBatch)
 
 	for (const roomID of childRooms) {
 		const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
@@ -226,7 +234,7 @@ async function syncSpaceExpressions(data, checkBeforeSync) {
 				// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
 				existing = fn([])
 			}
-			if (deepEqual(existing, content, {strict: true})) return
+			if (isDeepStrictEqual(existing, content)) return
 		}
 		api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content)
 	}

d2m/actions/delete-message.js

@@ -4,21 +4,25 @@ const passthrough = require("../../passthrough")
 const {sync, db, select, from} = passthrough
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
+/** @type {import("./speedbump")} */
+const speedbump = sync.require("./speedbump")
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data
  */
 async function deleteMessage(data) {
-	const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
-	if (!roomID) return
+	const row = select("channel_room", ["room_id", "speedbump_checked", "thread_parent"], {channel_id: data.channel_id}).get()
+	if (!row) return
 
 	const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all()
 	db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id)
 	db.prepare("DELETE FROM event_message WHERE message_id = ?").run(data.id)
 	for (const eventID of eventsToRedact) {
 		// Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs
-		await api.redactEvent(roomID, eventID)
+		await api.redactEvent(row.room_id, eventID)
 	}
+
+	await speedbump.updateCache(row.thread_parent || data.channel_id, row.speedbump_checked)
 }
 
 /**

d2m/actions/edit-message.js

@@ -1,18 +1,32 @@
 // @ts-check
 
+const assert = require("assert").strict
+
 const passthrough = require("../../passthrough")
 const {sync, db, select} = passthrough
 /** @type {import("../converters/edit-to-changes")} */
 const editToChanges = sync.require("../converters/edit-to-changes")
+/** @type {import("./register-pk-user")} */
+const registerPkUser = sync.require("./register-pk-user")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
  * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
-async function editMessage(message, guild) {
-	const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api)
+async function editMessage(message, guild, row) {
+	let {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api)
+
+	if (row && row.speedbump_webhook_id === message.webhook_id) {
+		// Handle the PluralKit public instance
+		if (row.speedbump_id === "466378653216014359") {
+			const root = await registerPkUser.fetchMessage(message.id)
+			assert(root.member)
+			senderMxid = await registerPkUser.ensureSimJoined(root, roomID)
+		}
+	}
 
 	// 1. Replace all the things.
 	for (const {oldID, newContent} of eventsToReplace) {

d2m/actions/expression.js

@@ -1,10 +1,9 @@
 // @ts-check
 
-const assert = require("assert").strict
 const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select} = passthrough
+const {sync, db} = passthrough
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
 

d2m/actions/lottie.js

@@ -0,0 +1,53 @@
+// @ts-check
+
+const DiscordTypes = require("discord-api-types/v10")
+const Ty = require("../../types")
+const assert = require("assert").strict
+
+const passthrough = require("../../passthrough")
+const {sync, db, select} = passthrough
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("../../matrix/mreq")} */
+const mreq = sync.require("../../matrix/mreq")
+/** @type {import("../converters/lottie")} */
+const convertLottie = sync.require("../converters/lottie")
+
+const INFO = {
+	mimetype: "image/png",
+	w: convertLottie.SIZE,
+	h: convertLottie.SIZE
+}
+
+/**
+ * @param {DiscordTypes.APIStickerItem} stickerItem
+ * @returns {Promise<{mxc_url: string, info: typeof INFO}>}
+ */
+async function convert(stickerItem) {
+	// Reuse sticker if already converted and uploaded
+	const existingMxc = select("lottie", "mxc_url", {sticker_id: stickerItem.id}).pluck().get()
+	if (existingMxc) return {mxc_url: existingMxc, info: INFO}
+
+	// Fetch sticker data from Discord
+	const res = await fetch(file.DISCORD_IMAGES_BASE + file.sticker(stickerItem))
+	if (res.status !== 200) throw new Error("Sticker data file not found.")
+	const text = await res.text()
+
+	// Convert to PNG (readable stream)
+	const readablePng = await convertLottie.convert(text)
+
+	// Upload to MXC
+	/** @type {Ty.R.FileUploaded} */
+	const root = await mreq.mreq("POST", "/media/v3/upload", readablePng, {
+		headers: {
+			"Content-Type": INFO.mimetype
+		}
+	})
+	assert(root.content_uri)
+
+	// Save the link for next time
+	db.prepare("INSERT INTO lottie (sticker_id, mxc_url) VALUES (?, ?)").run(stickerItem.id, root.content_uri)
+	return {mxc_url: root.content_uri, info: INFO}
+}
+
+module.exports.convert = convert

d2m/actions/register-pk-user.js

@@ -0,0 +1,153 @@
+// @ts-check
+
+const assert = require("assert")
+const reg = require("../../matrix/read-registration")
+const Ty = require("../../types")
+const fetch = require("node-fetch").default
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select} = passthrough
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("./register-user")} */
+const registerUser = sync.require("./register-user")
+
+/**
+ * @typedef WebhookAuthor Discord API message->author. A webhook as an author.
+ * @prop {string} username
+ * @prop {string?} avatar
+ * @prop {string} id
+ */
+
+/**
+ * A sim is an account that is being simulated by the bridge to copy events from the other side.
+ * @param {Ty.PkMessage} pkMessage
+ * @returns mxid
+ */
+async function createSim(pkMessage) {
+	// Choose sim name
+	const simName = "_pk_" + pkMessage.member.id
+	const localpart = reg.ooye.namespace_prefix + simName
+	const mxid = `@${localpart}:${reg.ooye.server_name}`
+
+	// Save chosen name in the database forever
+	db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, localpart, mxid)
+
+	// Register matrix user with that name
+	try {
+		await api.register(localpart)
+	} catch (e) {
+		// If user creation fails, manually undo the database change. Still isn't perfect, but should help.
+		// (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.)
+		db.prepare("DELETE FROM sim WHERE user_id = ?").run(pkMessage.member.uuid)
+		throw e
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user.
+ * If there is already a sim, use that one. If there isn't one yet, register a new sim.
+ * @param {Ty.PkMessage} pkMessage
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSim(pkMessage) {
+	let mxid = null
+	const existing = select("sim", "mxid", {user_id: pkMessage.member.uuid}).pluck().get()
+	if (existing) {
+		mxid = existing
+	} else {
+		mxid = await createSim(pkMessage)
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user and is joined to the room.
+ * @param {Ty.PkMessage} pkMessage
+ * @param {string} roomID
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSimJoined(pkMessage, roomID) {
+	// Ensure room ID is really an ID, not an alias
+	assert.ok(roomID[0] === "!")
+
+	// Ensure user
+	const mxid = await ensureSim(pkMessage)
+
+	// Ensure joined
+	const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get()
+	if (!existing) {
+		try {
+			await api.inviteToRoom(roomID, mxid)
+			await api.joinRoom(roomID, mxid)
+		} catch (e) {
+			if (e.message.includes("is already in the room.")) {
+				// Sweet!
+			} else {
+				throw e
+			}
+		}
+		db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid)
+	}
+	return mxid
+}
+
+/**
+ * @param {Ty.PkMessage} pkMessage
+ * @param {WebhookAuthor} author
+ */
+async function memberToStateContent(pkMessage, author) {
+	// We prefer to use the member's avatar URL data since the image upload can be cached across channels,
+	// unlike the userAvatar URL which is unique per channel, due to the webhook ID being in the URL.
+	const avatar = pkMessage.member.avatar_url || pkMessage.member.webhook_avatar_url || pkMessage.system.avatar_url || file.userAvatar(author)
+
+	const content = {
+		displayname: author.username,
+		membership: "join",
+		"moe.cadence.ooye.pk_member": pkMessage.member
+	}
+	if (avatar) content.avatar_url = await file.uploadDiscordFileToMxc(avatar)
+
+	return content
+}
+
+/**
+ * Sync profile data for a sim user. This function follows the following process:
+ * 1. Join the sim to the room if needed
+ * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
+ * 3. Compare against the previously known state content, which is helpfully stored in the database
+ * 4. If the state content has changed, send it to Matrix and update it in the database for next time
+ * @param {WebhookAuthor} author
+ * @param {Ty.PkMessage} pkMessage
+ * @param {string} roomID
+ * @returns {Promise<string>} mxid of the updated sim
+ */
+async function syncUser(author, pkMessage, roomID) {
+	const mxid = await ensureSimJoined(pkMessage, roomID)
+	// Update the sim_proxy table, so mentions can look up the original sender later
+	db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username)
+	// Sync the member state
+	const content = await memberToStateContent(pkMessage, author)
+	const currentHash = registerUser._hashProfileContent(content, 0)
+	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
+	// only do the actual sync if the hash has changed since we last looked
+	if (existingHash !== currentHash) {
+		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
+	}
+	return mxid
+}
+
+/** @returns {Promise<Ty.PkMessage>} */
+function fetchMessage(messageID) {
+	return fetch(`https://api.pluralkit.me/v2/messages/${messageID}`).then(res => res.json())
+}
+
+module.exports._memberToStateContent = memberToStateContent
+module.exports.ensureSim = ensureSim
+module.exports.ensureSimJoined = ensureSimJoined
+module.exports.syncUser = syncUser
+module.exports.fetchMessage = fetchMessage

d2m/actions/register-user.js

@@ -1,7 +1,9 @@
 // @ts-check
 
-const assert = require("assert")
+const assert = require("assert").strict
 const reg = require("../../matrix/read-registration")
+const DiscordTypes = require("discord-api-types/v10")
+const mixin = require("mixin-deep")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
@@ -9,6 +11,8 @@ const {discord, sync, db, select} = passthrough
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
+/** @type {import("../../discord/utils")} */
+const utils = sync.require("../../discord/utils")
 /** @type {import("../converters/user-to-mxid")} */
 const userToMxid = sync.require("../converters/user-to-mxid")
 /** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
@@ -18,7 +22,7 @@ require("xxhash-wasm")().then(h => hasher = h)
 
 /**
  * A sim is an account that is being simulated by the bridge to copy events from the other side.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @returns mxid
  */
 async function createSim(user) {
@@ -46,7 +50,7 @@ async function createSim(user) {
 /**
  * Ensure a sim is registered for the user.
  * If there is already a sim, use that one. If there isn't one yet, register a new sim.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @returns {Promise<string>} mxid
  */
 async function ensureSim(user) {
@@ -62,7 +66,7 @@ async function ensureSim(user) {
 
 /**
  * Ensure a sim is registered for the user and is joined to the room.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @param {string} roomID
  * @returns {Promise<string>} mxid
  */
@@ -92,8 +96,8 @@ async function ensureSimJoined(user, roomID) {
 }
 
 /**
- * @param {import("discord-api-types/v10").APIUser} user
- * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
  */
 async function memberToStateContent(user, member, guildID) {
 	let displayname = user.username
@@ -123,8 +127,46 @@ async function memberToStateContent(user, member, guildID) {
 	return content
 }
 
-function hashProfileContent(content) {
-	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
+/**
+ * https://gitdab.com/cadence/out-of-your-element/issues/9
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @returns {number} 0 to 100
+ */
+function memberToPowerLevel(user, member, guild, channel) {
+	const permissions = utils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites)
+	/*
+	 * PL 100 = Administrator = People who can brick the room. RATIONALE:
+	 * 	- Administrator.
+	 * 	- Manage Webhooks: People who remove the webhook can break the room.
+	 * 	- Manage Guild: People who can manage guild can add bots.
+	 * 	- Manage Channels: People who can manage the channel can delete it.
+	 * (Setting sim users to PL 100 is safe because even though we can't demote the sims we can use code to make the sims demote themselves.)
+	 */
+	if (guild.owner_id === user.id || utils.hasSomePermissions(permissions, ["Administrator", "ManageWebhooks", "ManageGuild", "ManageChannels"])) return 100
+	/*
+	 * PL 50 = Moderator = People who can manage people and messages in many ways. RATIONALE:
+	 * 	- Manage Messages: Can moderate by pinning or deleting the conversation.
+	 * 	- Manage Nicknames: Can moderate by removing inappropriate nicknames.
+	 * 	- Manage Threads: Can moderate by deleting conversations.
+	 * 	- Kick Members & Ban Members: Can moderate by removing disruptive people.
+	 * 	- Mute Members & Deafen Members: Can moderate by silencing disruptive people in ways they can't undo.
+	 * 	- Moderate Members.
+	 */
+	if (utils.hasSomePermissions(permissions, ["ManageMessages", "ManageNicknames", "ManageThreads", "KickMembers", "BanMembers", "MuteMembers", "DeafenMembers", "ModerateMembers"])) return 50
+	/* PL 20 = Mention Everyone for technical reasons. */
+	if (utils.hasSomePermissions(permissions, ["MentionEveryone"])) return 20
+	return 0
+}
+
+/**
+ * @param {any} content
+ * @param {number} powerLevel
+ */
+function _hashProfileContent(content, powerLevel) {
+	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}\u0000${powerLevel}`)
 	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
 	return signedHash
 }
@@ -133,52 +175,70 @@ function hashProfileContent(content) {
  * Sync profile data for a sim user. This function follows the following process:
  * 1. Join the sim to the room if needed
  * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
- * 3. Compare against the previously known state content, which is helpfully stored in the database
- * 4. If the state content has changed, send it to Matrix and update it in the database for next time
- * @param {import("discord-api-types/v10").APIUser} user
- * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ * 3. Calculate the power level the user should get based on their Discord permissions
+ * 4. Compare against the previously known state content, which is helpfully stored in the database
+ * 5. If the state content or power level have changed, send them to Matrix and update them in the database for next time
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {string} roomID
  * @returns {Promise<string>} mxid of the updated sim
  */
-async function syncUser(user, member, guildID, roomID) {
+async function syncUser(user, member, channel, guild, roomID) {
 	const mxid = await ensureSimJoined(user, roomID)
-	const content = await memberToStateContent(user, member, guildID)
-	const currentHash = hashProfileContent(content)
+	const content = await memberToStateContent(user, member, guild.id)
+	const powerLevel = memberToPowerLevel(user, member, guild, channel)
+	const currentHash = _hashProfileContent(content, powerLevel)
 	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
 	// only do the actual sync if the hash has changed since we last looked
 	if (existingHash !== currentHash) {
+		// Update room member state
 		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+		// Update power levels
+		const powerLevelsStateContent = await api.getStateEvent(roomID, "m.room.power_levels", "")
+		mixin(powerLevelsStateContent, {users: {[mxid]: powerLevel}})
+		if (powerLevel === 0) delete powerLevelsStateContent.users[mxid] // keep the event compact
+		await api.sendState(roomID, "m.room.power_levels", "", powerLevelsStateContent)
+		// Update cached hash
 		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
 	}
 	return mxid
 }
 
+/**
+ * @param {string} roomID
+ */
 async function syncAllUsersInRoom(roomID) {
 	const mxids = select("sim_member", "mxid", {room_id: roomID}).pluck().all()
 
 	const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
 	assert.ok(typeof channelID === "string")
 
-	/** @ts-ignore @type {import("discord-api-types/v10").APIGuildChannel} */
+	/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
 	const channel = discord.channels.get(channelID)
 	const guildID = channel.guild_id
 	assert.ok(typeof guildID === "string")
+	/** @ts-ignore @type {DiscordTypes.APIGuild} */
+	const guild = discord.guilds.get(guildID)
 
 	for (const mxid of mxids) {
 		const userID = select("sim", "user_id", {mxid}).pluck().get()
 		assert.ok(typeof userID === "string")
 
-		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIGuildMember>} */
+		/** @ts-ignore @type {Required<DiscordTypes.APIGuildMember>} */
 		const member = await discord.snow.guild.getGuildMember(guildID, userID)
-		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIUser>} user */
+		/** @ts-ignore @type {Required<DiscordTypes.APIUser>} user */
 		const user = member.user
 		assert.ok(user)
 
 		console.log(`[user sync] to matrix: ${user.username} in ${channel.name}`)
-		await syncUser(user, member, guildID, roomID)
+		await syncUser(user, member, channel, guild, roomID)
 	}
 }
 
 module.exports._memberToStateContent = memberToStateContent
+module.exports._hashProfileContent = _hashProfileContent
 module.exports.ensureSim = ensureSim
 module.exports.ensureSimJoined = ensureSimJoined
 module.exports.syncUser = syncUser

d2m/actions/send-message.js

@@ -1,6 +1,7 @@
 // @ts-check
 
-const assert = require("assert")
+const assert = require("assert").strict
+const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
 const { discord, sync, db } = passthrough
@@ -10,25 +11,44 @@ const messageToEvent = sync.require("../converters/message-to-event")
 const api = sync.require("../../matrix/api")
 /** @type {import("./register-user")} */
 const registerUser = sync.require("./register-user")
+/** @type {import("./register-pk-user")} */
+const registerPkUser = sync.require("./register-pk-user")
 /** @type {import("../actions/create-room")} */
 const createRoom = sync.require("../actions/create-room")
 /** @type {import("../../discord/utils")} */
 const dUtils = sync.require("../../discord/utils")
 
 /**
- * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
- * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
-async function sendMessage(message, guild) {
+async function sendMessage(message, channel, guild, row) {
 	const roomID = await createRoom.ensureRoom(message.channel_id)
 
 	let senderMxid = null
 	if (!dUtils.isWebhookMessage(message)) {
-		if (message.member) { // available on a gateway message create event
-			senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID)
+		if (message.author.id === discord.application.id) {
+			// no need to sync the bot's own user
+		} else if (message.member) { // available on a gateway message create event
+			senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID)
 		} else { // well, good enough...
 			senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
 		}
+	} else if (row && row.speedbump_webhook_id === message.webhook_id) {
+		// Handle the PluralKit public instance
+		if (row.speedbump_id === "466378653216014359") {
+			const root = await registerPkUser.fetchMessage(message.id)
+			// Member is null if member was deleted. We just got this message, so member surely exists.
+			if (!root.member) {
+				const e = new Error("PK API did not return a member")
+				message["__pk_response__"] = root
+				console.error(root)
+				throw e
+			}
+			senderMxid = await registerPkUser.syncUser(message.author, root, roomID)
+		}
 	}
 
 	const events = await messageToEvent.messageToEvent(message, guild, {}, {api})

d2m/actions/speedbump.js

@@ -0,0 +1,68 @@
+// @ts-check
+
+const DiscordTypes = require("discord-api-types/v10")
+const passthrough = require("../../passthrough")
+const {discord, select, db} = passthrough
+
+const SPEEDBUMP_SPEED = 4000 // 4 seconds delay
+const SPEEDBUMP_UPDATE_FREQUENCY = 2 * 60 * 60 // 2 hours
+
+/** @type {Set<any>} */
+const KNOWN_BOTS = new Set([
+	"466378653216014359" // PluralKit
+])
+
+/**
+ * Fetch new speedbump data for the channel and put it in the database as cache
+ * @param {string} channelID
+ * @param {number?} lastChecked
+ */
+async function updateCache(channelID, lastChecked) {
+	const now = Math.floor(Date.now() / 1000)
+	if (lastChecked && now - lastChecked < SPEEDBUMP_UPDATE_FREQUENCY) return
+	const webhooks = await discord.snow.webhook.getChannelWebhooks(channelID)
+	const found = webhooks.find(b => KNOWN_BOTS.has(b.application_id))
+	const foundApplication = found?.application_id
+	const foundWebhook = found?.id
+	db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_webhook_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(foundApplication, foundWebhook, now, channelID)
+}
+
+/** @type {Set<string>} set of messageID */
+const bumping = new Set()
+
+/**
+ * Slow down a message. After it passes the speedbump, return whether it's okay or if it's been deleted.
+ * @param {string} messageID
+ * @returns whether it was deleted
+ */
+async function doSpeedbump(messageID) {
+	bumping.add(messageID)
+	await new Promise(resolve => setTimeout(resolve, SPEEDBUMP_SPEED))
+	return !bumping.delete(messageID)
+}
+
+/**
+ * Check whether to slow down a message, and do it. After it passes the speedbump, return whether it's okay or if it's been deleted.
+ * @param {string} channelID
+ * @param {string} messageID
+ * @returns whether it was deleted, and data about the channel's (not thread's) speedbump
+ */
+async function maybeDoSpeedbump(channelID, messageID) {
+	let row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: channelID}).get()
+	if (row?.thread_parent) row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: row.thread_parent}).get() // webhooks belong to the channel, not the thread
+	if (!row?.speedbump_webhook_id) return {affected: false, row: null} // not affected, no speedbump
+	const affected = await doSpeedbump(messageID)
+	return {affected, row} // maybe affected, and there is a speedbump
+}
+
+/**
+ * @param {string} messageID
+ */
+function onMessageDelete(messageID) {
+	bumping.delete(messageID)
+}
+
+module.exports.updateCache = updateCache
+module.exports.doSpeedbump = doSpeedbump
+module.exports.maybeDoSpeedbump = maybeDoSpeedbump
+module.exports.onMessageDelete = onMessageDelete

d2m/converters/edit-to-changes.js

@@ -3,13 +3,24 @@
 const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select, from} = passthrough
+const {sync, select, from} = passthrough
 /** @type {import("./message-to-event")} */
 const messageToEvent = sync.require("../converters/message-to-event")
-/** @type {import("../actions/register-user")} */
-const registerUser = sync.require("../actions/register-user")
-/** @type {import("../actions/create-room")} */
-const createRoom = sync.require("../actions/create-room")
+/** @type {import("../../m2d/converters/utils")} */
+const utils = sync.require("../../m2d/converters/utils")
+
+function eventCanBeEdited(ev) {
+	// Discord does not allow files, images, attachments, or videos to be edited.
+	if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
+		return false
+	}
+	// Discord does not allow stickers to be edited.
+	if (ev.old.event_type === "m.sticker") {
+		return false
+	}
+	// Anything else is fair game.
+	return true
+}
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
@@ -19,12 +30,27 @@ const createRoom = sync.require("../actions/create-room")
  * @param {import("../../matrix/api")} api simple-as-nails dependency injection for the matrix API
  */
 async function editToChanges(message, guild, api) {
+	// If it is a user edit, allow deleting old messages (e.g. they might have removed text from an image). If it is the system adding a generated embed to a message, don't delete old messages since the system only sends partial data.
+
+	const isGeneratedEmbed = !("content" in message)
+
 	// Figure out what events we will be replacing
 
 	const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
 	assert(roomID)
 	/** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */
-	const senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
+	let senderMxid = null
+	if (message.author) {
+		senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
+	} else {
+		// Should be a system generated embed. We want the embed to be sent by the same user who sent the message, so that the messages get grouped in most clients.
+		const eventID = select("event_message", "event_id", {message_id: message.id}).pluck().get()
+		assert(eventID) // this should have been checked earlier in a calling function
+		const event = await api.getEvent(roomID, eventID)
+		if (utils.eventSenderIsFromDiscord(event.sender)) {
+			senderMxid = event.sender
+		}
+	}
 
 	const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all()
 
@@ -48,7 +74,8 @@ async function editToChanges(message, guild, api) {
 	let eventsToRedact = []
 	/** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */
 	let eventsToSend = []
-	//  4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. This is represented as nothing.
+	/**  4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. */
+	let unchangedEvents = []
 
 	function shift() {
 		newFallbackContent.shift()
@@ -81,19 +108,36 @@ async function editToChanges(message, guild, api) {
 		shift()
 	}
 	// Anything remaining in oldEventRows is present in the old version only and should be redacted.
-	eventsToRedact = oldEventRows
+	eventsToRedact = oldEventRows.map(e => ({old: e}))
+
+	// If this is a generated embed update, only allow the embeds to be updated, since the system only sends data about events. Ignore changes to other things.
+	if (isGeneratedEmbed) {
+		unchangedEvents.push(...eventsToRedact.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToRedact to unchangedEvents.
+		eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice")
+	}
+
+	// Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed!
+	// (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.)
+	// So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed.
+	unchangedEvents.push(...eventsToReplace.filter(ev => !eventCanBeEdited(ev))) // Move them from eventsToRedact to unchangedEvents.
+	eventsToReplace = eventsToReplace.filter(eventCanBeEdited)
 
 	// We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times.
 	/** @type {({column: string, eventID: string} | {column: string, nextEvent: true})[]} */
 	const promotions = []
 	for (const column of ["part", "reaction_part"]) {
+		const candidatesForParts = unchangedEvents.concat(eventsToReplace)
 		// If no events with part = 0 exist (or will exist), we need to do some management.
-		if (!eventsToReplace.some(e => e.old[column] === 0)) {
-			if (eventsToReplace.length) {
+		if (!candidatesForParts.some(e => e.old[column] === 0)) {
+			if (candidatesForParts.length) {
 				// We can choose an existing event to promote. Bigger order is better.
-				const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text")
-				eventsToReplace.sort((a, b) => order(b) - order(a))
-				promotions.push({column, eventID: eventsToReplace[0].old.event_id})
+				const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.old.event_subtype === "m.text")
+				candidatesForParts.sort((a, b) => order(b) - order(a))
+				if (column === "part") {
+					promotions.push({column, eventID: candidatesForParts[0].old.event_id}) // part should be the first one
+				} else {
+					promotions.push({column, eventID: candidatesForParts[candidatesForParts.length - 1].old.event_id}) // reaction_part should be the last one
+				}
 			} else {
 				// No existing events to promote, but new events are being sent. Whatever gets sent will be the next part = 0.
 				promotions.push({column, nextEvent: true})
@@ -101,24 +145,8 @@ async function editToChanges(message, guild, api) {
 		}
 	}
 
-	// Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed!
-	// (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.)
-	// So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed.
-	eventsToReplace = eventsToReplace.filter(ev => {
-		// Discord does not allow files, images, attachments, or videos to be edited.
-		if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
-			return false
-		}
-		// Discord does not allow stickers to be edited.
-		if (ev.old.event_type === "m.sticker") {
-			return false
-		}
-		// Anything else is fair game.
-		return true
-	})
-
 	// Removing unnecessary properties before returning
-	eventsToRedact = eventsToRedact.map(e => e.event_id)
+	eventsToRedact = eventsToRedact.map(e => e.old.event_id)
 	eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)}))
 
 	return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions}

d2m/converters/edit-to-changes.test.js

@@ -175,3 +175,98 @@ test("edit2changes: edit of reply to skull webp attachment with content", async
 		}
 	}])
 })
+
+test("edit2changes: edits the text event when multiple rows have part = 0 (should never happen in real life, but make sure the safety net works)", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_0, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* only the content can be edited",
+			"m.mentions": {},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "only the content can be edited",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999"
+			}
+		}
+	}])
+})
+
+test("edit2changes: promotes the text event when multiple rows have part = 1 (should never happen in real life, but make sure the safety net works)", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_1, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* only the content can be edited",
+			"m.mentions": {},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "only the content can be edited",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111"
+			}
+		}
+	}])
+	t.deepEqual(promotions, [
+		{
+			column: "part",
+			eventID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111"
+		},
+		{
+			column: "reaction_part",
+			eventID: "$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd111"
+		}
+	])
+})
+
+test("edit2changes: generated embed", async t => {
+	let called = 0
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.embed_generated_social_media_image, data.guild.general, {
+		async getEvent(roomID, eventID) {
+			called++
+			t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+			t.equal(eventID, "$mPSzglkCu-6cZHbYro0RW2u5mHvbH9aXDjO5FCzosc0")
+			return {sender: "@_ooye_cadence:cadence.moe"}
+		}
+	})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToReplace, [])
+	t.deepEqual(eventsToSend, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via hthrflwrs on cohost"
+			+ "\n| \n| ## This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO https://cohost.org/jkap/post/4794219-empty"
+			+ "\n| \n| 1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:"
+			+ "\n| \n|  * Both players draw eight cards"
+			+ "\n|  * Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand"
+			+ "\n|  * Both players present their best five-or-less-card pok...",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub>hthrflwrs on cohost</sub>`
+			+ `</p><p><strong><a href="https://cohost.org/jkap/post/4794219-empty">This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO</a></strong>`
+			+ `</p><p>1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:`
+			+ `<br><br><ul><li>Both players draw eight cards`
+			+ `</li><li>Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand`
+			+ `</li><li>Both players present their best five-or-less-card pok...</li></ul></p></blockquote>`,
+		"m.mentions": {}
+	}])
+	t.deepEqual(promotions, []) // TODO: it would be ideal to promote this to reaction_part = 0. this is OK to do because the main message won't have had any reactions yet.
+	t.equal(senderMxid, "@_ooye_cadence:cadence.moe")
+	t.equal(called, 1)
+})

d2m/converters/lottie.js

@@ -1,25 +1,11 @@
 // @ts-check
 
-const DiscordTypes = require("discord-api-types/v10")
-const Ty = require("../../types")
-const assert = require("assert").strict
+const assert = require("assert")
+const stream = require("stream")
 const {PNG} = require("pngjs")
 
-const passthrough = require("../../passthrough")
-const {sync, db, discord, select} = passthrough
-/** @type {import("../../matrix/file")} */
-const file = sync.require("../../matrix/file")
-//** @type {import("../../matrix/mreq")} */
-const mreq = sync.require("../../matrix/mreq")
-
 const SIZE = 160 // Discord's display size on 1x displays is 160
 
-const INFO = {
-	mimetype: "image/png",
-	w: SIZE,
-	h: SIZE
-}
-
 /**
  * @typedef RlottieWasm
  * @prop {(string) => boolean} load load lottie data from string of json
@@ -34,20 +20,15 @@ const Rlottie = (async () => {
 })()
 
 /**
- * @param {DiscordTypes.APIStickerItem} stickerItem
- * @returns {Promise<{mxc_url: string, info: typeof INFO}>}
+ * @param {string} text
+ * @returns {Promise<import("stream").Readable>}
  */
-async function convert(stickerItem) {
-	const existingMxc = select("lottie", "mxc_url", {sticker_id: stickerItem.id}).pluck().get()
-	if (existingMxc) return {mxc_url: existingMxc, info: INFO}
+async function convert(text) {
 	const r = await Rlottie
-	const res = await fetch(file.DISCORD_IMAGES_BASE + file.sticker(stickerItem))
-	if (res.status !== 200) throw new Error("Sticker data file not found.")
-	const text = await res.text()
 	/** @type RlottieWasm */
 	const rh = new r.RlottieWasm()
 	const status = rh.load(text)
-	if (!status) throw new Error(`Rlottie unable to load ${text.length} byte data file.`)
+	assert(status, `Rlottie unable to load ${text.length} byte data file.`)
 	const rendered = rh.render(0, SIZE, SIZE)
 	let png = new PNG({
 		width: SIZE,
@@ -58,17 +39,10 @@ async function convert(stickerItem) {
 		inputHasAlpha: true,
 	})
 	png.data = Buffer.from(rendered)
-	// @ts-ignore wrong type from pngjs
-	const readablePng = png.pack()
-	/** @type {Ty.R.FileUploaded} */
-	const root = await mreq.mreq("POST", "/media/v3/upload", readablePng, {
-		headers: {
-			"Content-Type": INFO.mimetype
-		}
-	})
-	assert(root.content_uri)
-	db.prepare("INSERT INTO lottie (sticker_id, mxc_url) VALUES (?, ?)").run(stickerItem.id, root.content_uri)
-	return {mxc_url: root.content_uri, info: INFO}
+	// png.pack() is a bad stream and will throw away any data it sends if it's not connected to a destination straight away.
+	// We use Duplex.from to convert it into a good stream.
+	return stream.Duplex.from(png.pack())
 }
 
 module.exports.convert = convert
+module.exports.SIZE = SIZE

d2m/converters/lottie.test.js

@@ -0,0 +1,34 @@
+// @ts-check
+
+const fs = require("fs")
+const stream = require("stream")
+const {test} = require("supertape")
+const {convert} = require("./lottie")
+
+const WRITE_PNG = false
+
+test("lottie: can convert and save PNG", async t => {
+	const input = await fs.promises.readFile("test/res/lottie-bee.json", "utf8")
+	const resultStream = await convert(input)
+	/* c8 ignore next 3 */
+	if (WRITE_PNG) {
+		resultStream.pipe(fs.createWriteStream("test/res/lottie-bee.png"))
+		t.fail("PNG written to /test/res/lottie-bee.png, please manually check it")
+	} else {
+		const expected = await fs.promises.readFile("test/res/lottie-bee.png")
+		const actual = Buffer.alloc(expected.length)
+		let i = 0
+		await stream.promises.pipeline(
+			resultStream,
+			async function* (source) {
+				for await (const chunk of source) {
+					chunk.copy(actual, i)
+					i += chunk.length
+				}
+			},
+			new stream.PassThrough()
+		)
+		t.equal(i, actual.length, `allocated ${actual.length} bytes, but wrote ${i}`)
+		t.deepEqual(actual, expected)
+	}
+})

d2m/converters/message-to-event.embeds.test.js

@@ -9,9 +9,9 @@ test("message2event embeds: nothing but a field", async t => {
 		$type: "m.room.message",
 		"m.mentions": {},
 		msgtype: "m.notice",
-		body: "> ### Amanda 🎵#2192 :online:"
-			+ "\n> willow tree, branch 0"
-			+ "\n> **❯ Uptime:**\n> 3m 55s\n> **❯ Memory:**\n> 64.45MB",
+		body: "| ### Amanda 🎵#2192 :online:"
+			+ "\n| willow tree, branch 0"
+			+ "\n| **❯ Uptime:**\n| 3m 55s\n| **❯ Memory:**\n| 64.45MB",
 		format: "org.matrix.custom.html",
 		formatted_body: '<blockquote><p><strong>Amanda 🎵#2192 <img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ\" title=\":online:\" alt=\":online:\">'
 			+ '<br>willow tree, branch 0</strong>'
@@ -26,17 +26,15 @@ test("message2event embeds: reply with just an embed", async t => {
 		$type: "m.room.message",
 		msgtype: "m.notice",
 		"m.mentions": {},
-		body: "> ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
-			+ "\n> \n> ## https://twitter.com/i/status/1707484191963648161"
-			+ "\n> \n> does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
-			+ "\n> \n> ### Retweets"
-			+ "\n> 119"
-			+ "\n> \n> ### Likes"
-			+ "\n> 5581"
-			+ "\n> — Twitter",
+		body: "| ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
+			+ "\n| \n| does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
+			+ "\n| \n| ### Retweets"
+			+ "\n| 119"
+			+ "\n| \n| ### Likes"
+			+ "\n| 5581"
+			+ "\n| — Twitter",
 		format: "org.matrix.custom.html",
-		formatted_body: '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong></p>'
-			+ '<p><strong><a href="https://twitter.com/i/status/1707484191963648161">https://twitter.com/i/status/1707484191963648161</a></strong>'
+		formatted_body: '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong>'
 			+ '</p><p>does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
 			+ '</p><p><strong>Retweets</strong><br>119</p><p><strong>Likes</strong><br>5581</p>— Twitter</blockquote>'
 	}])
@@ -75,22 +73,48 @@ test("message2event embeds: image embed and attachment", async t => {
 })
 
 test("message2event embeds: blockquote in embed", async t => {
-	const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general)
+	let called = 0
+	const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:example.invalid": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
 	t.deepEqual(events, [{
 		$type: "m.room.message",
 		msgtype: "m.text",
 		body: ":emoji: **4 |** #wonderland",
 		format: "org.matrix.custom.html",
-		formatted_body: `<img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO\" title=\":emoji:\" alt=\":emoji:\"> <strong>4 |</strong> <a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe\">#wonderland</a>`,
+		formatted_body: `<img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO\" title=\":emoji:\" alt=\":emoji:\"> <strong>4 |</strong> <a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe?via=cadence.moe&via=example.invalid\">#wonderland</a>`,
 		"m.mentions": {}
 	}, {
 		$type: "m.room.message",
 		msgtype: "m.notice",
-		body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo)",
+		body: "| ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n| \n| reply draft\n| > The following is a message composed via consensus of the Stinker Council.\n| > \n| > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n| > \n| > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n| > \n| > There will be no further communication.\n| \n| [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)",
 		format: "org.matrix.custom.html",
-		formatted_body: "<blockquote><p><strong><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\">⏺️ minimus</a></strong></p><p>reply draft<br><blockquote>The following is a message composed via consensus of the Stinker Council.<br><br>For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.<br><br>Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.<br><br>There will be no further communication.</blockquote></p><p><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\">Go to Message</a></p></blockquote>",
+		formatted_body: "<blockquote><p><strong><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">⏺️ minimus</a></strong></p><p>reply draft<br><blockquote>The following is a message composed via consensus of the Stinker Council.<br><br>For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.<br><br>Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.<br><br>There will be no further communication.</blockquote></p><p><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&amp;via=example.invalid\">Go to Message</a></p></blockquote>",
 		"m.mentions": {}
 	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
 })
 
 test("message2event embeds: crazy html is all escaped", async t => {
@@ -98,12 +122,12 @@ test("message2event embeds: crazy html is all escaped", async t => {
 	t.deepEqual(events, [{
 		$type: "m.room.message",
 		msgtype: "m.notice",
-		body: "> ## ⏺️ <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
-			+ "\n> \n> ## <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
-			+ "\n> \n> <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
-			+ "\n> \n> ### <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
-			+ "\n> <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
-			+ "\n> — <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+		body: "| ## ⏺️ <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n| \n| ## <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n| \n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| \n| ### <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n| — <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
 		format: "org.matrix.custom.html",
 		formatted_body: `<blockquote>`
 			+ `<p><strong><a href="https://a.co/&amp;amp;&lt;script&gt;">⏺️ &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</a></strong></p>`
@@ -115,3 +139,180 @@ test("message2event embeds: crazy html is all escaped", async t => {
 		"m.mentions": {}
 	}])
 })
+
+test("message2event embeds: title without url", async t => {
+	const events = await messageToEvent(data.message_with_embeds.title_without_url, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## Hi, I'm Amanda!\n| \n| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><strong>Hi, I'm Amanda!</strong></p><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: url without title", async t => {
+	const events = await messageToEvent(data.message_with_embeds.url_without_title, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: author without url", async t => {
+	const events = await messageToEvent(data.message_with_embeds.author_without_url, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## Amanda\n| \n| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><strong>Amanda</strong></p><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: author url without name", async t => {
+	const events = await messageToEvent(data.message_with_embeds.author_url_without_name, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: vx image", async t => {
+	const events = await messageToEvent(data.message_with_embeds.vx_image, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://vxtwitter.com/TomorrowCorp/status/1760330671074287875 we got a release date!!!",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://vxtwitter.com/TomorrowCorp/status/1760330671074287875">https://vxtwitter.com/TomorrowCorp/status/1760330671074287875</a> we got a release date!!!',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via vxTwitter / fixvx https://github.com/dylanpdx/BetterTwitFix"
+			+ "\n| "
+			+ "\n| ## Twitter https://twitter.com/tomorrowcorp/status/1760330671074287875"
+			+ "\n| "
+			+ "\n| ## Tomorrow Corporation (@TomorrowCorp) https://vxtwitter.com/TomorrowCorp/status/1760330671074287875"
+			+ "\n| "
+			+ "\n| Mark your calendar with a wet black stain! World of Goo 2 releases on May 23, 2024 on Nintendo Switch, Epic Games Store (Win/Mac), and http://WorldOfGoo2.com (Win/Mac/Linux)."
+			+ "\n| "
+			+ "\n| https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms"
+			+ "\n| "
+			+ "\n| 💖 123 🔁 36"
+			+ "\n| "
+			+ "\n| 📸 https://pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://github.com/dylanpdx/BetterTwitFix">vxTwitter / fixvx</a></sub>`
+			+ `</p><p><strong><a href="https://twitter.com/tomorrowcorp/status/1760330671074287875">Twitter</a></strong>`
+			+ `</p><p><strong><a href="https://vxtwitter.com/TomorrowCorp/status/1760330671074287875">Tomorrow Corporation (@TomorrowCorp)</a></strong>`
+			+ `</p><p>Mark your calendar with a wet black stain! World of Goo 2 releases on May 23, 2024 on Nintendo Switch, Epic Games Store (Win/Mac), and <a href="http://WorldOfGoo2.com">http://WorldOfGoo2.com</a> (Win/Mac/Linux).`
+			+ `<br><br><a href="https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms">https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms</a>`
+			+ `<br><br>💖 123 🔁 36`
+			+ `</p><p>📸 https://pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: vx video", async t => {
+	const events = await messageToEvent(data.message_with_embeds.vx_video, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://vxtwitter.com/McDonalds/status/1759971752254341417",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://vxtwitter.com/McDonalds/status/1759971752254341417">https://vxtwitter.com/McDonalds/status/1759971752254341417</a>',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via vxTwitter / fixvx https://github.com/dylanpdx/BetterTwitFix"
+			+ "\n| \n| ## McDonald’s🤝@studiopierrot"
+			+ "\n| \n| 💖 89 🔁 21 https://twitter.com/McDonalds/status/1759971752254341417"
+			+ "\n| \n| ## McDonald's (@McDonalds) https://vxtwitter.com/McDonalds/status/1759971752254341417"
+			+ "\n| \n| 🎞️ https://video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4?tag=12",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://github.com/dylanpdx/BetterTwitFix">vxTwitter / fixvx</a></sub>`
+			+ `</p><p><strong><a href="https://twitter.com/McDonalds/status/1759971752254341417">McDonald’s🤝@studiopierrot\n\n💖 89 🔁 21</a></strong>`
+			+ `</p><p><strong><a href="https://vxtwitter.com/McDonalds/status/1759971752254341417">McDonald's (@McDonalds)</a></strong>`
+			+ `</p><p>🎞️ https://video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4?tag=12</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: youtube video", async t => {
+	const events = await messageToEvent(data.message_with_embeds.youtube_video, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E\n\n\nJutomi I'm gonna make these sounds in your walls tonight",
+		format: "org.matrix.custom.html",
+		formatted_body: `<a href="https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E">https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E</a><br><br><br>Jutomi I'm gonna make these sounds in your walls tonight`,
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via YouTube https://www.youtube.com"
+			+ "\n| \n| ## Happy O Funny https://www.youtube.com/channel/UCEpQ9aEb1NafpvWp5Aoizrg"
+      	+ "\n| \n| ## Shoebill stork clattering sounds like machine guun~!! (Japan Matsue... https://www.youtube.com/watch?v=kDMHHw8JqLE"
+			+ "\n| \n| twitter"
+			+ "\n| https://twitter.com/matsuevogelpark"
+			+ "\n| \n| The shoebill (Balaeniceps rex) also known as whalehead, whale-headed stork, or shoe-billed stork, is a very large stork-like bird. It derives its name from its enormous shoe-shaped bill"
+			+ "\n| some people also called them the living dinosaur~~"
+			+ "\n| \n| #shoebill #livingdinosaur #happyofunny #weirdcreature #weirdsoun..."
+			+ "\n| \n| 🎞️ https://www.youtube.com/embed/kDMHHw8JqLE",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://www.youtube.com">YouTube</a></sub></p>`
+			+ `<p><strong><a href="https://www.youtube.com/channel/UCEpQ9aEb1NafpvWp5Aoizrg">Happy O Funny</a></strong>`
+			+ `</p><p><strong><a href="https://www.youtube.com/watch?v=kDMHHw8JqLE">Shoebill stork clattering sounds like machine guun~!! (Japan Matsue...</a></strong>`
+			+ `</p><p>twitter<br><a href="https://twitter.com/matsuevogelpark">https://twitter.com/matsuevogelpark</a><br><br>The shoebill (Balaeniceps rex) also known as whalehead, whale-headed stork, or shoe-billed stork, is a very large stork-like bird. It derives its name from its enormous shoe-shaped bill<br>some people also called them the living dinosaur~~<br><br>#shoebill #livingdinosaur #happyofunny #weirdcreature #weirdsoun...`
+			+ `</p><p>🎞️ https://www.youtube.com/embed/kDMHHw8JqLE`
+			+ `</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: if discord creates an embed preview for a discord channel link, don't copy that embed", async t => {
+	const events = await messageToEvent(data.message_with_embeds.discord_server_included_punctuation_bad_discord, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "(test https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&via=matrix.org)",
+		format: "org.matrix.custom.html",
+		formatted_body: `(test <a href="https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&amp;via=matrix.org">https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&amp;via=matrix.org</a>)`,
+		"m.mentions": {}
+	}])
+})

d2m/converters/message-to-event.js

@@ -12,8 +12,8 @@ const {sync, db, discord, select, from} = passthrough
 const file = sync.require("../../matrix/file")
 /** @type {import("./emoji-to-key")} */
 const emojiToKey = sync.require("./emoji-to-key")
-/** @type {import("./lottie")} */
-const lottie = sync.require("./lottie")
+/** @type {import("../actions/lottie")} */
+const lottie = sync.require("../actions/lottie")
 /** @type {import("../../m2d/converters/utils")} */
 const mxUtils = sync.require("../../m2d/converters/utils")
 /** @type {import("../../discord/utils")} */
@@ -39,15 +39,19 @@ function getDiscordParseCallbacks(message, guild, useHTML) {
 				return `@${username}:`
 			}
 		},
-		/** @param {{id: string, type: "discordChannel"}} node */
+		/** @param {{id: string, type: "discordChannel", row: {room_id: string, name: string, nick: string?}?, via: string}} node */
 		channel: node => {
-			const row = select("channel_room", ["room_id", "name", "nick"], {channel_id: node.id}).get()
-			if (!row) {
-				return `<#${node.id}>` // fallback for when this channel is not bridged
+			if (!node.row) { // fallback for when this channel is not bridged
+				const channel = discord.channels.get(node.id)
+				if (channel) {
+					return `#${channel.name} [channel not bridged]`
+				} else {
+					return `#unknown-channel [channel from an unbridged server]`
+				}
 			} else if (useHTML) {
-				return `<a href="https://matrix.to/#/${row.room_id}">#${row.nick || row.name}</a>`
+				return `<a href="https://matrix.to/#/${node.row.room_id}?${node.via}">#${node.row.nick || node.row.name}</a>`
 			} else {
-				return `#${row.nick || row.name}`
+				return `#${node.row.nick || node.row.name}`
 			}
 		},
 		/** @param {{animated: boolean, name: string, id: string, type: "discordEmoji"}} node */
@@ -74,10 +78,14 @@ function getDiscordParseCallbacks(message, guild, useHTML) {
 				return `@${role.name}:`
 			}
 		},
-		everyone: node =>
-			"@room",
-		here: node =>
-			"@here"
+		everyone: () => {
+			if (message.mention_everyone) return "@room"
+			return "@everyone"
+		},
+		here: () => {
+			if (message.mention_everyone) return "@room"
+			return "@here"
+		}
 	}
 }
 
@@ -195,6 +203,7 @@ async function attachmentToEvent(mentions, attachment) {
 async function messageToEvent(message, guild, options = {}, di) {
 	const events = []
 
+	/* c8 ignore next 7 */
 	if (message.type === DiscordTypes.MessageType.ThreadCreated) {
 		// This is the kind of message that appears when somebody makes a thread which isn't close enough to the message it's based off.
 		// It lacks the lines and the pill, so it looks kind of like a member join message, and it says:
@@ -240,6 +249,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 	let repliedToEventRow = null
 	let repliedToEventSenderMxid = null
 
+	if (message.mention_everyone) mentions.room = true
+
 	function addMention(mxid) {
 		if (!mentions.user_ids) mentions.user_ids = []
 		if (!mentions.user_ids.includes(mxid)) mentions.user_ids.push(mxid)
@@ -252,6 +263,37 @@ async function messageToEvent(message, guild, options = {}, di) {
 		if (row) {
 			repliedToEventRow = row
 		}
+	} else if (dUtils.isWebhookMessage(message) && message.embeds[0]?.author?.name?.endsWith("↩️")) {
+		// It could be a PluralKit emulated reply, let's see if it has a message link
+		const isEmulatedReplyToText = message.embeds[0].description?.startsWith("**[Reply to:]")
+		const isEmulatedReplyToAttachment = message.embeds[0].description?.startsWith("*[(click to see attachment")
+		if (isEmulatedReplyToText || isEmulatedReplyToAttachment) {
+			assert(message.embeds[0].description)
+			const match = message.embeds[0].description.match(/\/channels\/[0-9]*\/[0-9]*\/([0-9]{2,})/)
+			if (match) {
+				const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1])
+				if (row) {
+					/*
+						we generate a partial referenced_message based on what PK provided. we don't need everything, since this will only be used for further message-to-event converting.
+						the following properties are necessary:
+						- content: used for generating the reply fallback
+						- author: used for the top of the reply fallback (only used for discord authors. for matrix authors, repliedToEventSenderMxid is set.)
+					*/
+					const emulatedMessageContent =
+						( isEmulatedReplyToAttachment ? "[Media]"
+						: message.embeds[0].description.replace(/^.*?\)\*\*\s*/, ""))
+					message.referenced_message = {
+						content: emulatedMessageContent,
+						// @ts-ignore
+						author: {
+							username: message.embeds[0].author.name.replace(/\s*↩️\s*$/, "")
+						}
+					}
+					message.embeds.shift()
+					repliedToEventRow = row
+				}
+			}
+		}
 	}
 	if (repliedToEventRow && repliedToEventRow.source === 0) { // reply was originally from Matrix
 		// Need to figure out who sent that event...
@@ -261,6 +303,20 @@ async function messageToEvent(message, guild, options = {}, di) {
 		addMention(repliedToEventSenderMxid)
 	}
 
+	/** @type {Map<string, Promise<string>>} */
+	const viaMemo = new Map()
+	/**
+	 * @param {string} roomID
+	 * @returns {Promise<string>} string encoded URLSearchParams
+	 */
+	function getViaServersMemo(roomID) {
+		// @ts-ignore
+		if (viaMemo.has(roomID)) return viaMemo.get(roomID)
+		const promise = mxUtils.getViaServersQuery(roomID, di.api).then(p => p.toString())
+		viaMemo.set(roomID, promise)
+		return promise
+	}
+
 	/**
 	 * Translate Discord message links to Matrix event links.
 	 * If OOYE has handled this message in the past, this is an instant database lookup.
@@ -277,12 +333,13 @@ async function messageToEvent(message, guild, options = {}, di) {
 			const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
 			if (roomID) {
 				const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get()
+				const via = await getViaServersMemo(roomID)
 				if (eventID && roomID) {
-					result = `https://matrix.to/#/${roomID}/${eventID}`
+					result = `https://matrix.to/#/${roomID}/${eventID}?${via}`
 				} else {
 					const ts = dUtils.snowflakeToTimestampExact(messageID)
 					const {event_id} = await di.api.getEventForTimestamp(roomID, ts)
-					result = `https://matrix.to/#/${roomID}/${event_id}`
+					result = `https://matrix.to/#/${roomID}/${event_id}?${via}`
 				}
 			} else {
 				result = `${match[0]} [event is from another server]`
@@ -314,12 +371,27 @@ async function messageToEvent(message, guild, options = {}, di) {
 			return emojiToKey.emojiToKey({id, name, animated}) // Register the custom emoji if needed
 		}))
 
-		let html = markdown.toHTML(content, {
+		async function transformParsedVia(parsed) {
+			for (const node of parsed) {
+				if (node.type === "discordChannel") {
+					node.row = select("channel_room", ["room_id", "name", "nick"], {channel_id: node.id}).get()
+					if (node.row?.room_id) {
+						node.via = await getViaServersMemo(node.row.room_id)
+					}
+				}
+				if (Array.isArray(node.content)) {
+					await transformParsedVia(node.content)
+				}
+			}
+			return parsed
+		}
+
+		let html = await markdown.toHtmlWithPostParser(content, transformParsedVia, {
 			discordCallback: getDiscordParseCallbacks(message, guild, true),
 			...customOptions
 		}, customParser, customHtmlOutput)
 
-		let body = markdown.toHTML(content, {
+		let body = await markdown.toHtmlWithPostParser(content, transformParsedVia, {
 			discordCallback: getDiscordParseCallbacks(message, guild, false),
 			discordOnly: true,
 			escapeHTML: false,
@@ -329,6 +401,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 		return {body, html}
 	}
 
+	// FIXME: What was the scanMentions parameter supposed to activate? It's unused.
 	async function addTextEvent(body, html, msgtype, {scanMentions}) {
 		// Star * prefix for fallback edits
 		if (options.includeEditFallbackStar) {
@@ -350,7 +423,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 			if (repliedToEventRow?.source === 0 && repliedToEventSenderMxid) {
 				const match = repliedToEventSenderMxid.match(/^@([^:]*)/)
 				assert(match)
-				repliedToDisplayName = match[1] || "a Matrix user" // grab the localpart as the display name, whatever
+				repliedToDisplayName = message.referenced_message?.author.username || match[1] || "a Matrix user" // grab the localpart as the display name, whatever
 				repliedToUserHtml = `<a href="https://matrix.to/#/${repliedToEventSenderMxid}">${repliedToDisplayName}</a>`
 			} else {
 				repliedToDisplayName = message.referenced_message?.author.global_name || message.referenced_message?.author.username || "a Discord user"
@@ -407,32 +480,35 @@ async function messageToEvent(message, guild, options = {}, di) {
 		message.content = "changed the channel name to **" + message.content + "**"
 	}
 
-	// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
-	const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
-	if (matches.length && matches.some(m => m[1].match(/[a-z]/i))) {
-		const writtenMentionsText = matches.map(m => m[1].toLowerCase())
-		const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
-		assert(roomID)
-		const {joined} = await di.api.getJoinedMembers(roomID)
-		for (const [mxid, member] of Object.entries(joined)) {
-			if (!userRegex.some(rx => mxid.match(rx))) {
-				const localpart = mxid.match(/@([^:]*)/)
-				assert(localpart)
-				const displayName = member.display_name || localpart[1]
-				if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+
+	if (message.content) {
+		// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
+		const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
+		if (matches.length && matches.some(m => m[1].match(/[a-z]/i) && m[1] !== "everyone" && m[1] !== "here")) {
+			const writtenMentionsText = matches.map(m => m[1].toLowerCase())
+			const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
+			assert(roomID)
+			const {joined} = await di.api.getJoinedMembers(roomID)
+			for (const [mxid, member] of Object.entries(joined)) {
+				if (!userRegex.some(rx => mxid.match(rx))) {
+					const localpart = mxid.match(/@([^:]*)/)
+					assert(localpart)
+					const displayName = member.display_name || localpart[1]
+					if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+				}
 			}
 		}
-	}
 
-	// Text content appears first
-	if (message.content) {
+		// Text content appears first
 		const {body, html} = await transformContent(message.content)
 		await addTextEvent(body, html, msgtype, {scanMentions: true})
 	}
 
 	// Then attachments
-	const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions)))
-	events.push(...attachmentEvents)
+	if (message.attachments) {
+		const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions)))
+		events.push(...attachmentEvents)
+	}
 
 	// Then embeds
 	for (const embed of message.embeds || []) {
@@ -440,13 +516,26 @@ async function messageToEvent(message, guild, options = {}, di) {
 			continue // Matrix's own URL previews are fine for images.
 		}
 
+		if (embed.url?.startsWith("https://discord.com/")) {
+			continue // If discord creates an embed preview for a discord channel link, don't copy that embed
+		}
+
 		// Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once
 		const rep = new mxUtils.MatrixStringBuilder()
 
+		// Provider
+		if (embed.provider?.name) {
+			if (embed.provider.url) {
+				rep.addParagraph(`via ${embed.provider.name} ${embed.provider.url}`, tag`<sub><a href="${embed.provider.url}">${embed.provider.name}</a></sub>`)
+			} else {
+				rep.addParagraph(`via ${embed.provider.name}`, tag`<sub>${embed.provider.name}</sub>`)
+			}
+		}
+
 		// Author and URL into a paragraph
 		let authorNameText = embed.author?.name || ""
 		if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image
-		if (authorNameText || embed.author?.url) {
+		if (authorNameText) {
 			if (embed.author?.url) {
 				const authorURL = await transformContentMessageLinks(embed.author.url)
 				rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`<strong><a href="${authorURL}">${authorNameText}</a></strong>`)
@@ -463,11 +552,11 @@ async function messageToEvent(message, guild, options = {}, di) {
 			} else {
 				rep.addParagraph(`## ${body}`, `<strong>${html}</strong>`)
 			}
-		} else if (embed.url) {
-			rep.addParagraph(`## ${embed.url}`, tag`<strong><a href="${embed.url}">${embed.url}</a></strong>`)
 		}
 
-		if (embed.description) {
+		let embedTypeShouldShowDescription = embed.type !== "video" // Discord doesn't display descriptions for videos
+		if (embed.provider?.name === "YouTube") embedTypeShouldShowDescription = true // But I personally like showing the descriptions for YouTube videos specifically
+		if (embed.description && embedTypeShouldShowDescription) {
 			const {body, html} = await transformContent(embed.description)
 			rep.addParagraph(body, html)
 		}
@@ -481,12 +570,16 @@ async function messageToEvent(message, guild, options = {}, di) {
 			rep.addParagraph(fieldRep.get().body, fieldRep.get().formatted_body)
 		}
 
-		if (embed.image?.url) rep.addParagraph(`📸 ${embed.image.url}`)
+		let chosenImage = embed.image?.url
+		// the thumbnail seems to be used for "article" type but displayed big at the bottom by discord
+		if (embed.type === "article" && embed.thumbnail?.url && !chosenImage) chosenImage = embed.thumbnail.url
+		if (chosenImage) rep.addParagraph(`📸 ${chosenImage}`)
+
 		if (embed.video?.url) rep.addParagraph(`🎞️ ${embed.video.url}`)
 
 		if (embed.footer?.text) rep.addLine(`— ${embed.footer.text}`, tag`— ${embed.footer.text}`)
 		let {body, formatted_body: html} = rep.get()
-		body = body.split("\n").map(l => "> " + l).join("\n")
+		body = body.split("\n").map(l => "| " + l).join("\n")
 		html = `<blockquote>${html}</blockquote>`
 
 		// Send as m.notice to apply the usual automated/subtle appearance, showing this wasn't actually typed by the person
@@ -497,25 +590,17 @@ async function messageToEvent(message, guild, options = {}, di) {
 	if (message.sticker_items) {
 		const stickerEvents = await Promise.all(message.sticker_items.map(async stickerItem => {
 			const format = file.stickerFormat.get(stickerItem.format_type)
+			assert(format?.mime)
 			if (format?.mime === "lottie") {
-				try {
-					const {mxc_url, info} = await lottie.convert(stickerItem)
-					return {
-						$type: "m.sticker",
-						"m.mentions": mentions,
-						body: stickerItem.name,
-						info,
-						url: mxc_url
-					}
-				} catch (e) {
-					return {
-						$type: "m.room.message",
-						"m.mentions": mentions,
-						msgtype: "m.notice",
-						body: `Failed to convert Lottie sticker:\n${e.toString()}\n${e.stack}`
-					}
+				const {mxc_url, info} = await lottie.convert(stickerItem)
+				return {
+					$type: "m.sticker",
+					"m.mentions": mentions,
+					body: stickerItem.name,
+					info,
+					url: mxc_url
 				}
-			} else if (format?.mime) {
+			} else {
 				let body = stickerItem.name
 				const sticker = guild.stickers.find(sticker => sticker.id === stickerItem.id)
 				if (sticker && sticker.description) body += ` - ${sticker.description}`
@@ -529,12 +614,6 @@ async function messageToEvent(message, guild, options = {}, di) {
 					url: await file.uploadDiscordFileToMxc(file.sticker(stickerItem))
 				}
 			}
-			return {
-				$type: "m.room.message",
-				"m.mentions": mentions,
-				msgtype: "m.notice",
-				body: `Unsupported sticker format ${format?.mime}. Name: ${stickerItem.name}`
-			}
 		}))
 		events.push(...stickerEvents)
 	}

d2m/converters/message-to-event.pk.test.js

@@ -0,0 +1,134 @@
+const {test} = require("supertape")
+const {messageToEvent} = require("./message-to-event")
+const data = require("../../test/data")
+const Ty = require("../../types")
+
+/**
+ * @param {string} roomID
+ * @param {string} eventID
+ * @returns {(roomID: string, eventID: string) => Promise<Ty.Event.Outer<Ty.Event.M_Room_Message>>}
+ */
+function mockGetEvent(t, roomID_in, eventID_in, outer) {
+	return async function(roomID, eventID) {
+		t.equal(roomID, roomID_in)
+		t.equal(eventID, eventID_in)
+		return new Promise(resolve => {
+			setTimeout(() => {
+				resolve({
+					event_id: eventID_in,
+					room_id: roomID_in,
+					origin_server_ts: 1680000000000,
+					unsigned: {
+						age: 2245,
+						transaction_id: "$local.whatever"
+					},
+					...outer
+				})
+			})
+		})
+	}
+}
+
+test("message2event: pk reply to matrix is converted to native matrix reply", async t => {
+	const events = await messageToEvent(data.pk_message.pk_reply_to_matrix, {}, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU", {
+				type: "m.room.message",
+				sender: "@cadence:cadence.moe",
+				content: {
+					msgtype: "m.text",
+					body: "now for my next experiment:"
+				}
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {
+			user_ids: [
+				"@cadence:cadence.moe"
+			]
+		},
+		msgtype: "m.text",
+		body: "> cadence [they]: now for my next experiment:\n\nthis is a reply",
+		format: "org.matrix.custom.html",
+		formatted_body: '<mx-reply><blockquote><a href="https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU">In reply to</a> <a href="https://matrix.to/#/@cadence:cadence.moe">cadence [they]</a><br>'
+			+ "now for my next experiment:</blockquote></mx-reply>"
+			+ "this is a reply",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU"
+			}
+		}
+	}])
+})
+
+test("message2event: pk reply to discord is converted to native matrix reply", async t => {
+	const events = await messageToEvent(data.pk_message.pk_reply_to_discord, {}, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU", {
+				type: "m.room.message",
+				sender: "@_ooye_.wing.:cadence.moe",
+				content: {
+					msgtype: "m.text",
+					body: "some text"
+				}
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		"m.mentions": {},
+		body: "> wing: some text\n\nthis is a reply",
+		format: "org.matrix.custom.html",
+		formatted_body: '<mx-reply><blockquote><a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA">In reply to</a> wing<br>'
+			+ "some text</blockquote></mx-reply>"
+			+ "this is a reply",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA"
+			}
+		}
+	}])
+})
+
+test("message2event: pk reply to matrix attachment is converted to native matrix reply", async t => {
+	const events = await messageToEvent(data.pk_message.pk_reply_to_matrix_attachment, {}, {}, {
+		api: {
+			getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y", {
+				sender: "@ampflower:matrix.org",
+				type: "m.room.message",
+				content: {
+					body: "catnod.gif",
+					filename: "catnod.gif",
+					info: {
+						h: 128,
+						mimetype: "image/gif",
+						size: 20816,
+						w: 128
+					},
+					msgtype: "m.image",
+					url: "mxc://matrix.org/jtzXIawXCkFIHSsMUNsKkUJX"
+				}
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		"m.mentions": {
+			user_ids: ["@ampflower:matrix.org"]
+		},
+		body: "> Ampflower 🌺: [Media]\n\nCat nod",
+		format: "org.matrix.custom.html",
+		formatted_body: '<mx-reply><blockquote><a href="https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y">In reply to</a> <a href="https://matrix.to/#/@ampflower:matrix.org">Ampflower 🌺</a><br>'
+			+ "[Media]</blockquote></mx-reply>"
+			+ "Cat nod",
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y"
+			}
+		}
+	}])
+})

d2m/converters/message-to-event.test.js

@@ -62,14 +62,60 @@ test("message2event: simple user mention", async t => {
 })
 
 test("message2event: simple room mention", async t => {
-	const events = await messageToEvent(data.message.simple_room_mention, data.guild.general, {})
+	let called = 0
+	const events = await messageToEvent(data.message.simple_room_mention, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
 	t.deepEqual(events, [{
 		$type: "m.room.message",
 		"m.mentions": {},
 		msgtype: "m.text",
 		body: "#main",
 		format: "org.matrix.custom.html",
-		formatted_body: '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe">#main</a>'
+		formatted_body: '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe?via=cadence.moe&via=matrix.org">#main</a>'
+	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
+})
+
+test("message2event: unknown room mention", async t => {
+	const events = await messageToEvent(data.message.unknown_room_mention, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "#unknown-channel [channel from an unbridged server]"
+	}])
+})
+
+test("message2event: unbridged room mention", async t => {
+	const events = await messageToEvent(data.message.unbridged_room_mention, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "#bad-boots-prison [channel not bridged]"
 	}])
 })
 
@@ -98,15 +144,41 @@ test("message2event: manually constructed unknown roles should use fallback", as
 })
 
 test("message2event: simple message link", async t => {
-	const events = await messageToEvent(data.message.simple_message_link, data.guild.general, {})
+	let called = 0
+	const events = await messageToEvent(data.message.simple_message_link, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:super.invalid": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
 	t.deepEqual(events, [{
 		$type: "m.room.message",
 		"m.mentions": {},
 		msgtype: "m.text",
-		body: "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg",
+		body: "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg?via=cadence.moe&via=super.invalid",
 		format: "org.matrix.custom.html",
-		formatted_body: '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg</a>'
+		formatted_body: '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg?via=cadence.moe&amp;via=super.invalid">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg?via=cadence.moe&amp;via=super.invalid</a>'
 	}])
+	t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each")
 })
 
 test("message2event: message link that OOYE doesn't know about", async t => {
@@ -120,6 +192,27 @@ test("message2event: message link that OOYE doesn't know about", async t => {
 					event_id: "$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U",
 					origin_server_ts: 1613287812754
 				}
+			},
+			async getStateEvent(roomID, type, key) { // for ?via calculation
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) { // for ?via calculation
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
 			}
 		}
 	})
@@ -128,12 +221,12 @@ test("message2event: message link that OOYE doesn't know about", async t => {
 		"m.mentions": {},
 		msgtype: "m.text",
 		body: "Me: I'll scroll up to find a certain message I'll send\n_scrolls up and clicks message links for god knows how long_\n_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n_stumbles upon:_ "
-			+ "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U",
+			+ "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U?via=cadence.moe&via=matrix.org",
 		format: "org.matrix.custom.html",
 		formatted_body: "Me: I'll scroll up to find a certain message I'll send<br><em>scrolls up and clicks message links for god knows how long</em><br><em>completely forgets what they were looking for and simply begins scrolling up to find some fun moments</em><br><em>stumbles upon:</em> "
-			+ '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U</a>'
+			+ '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U?via=cadence.moe&amp;via=matrix.org">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U?via=cadence.moe&amp;via=matrix.org</a>'
 	}])
-	t.equal(called, 1, "getEventForTimestamp should be called once")
+	t.equal(called, 3, "getEventForTimestamp, getStateEvent, and getJoinedMembers should be called once each")
 })
 
 test("message2event: message link from another server", async t => {
@@ -214,6 +307,21 @@ test("message2event: stickers", async t => {
 	}])
 })
 
+test("message2event: lottie sticker", async t => {
+	const events = await messageToEvent(data.message.lottie_sticker, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.sticker",
+		"m.mentions": {},
+		body: "8",
+		info: {
+			mimetype: "image/png",
+			w: 160,
+			h: 160
+		},
+		url: "mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR"
+	}])
+})
+
 test("message2event: skull webp attachment with content", async t => {
 	const events = await messageToEvent(data.message.skull_webp_attachment_with_content, data.guild.general, {})
 	t.deepEqual(events, [{
@@ -337,6 +445,81 @@ test("message2event: simple reply to matrix user, reply fallbacks disabled", asy
 	}])
 })
 
+test("message2event: reply with a video", async t => {
+	const events = await messageToEvent(data.message.reply_with_video, data.guild.general, {
+		api: {
+			getEvent: mockGetEvent(t, "!kLRqKKUQXcibIMtOpl:cadence.moe", "$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw", {
+				type: "m.room.message",
+				content: {
+					msgtype: "m.text",
+					body: 'deadpicord "extremity you woke up at 4 am"'
+				},
+				sender: "@_ooye_extremity:cadence.moe"
+			})
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.video",
+		body: "Ins_1960637570.mp4",
+		filename: "Ins_1960637570.mp4",
+		url: "mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU",
+		external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4?ex=65bbee8f&is=65a9798f&hm=ae14f7824c3d526c5e11c162e012e1ee405fd5776e1e9302ed80ccd86503cfda&",
+		info: {
+			h: 854,
+			mimetype: "video/mp4",
+			size: 860559,
+			w: 480,
+		},
+		"m.mentions": {},
+		"m.relates_to": {
+			"m.in_reply_to": {
+				event_id: "$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw"
+			}
+		}
+	}])
+})
+
+test("message2event: voice message", async t => {
+	const events = await messageToEvent(data.message.voice_message)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+      body: "voice-message.ogg",
+      external_url: "https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg?ex=65c92d4c&is=65b6b84c&hm=0654bab5027474cbe23875954fa117cf44d8914c144cd151879590fa1baf8b1c&",
+      filename: "voice-message.ogg",
+      info: {
+        duration: 3960.0000381469727,
+        mimetype: "audio/ogg",
+        size: 10584,
+		},
+      "m.mentions": {},
+      msgtype: "m.audio",
+      url: "mxc://cadence.moe/MRRPDggXQMYkrUjTpxQbmcxB"
+	}])
+})
+
+test("message2event: misc file", async t => {
+	const events = await messageToEvent(data.message.misc_file)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "final final final revised draft",
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+      body: "the.yml",
+      external_url: "https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml?ex=65cd6270&is=65baed70&hm=8c5f1b571784e3c7f99628492298815884e351ae0dc7c2ae40dd22d97caf27d9&",
+      filename: "the.yml",
+		info: {
+			mimetype: "text/plain; charset=utf-8",
+			size: 2274
+		},
+      "m.mentions": {},
+      msgtype: "m.file",
+      url: "mxc://cadence.moe/HnQIYQmmlIKwOQsbFsIGpzPP"
+	}])
+})
+
 test("message2event: simple reply in thread to a matrix user's reply", async t => {
 	const events = await messageToEvent(data.message.simple_reply_to_reply_in_thread, data.guild.general, {}, {
 		api: {
@@ -370,9 +553,9 @@ test("message2event: simple reply in thread to a matrix user's reply", async t =
 			user_ids: ["@cadence:cadence.moe"]
 		},
 		msgtype: "m.text",
-      body: "> cadence: What about them?\n\nWell, they don't seem to...",
+      body: "> cadence [they]: What about them?\n\nWell, they don't seem to...",
       format: "org.matrix.custom.html",
-      formatted_body: "<mx-reply><blockquote><a href=\"https://matrix.to/#/!FuDZhlOAtqswlyxzeR:cadence.moe/$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo\">In reply to</a> <a href=\"https://matrix.to/#/@cadence:cadence.moe\">cadence</a><br>What about them?</blockquote></mx-reply>Well, they don't seem to...",
+      formatted_body: "<mx-reply><blockquote><a href=\"https://matrix.to/#/!FuDZhlOAtqswlyxzeR:cadence.moe/$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo\">In reply to</a> <a href=\"https://matrix.to/#/@cadence:cadence.moe\">cadence [they]</a><br>What about them?</blockquote></mx-reply>Well, they don't seem to...",
 	}])
 })
 
@@ -606,3 +789,63 @@ test("message2event: crossposted announcements say where they are crossposted fr
 		formatted_body: "🔀 <strong>Chewey Bot Official Server #announcements</strong><br>All text based commands are now inactive on Chewey Bot<br>To continue using commands you'll need to use them as slash commands"
 	}])
 })
+
+test("message2event: @everyone", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@room",
+		"m.mentions": {
+			room: true
+		}
+	}])
+})
+
+test("message2event: @here", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_here)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@room",
+		"m.mentions": {
+			room: true
+		}
+	}])
+})
+
+test("message2event: @everyone without permission", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone_without_permission)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@everyone <-- this is testing that it DOESN'T mention. if this mentions everyone then my apologies.",
+		format: "org.matrix.custom.html",
+		formatted_body: "@everyone &lt;-- this is testing that it DOESN'T mention. if this mentions everyone then my apologies.",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: @here without permission", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_here_without_permission)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@here <-- this is testing that it DOESN'T mention. if this mentions people then my apologies.",
+		format: "org.matrix.custom.html",
+		formatted_body: "@here &lt;-- this is testing that it DOESN'T mention. if this mentions people then my apologies.",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: @everyone within a link", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone_within_link)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://github.com/@everyone",
+		format: "org.matrix.custom.html",
+		formatted_body: `<a href="https://github.com/@everyone">https://github.com/@everyone</a>`,
+		"m.mentions": {}
+	}])
+})

d2m/converters/remove-reaction.js

@@ -12,7 +12,7 @@ const utils = sync.require("../../m2d/converters/utils")
  * @typedef ReactionRemoveRequest
  * @prop {string} eventID
  * @prop {string | null} mxid
- * @prop {BigInt} [hash]
+ * @prop {bigint} [hash]
  */
 
 /**

d2m/converters/thread-to-announcement.js

@@ -6,6 +6,8 @@ const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
 /** @type {import("../../matrix/read-registration")} */
 const reg = sync.require("../../matrix/read-registration.js")
+/** @type {import("../../m2d/converters/utils")} */
+const mxUtils = sync.require("../../m2d/converters/utils")
 
 const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
 
@@ -29,8 +31,9 @@ async function threadToAnnouncement(parentRoomID, threadRoomID, creatorMxid, thr
 
 	const msgtype = creatorMxid ? "m.emote" : "m.text"
 	const template = creatorMxid ? "started a thread:" : "Thread started:"
-	let body = `${template} ${thread.name} https://matrix.to/#/${threadRoomID}`
-	let html = `${template} <a href="https://matrix.to/#/${threadRoomID}">${thread.name}</a>`
+	const via = await mxUtils.getViaServersQuery(threadRoomID, di.api)
+	let body = `${template} ${thread.name} https://matrix.to/#/${threadRoomID}?${via.toString()}`
+	let html = `${template} <a href="https://matrix.to/#/${threadRoomID}?${via.toString()}">${thread.name}</a>`
 
 	return {
 		msgtype,

d2m/converters/thread-to-announcement.test.js

@@ -29,16 +29,34 @@ function mockGetEvent(t, roomID_in, eventID_in, outer) {
 	}
 }
 
+const viaApi = {
+	async getStateEvent(roomID, type, key) {
+		return {
+			users: {
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}
+	},
+	async getJoinedMembers(roomID) {
+		return {
+			joined: {
+				"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+				"@user:matrix.org": {display_name: null, avatar_url: null}
+			}
+		}
+	}
+}
+
 test("thread2announcement: no known creator, no branched from event", async t => {
 	const content = await threadToAnnouncement("!parent", "!thread", null, {
 		name: "test thread",
 		id: "-1"
-	})
+	}, {api: viaApi})
 	t.deepEqual(content, {
 		msgtype: "m.text",
-		body: "Thread started: test thread https://matrix.to/#/!thread",
+		body: "Thread started: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
 		format: "org.matrix.custom.html",
-		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread">test thread</a>`,
+		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {}
 	})
 })
@@ -47,12 +65,12 @@ test("thread2announcement: known creator, no branched from event", async t => {
 	const content = await threadToAnnouncement("!parent", "!thread", "@_ooye_crunch_god:cadence.moe", {
 		name: "test thread",
 		id: "-1"
-	})
+	}, {api: viaApi})
 	t.deepEqual(content, {
 		msgtype: "m.emote",
-		body: "started a thread: test thread https://matrix.to/#/!thread",
+		body: "started a thread: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
 		format: "org.matrix.custom.html",
-		formatted_body: `started a thread: <a href="https://matrix.to/#/!thread">test thread</a>`,
+		formatted_body: `started a thread: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {}
 	})
 })
@@ -70,14 +88,15 @@ test("thread2announcement: no known creator, branched from discord event", async
 					msgtype: 'm.text',
 					body: 'testing testing testing'
 				}
-			})
+			}),
+			...viaApi
 		}
 	})
 	t.deepEqual(content, {
 		msgtype: "m.text",
-		body: "Thread started: test thread https://matrix.to/#/!thread",
+		body: "Thread started: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
 		format: "org.matrix.custom.html",
-		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread">test thread</a>`,
+		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {},
 		"m.relates_to": {
 			"m.in_reply_to": {
@@ -100,14 +119,15 @@ test("thread2announcement: known creator, branched from discord event", async t
 					msgtype: 'm.text',
 					body: 'testing testing testing'
 				}
-			})
+			}),
+			...viaApi
 		}
 	})
 	t.deepEqual(content, {
 		msgtype: "m.emote",
-		body: "started a thread: test thread https://matrix.to/#/!thread",
+		body: "started a thread: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
 		format: "org.matrix.custom.html",
-		formatted_body: `started a thread: <a href="https://matrix.to/#/!thread">test thread</a>`,
+		formatted_body: `started a thread: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {},
 		"m.relates_to": {
 			"m.in_reply_to": {
@@ -130,14 +150,15 @@ test("thread2announcement: no known creator, branched from matrix event", async
 					body: "so can you reply to my webhook uwu"
 				},
 				sender: "@cadence:cadence.moe"
-			})
+			}),
+			...viaApi
 		}
 	})
 	t.deepEqual(content, {
 		msgtype: "m.text",
-		body: "Thread started: test thread https://matrix.to/#/!thread",
+		body: "Thread started: test thread https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org",
 		format: "org.matrix.custom.html",
-		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread">test thread</a>`,
+		formatted_body: `Thread started: <a href="https://matrix.to/#/!thread?via=cadence.moe&via=matrix.org">test thread</a>`,
 		"m.mentions": {
 			user_ids: ["@cadence:cadence.moe"]
 		},

d2m/converters/user-to-mxid.js

@@ -60,7 +60,7 @@ function userToSimName(user) {
 
 	// 1. Is sim user already registered?
 	const existing = select("sim", "sim_name", {user_id: user.id}).pluck().get()
-	if (existing) return existing
+	assert.equal(existing, null, "Shouldn't try to create a new name for an existing sim")
 
 	// 2. Register based on username (could be new or old format)
 	// (Unless it's a special user, in which case copy their provided mappings.)

d2m/discord-client.js

@@ -47,7 +47,16 @@ class DiscordClient {
 		if (listen !== "no") {
 			this.cloud.on("event", message => discordPackets.onPacket(this, message, listen))
 		}
-		this.cloud.on("error", console.error)
+
+		const addEventLogger = (eventName, logName) => {
+			this.cloud.on(eventName, (...args) => {
+				const d = new Date().toISOString().slice(0, 19)
+				console.error(`[${d} Client ${logName}]`, ...args)
+			})
+		}
+		addEventLogger("error", "Error")
+		addEventLogger("disconnected", "Disconnected")
+		addEventLogger("ready", "Ready")
 	}
 }
 

d2m/event-dispatcher.js

@@ -26,9 +26,13 @@ const updatePins = sync.require("./actions/update-pins")
 /** @type {import("../matrix/api")}) */
 const api = sync.require("../matrix/api")
 /** @type {import("../discord/utils")} */
-const utils = sync.require("../discord/utils")
+const dUtils = sync.require("../discord/utils")
 /** @type {import("../discord/discord-command-handler")}) */
 const discordCommandHandler = sync.require("../discord/discord-command-handler")
+/** @type {import("../m2d/converters/utils")} */
+const mxUtils = require("../m2d/converters/utils")
+/** @type {import("./actions/speedbump")} */
+const speedbump = sync.require("./actions/speedbump")
 
 /** @type {any} */ // @ts-ignore bad types from semaphore
 const Semaphore = require("@chriscdn/promise-semaphore")
@@ -68,20 +72,17 @@ module.exports = {
 				stackLines = stackLines.slice(0, cloudstormLine - 2)
 			}
 		}
-		let formattedBody = "\u26a0 <strong>Bridged event from Discord not delivered</strong>"
-			+ `<br>Gateway event: ${gatewayMessage.t}`
-			+ `<br>${e.toString()}`
+
+		const builder = new mxUtils.MatrixStringBuilder()
+		builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 <strong>Bridged event from Discord not delivered</strong>")
+		builder.addLine(`Gateway event: ${gatewayMessage.t}`)
+		builder.addLine(e.toString())
 		if (stackLines) {
-			formattedBody += `<br><details><summary>Error trace</summary>`
-				+ `<pre>${stackLines.join("\n")}</pre></details>`
+			builder.addLine(`Error trace:\n${stackLines.join("\n")}`, `<details><summary>Error trace</summary><pre>${stackLines.join("\n")}</pre></details>`)
 		}
-		formattedBody += `<details><summary>Original payload</summary>`
-			+ `<pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`,
+		builder.addLine("", `<details><summary>Original payload</summary><pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`)
 		api.sendEvent(roomID, "m.room.message", {
-			msgtype: "m.text",
-			body: "\u26a0 Bridged event from Discord not delivered. See formatted content for full details.",
-			format: "org.matrix.custom.html",
-			formatted_body: formattedBody,
+			...builder.get(),
 			"moe.cadence.ooye.error": {
 				source: "discord",
 				payload: gatewayMessage
@@ -113,9 +114,8 @@ module.exports = {
 			const member = guild.members.find(m => m.user?.id === client.user.id)
 			if (!member) return
 			if (!("permission_overwrites" in channel)) continue
-			const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
-			if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel
+			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
+			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look back in this channel
 
 			/** More recent messages come first. */
 			// console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
@@ -162,9 +162,8 @@ module.exports = {
 			const lastPin = updatePins.convertTimestamp(channel.last_pin_timestamp)
 
 			// Permissions check
-			const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
-			if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel
+			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
+			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look up the pins in this channel
 
 			const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get()
 			if (!row) continue // Only care about already bridged channels
@@ -235,19 +234,21 @@ module.exports = {
 	 */
 	async onMessageCreate(client, message) {
 		if (message.author.username === "Deleted User") return // Nothing we can do for deleted users.
-		if (message.webhook_id) {
-			const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get()
-			if (row) {
-				// The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
-				return
-			}
-		}
 		const channel = client.channels.get(message.channel_id)
 		if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
 		const guild = client.guilds.get(channel.guild_id)
 		assert(guild)
 
-		await sendMessage.sendMessage(message, guild),
+		if (message.webhook_id) {
+			const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get()
+			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
+		}
+
+		const {affected, row} = await speedbump.maybeDoSpeedbump(message.channel_id, message.id)
+		if (affected) return
+
+		// @ts-ignore
+		await sendMessage.sendMessage(message, channel, guild, row),
 		await discordCommandHandler.execute(message, channel, guild)
 	},
 
@@ -258,22 +259,27 @@ module.exports = {
 	async onMessageUpdate(client, data) {
 		if (data.webhook_id) {
 			const row = select("webhook", "webhook_id", {webhook_id: data.webhook_id}).pluck().get()
-			if (row) {
-				// The update was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
-				return
-			}
+			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
 		}
+
+		// Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from.
+		const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id)
+		if (affected) return
+
 		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
 		// If the message content is a string then it includes all interesting fields and is meaningful.
-		if (typeof data.content === "string") {
+		// Otherwise, if there are embeds, then the system generated URL preview embeds.
+		if (typeof data.content === "string" || "embeds" in data) {
 			/** @type {DiscordTypes.GatewayMessageCreateDispatchData} */
 			// @ts-ignore
 			const message = data
+
 			const channel = client.channels.get(message.channel_id)
 			if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
 			const guild = client.guilds.get(channel.guild_id)
 			assert(guild)
-			await editMessage.editMessage(message, guild)
+			// @ts-ignore
+			await editMessage.editMessage(message, guild, row)
 		}
 	},
 
@@ -300,6 +306,7 @@ module.exports = {
 	 * @param {DiscordTypes.GatewayMessageDeleteDispatchData} data
 	 */
 	async onMessageDelete(client, data) {
+		speedbump.onMessageDelete(data.id)
 		await deleteMessage.deleteMessage(data)
 	},
 

db/migrations/0009-add-speedbump-id.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE channel_room ADD COLUMN speedbump_id TEXT;
+ALTER TABLE channel_room ADD COLUMN speedbump_webhook_id TEXT;
+ALTER TABLE channel_room ADD COLUMN speedbump_checked INTEGER;
+
+COMMIT;

db/migrations/0010-add-sim-proxy.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS sim_proxy (
+	user_id TEXT NOT NULL,
+	proxy_owner_id TEXT NOT NULL,
+	displayname TEXT NOT NULL,
+	PRIMARY KEY(user_id)
+) WITHOUT ROWID;

db/migrations/0011-move-bridge-bot-to-real-id.up.js

@@ -0,0 +1,16 @@
+/*
+	a. If the bridge bot sim already has the correct ID:
+		- No rows updated.
+
+	b. If the bridge bot sim has the wrong ID but there's no duplicate:
+		- One row updated.
+
+	c. If the bridge bot sim has the wrong ID and there's a duplicate:
+		- One row updated (replaces an existing row).
+*/
+
+module.exports = async function(db) {
+	const config = require("../../config")
+	const id = Buffer.from(config.discordToken.split(".")[0], "base64").toString()
+	db.prepare("UPDATE OR REPLACE sim SET user_id = ? WHERE user_id = '0'").run(id)
+}

db/orm-defs.d.ts

@@ -7,6 +7,9 @@ export type Models = {
 		thread_parent: string | null
 		custom_avatar: string | null
 		last_bridged_pin_timestamp: number | null
+		speedbump_id: string | null
+		speedbump_webhook_id: string | null
+		speedbump_checked: number | null
 	}
 
 	event_message: {
@@ -60,6 +63,12 @@ export type Models = {
 		hashed_profile_content: number
 	}
 
+	sim_proxy: {
+		user_id: string
+		proxy_owner_id: string
+		displayname: string
+	}
+
 	webhook: {
 		channel_id: string
 		webhook_id: string
@@ -91,9 +100,10 @@ export type Prepared<Row> = {
 	safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
 	raw: () => Prepared<Row[keyof Row][]>
 	all: (..._: any[]) => Row[]
-	get: (..._: any[]) => Row | null
+	get: (..._: any[]) => Row | null | undefined
 }
 
 export type AllKeys<U> = U extends any ? keyof U : never
 export type PickTypeOf<T, K extends AllKeys<T>> = T extends { [k in K]?: any } ? T[K] : never
 export type Merge<U> = {[x in AllKeys<U>]: PickTypeOf<U, x>}
+export type Nullable<T> = {[k in keyof T]: T[k] | null}

db/orm.js

@@ -38,6 +38,8 @@ class From {
 		/** @private @type {Table[]} */
 		this.tables = [table]
 		/** @private */
+		this.directions = []
+		/** @private */
 		this.sql = ""
 		/** @private */
 		this.cols = []
@@ -53,12 +55,14 @@ class From {
 	 * @template {keyof U.Models} Table2
 	 * @param {Table2} table
 	 * @param {Col & (keyof U.Models[Table2])} col
+	 * @param {"inner" | "left"} [direction]
 	 */
-	join(table, col) {
+	join(table, col, direction = "inner") {
 		/** @type {From<Table | Table2, keyof U.Merge<U.Models[Table | Table2]>>} */
 		// @ts-ignore
 		const r = this
 		r.tables.push(table)
+		r.directions.push(direction.toUpperCase())
 		r.using.push(col)
 		return r
 	}
@@ -112,7 +116,8 @@ class From {
 		for (let i = 1; i < this.tables.length; i++) {
 			const table = this.tables[i]
 			const col = this.using[i-1]
-			sql += `INNER JOIN ${table} USING (${col}) `
+			const direction = this.directions[i-1]
+			sql += `${direction} JOIN ${table} USING (${col}) `
 		}
 		sql += this.sql
 		/** @type {U.Prepared<Pick<U.Merge<U.Models[Table]>, Col>>} */

db/orm.test.js

@@ -44,3 +44,12 @@ test("orm: from: where and pluck works", t => {
 	const subtypes = from("event_message").where({message_id: "1141501302736695316"}).pluck("event_subtype").all()
 	t.deepEqual(subtypes.sort(), ["m.image", "m.text"])
 })
+
+test("orm: from: join direction works", t => {
+	const hasOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "_pk_zoego"}).get()
+	t.deepEqual(hasOwner, {user_id: "43d378d5-1183-47dc-ab3c-d14e21c3fe58", proxy_owner_id: "196188877885538304"})
+	const hasNoOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
+	t.deepEqual(hasNoOwner, {user_id: "820865262526005258", proxy_owner_id: null})
+	const hasNoOwnerInner = from("sim").join("sim_proxy", "user_id", "inner").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
+	t.deepEqual(hasNoOwnerInner, undefined)
+})

discord/discord-command-handler.js

@@ -137,7 +137,7 @@ const commands = [{
 			// Check CREATE_INSTANT_INVITE permission
 			assert(message.member)
 			const guildPermissions = utils.getPermissions(message.member.roles, guild.roles)
-			if (!(guildPermissions & BigInt(1))) {
+			if (!(guildPermissions & DiscordTypes.PermissionFlagsBits.CreateInstantInvite)) {
 				return discord.snow.channel.createMessage(channel.id, {
 					...ctx,
 					content: "You don't have permission to invite people to this Discord server."

discord/utils.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 const DiscordTypes = require("discord-api-types/v10")
+const assert = require("assert").strict
 
 const EPOCH = 1420070400000
 
@@ -25,7 +26,7 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) {
 	}
 
 	if (channelOverwrites) {
-		/** @type {((overwrite: Required<DiscordTypes.APIGuildChannel>["permission_overwrites"][0]) => any)[]} */
+		/** @type {((overwrite: Required<DiscordTypes.APIOverwrite>) => any)[]} */
 		const actions = [
 			// Channel @everyone deny
 			overwrite => overwrite.id === everyoneID && (allowed &= ~BigInt(overwrite.deny)),
@@ -49,6 +50,48 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) {
 	return allowed
 }
 
+/**
+ * Note: You can only provide one permission bit to permissionToCheckFor. To check multiple permissions, call `hasAllPermissions` or `hasSomePermissions`.
+ * It is designed like this to avoid developer error with bit manipulations.
+ *
+ * @param {bigint} resolvedPermissions
+ * @param {bigint} permissionToCheckFor
+ * @returns {boolean} whether the user has the requested permission
+ * @example
+ * const permissions = getPermissions(userRoles, guildRoles, userID, channelOverwrites)
+ * hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel)
+ */
+function hasPermission(resolvedPermissions, permissionToCheckFor) {
+	// Make sure permissionToCheckFor has exactly one permission in it
+	assert.equal(permissionToCheckFor.toString(2).match(/1/g)?.length, 1)
+	// Do the actual calculation
+	return (resolvedPermissions & permissionToCheckFor) === permissionToCheckFor
+}
+
+/**
+ * @param {bigint} resolvedPermissions
+ * @param {(keyof DiscordTypes.PermissionFlagsBits)[]} permissionsToCheckFor
+ * @returns {boolean} whether the user has any of the requested permissions
+ * @example
+ * const permissions = getPermissions(userRoles, guildRoles, userID, channelOverwrites)
+ * hasSomePermissions(permissions, ["ViewChannel", "ReadMessageHistory"])
+ */
+function hasSomePermissions(resolvedPermissions, permissionsToCheckFor) {
+	return permissionsToCheckFor.some(x => hasPermission(resolvedPermissions, DiscordTypes.PermissionFlagsBits[x]))
+}
+
+/**
+ * @param {bigint} resolvedPermissions
+ * @param {(keyof DiscordTypes.PermissionFlagsBits)[]} permissionsToCheckFor
+ * @returns {boolean} whether the user has all of the requested permissions
+ * @example
+ * const permissions = getPermissions(userRoles, guildRoles, userID, channelOverwrites)
+ * hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])
+ */
+function hasAllPermissions(resolvedPermissions, permissionsToCheckFor) {
+	return permissionsToCheckFor.every(x => hasPermission(resolvedPermissions, DiscordTypes.PermissionFlagsBits[x]))
+}
+
 /**
  * Command interaction responses have a webhook_id for some reason, but still have real author info of a real bot user in the server.
  * @param {DiscordTypes.APIMessage} message
@@ -69,6 +112,9 @@ function timestampToSnowflakeInexact(timestamp) {
 }
 
 module.exports.getPermissions = getPermissions
+module.exports.hasPermission = hasPermission
+module.exports.hasSomePermissions = hasSomePermissions
+module.exports.hasAllPermissions = hasAllPermissions
 module.exports.isWebhookMessage = isWebhookMessage
 module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact
 module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact

discord/utils.test.js

@@ -1,3 +1,4 @@
+const DiscordTypes = require("discord-api-types/v10")
 const {test} = require("supertape")
 const data = require("../test/data")
 const utils = require("./utils")
@@ -82,3 +83,27 @@ test("getPermissions: channel overwrite to allow role works", t => {
 	const want = BigInt(1 << 10 | 1 << 16)
 	t.equal((permissions & want), want)
 })
+
+test("hasSomePermissions: detects the permission", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.MentionEveryone | DiscordTypes.PermissionFlagsBits.BanMembers
+	const canRemoveMembers = utils.hasSomePermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, true)
+})
+
+test("hasSomePermissions: doesn't detect not the permission", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.MentionEveryone | DiscordTypes.PermissionFlagsBits.SendMessages
+	const canRemoveMembers = utils.hasSomePermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, false)
+})
+
+test("hasAllPermissions: detects the permissions", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.KickMembers | DiscordTypes.PermissionFlagsBits.BanMembers | DiscordTypes.PermissionFlagsBits.MentionEveryone
+	const canRemoveMembers = utils.hasAllPermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, true)
+})
+
+test("hasAllPermissions: doesn't detect not the permissions", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.MentionEveryone | DiscordTypes.PermissionFlagsBits.SendMessages | DiscordTypes.PermissionFlagsBits.KickMembers
+	const canRemoveMembers = utils.hasAllPermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, false)
+})

docs/how-to-add-a-new-event-type.md

@@ -32,13 +32,13 @@ What does it look like on Discord-side?
 
 This is an API request to get the pinned messages. To update this, an API request will pin or unpin any specific message, adding or removing it from the list.
 
-## What will the converter look like?
+## What will the converter do?
 
 The converter will be very different in both directions.
 
-For d2m, we will get the list of pinned messages, we will convert each message ID into the ID of an event we already have, and then we will set the entire `m.room.pinned_events` state to that list.
+**For d2m, we will get the list of pinned messages, we will convert each message ID into the ID of an event we already have, and then we will set the entire `m.room.pinned_events` state to that list.**
 
-For m2d, we will have to diff the list of pinned messages against the previous version of the list, and for each event that was pinned or unpinned, we will send an API request to Discord to change its state.
+**For m2d, we will have to diff the list of pinned messages against the previous version of the list, and for each event that was pinned or unpinned, we will send an API request to Discord to change its st**ate.
 
 ## Missing messages
 
@@ -53,7 +53,7 @@ In this situation we need to stop and think about the possible paths forward we
 
 The latter method would still make the message appear at the bottom of the timeline for most Matrix clients, since for most the timestamp doesn't determine the actual _order._ It would then be confusing why an odd message suddenly appeared, because a pins change isn't that noticable in the room.
 
-To avoid this problem, I'll just go with the former method and ignore the message, so Matrix will only have some of the pins that Discord has. We will need to watch out if a Matrix user edits this list of partial pins, because if we _only_ pinned things on Discord that were pinned on Matrix, those partial pins Discord would be lost from Discord side.
+To avoid this problem, I'll just go with the former method and ignore the message, so Matrix will only have some of the pins that Discord has. We will need to watch out if a Matrix user edits this list of partial pins, because if we _only_ pinned things on Discord that were pinned on Matrix, then pins Matrix doesn't know about would be lost from Discord side.
 
 In this situation I will prefer to keep the pins list inconsistent between both sides and only bridge _changes_ to the list.
 
@@ -61,7 +61,9 @@ If you were implementing this for real, you might have made different decisions
 
 ## Test data for the d2m converter
 
-Let's start writing the d2m converter. It's helpful to write unit tests for Out Of Your Element, since this lets you check if it worked without having to start up a local copy of the bridge or play around with the interface.
+Let's start writing the d2m converter. It's helpful to write automated tests for Out Of Your Element, since this lets you check if it worked without having to start up a local copy of the bridge or mess around with the interface.
+
+To test the Discord-to-Matrix pin converter, we'll need some samples of Discord message objects. Then we can put these sample message objects through the converter and check what comes out the other side.
 
 Normally for getting test data, I would `curl` the Discord API to grab some real data and put it into `data.js` (and possibly also `ooye-test-data.sql`. But this time, I'll fabricate some test data. Here it is:
 
@@ -74,7 +76,7 @@ Normally for getting test data, I would `curl` the Discord API to grab some real
 ]
 ```
 
-"These aren't message objects!" I hear you cry. Correct. I already know that my implementation is not going to care about any properties on these message object other than the IDs, so I'm just making a list of IDs to save time.
+"These aren't message objects!" I hear you cry. Correct. I already know that my implementation is not going to care about any properties on these message object other than the IDs, so to save time, I'm just making a list of IDs.
 
 These IDs were carefully chosen. The first three are already in `ooye-test-data.sql` and are associated with event IDs. This is great, because in our test case, the Discord IDs will be converted to those event IDs. The fourth ID doesn't exist on Matrix-side. This is to test that partial pins are handled as expected, like I wrote in the previous section.
 
@@ -104,7 +106,7 @@ index c36f252..4919beb 100644
 
 ## Writing the d2m converter
 
-We can write a function that operates on this data to convert it to events. This is a _converter,_ not an _action._ it won't _do_ anything by itself. So it goes in the converters folder. The actual function is pretty simple since I've already planned what to do:
+We can write a function that operates on this data to convert it to events. This is a _converter,_ not an _action._ It won't _do_ anything by itself. So it goes in the converters folder. I've already planned (in the "What will the converter do?" section) what to do, so writing the function is pretty simple:
 
 ```diff
 diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js
@@ -133,9 +135,36 @@ index 0000000..e4107be
 +module.exports.pinsToList = pinsToList
 ```
 
+### Explaining the code
+
+All converters have a `function` which does the work, and the function is added to `module.exports` so that other files can use it.
+
+Importing `select` from `passthrough` lets us do database access. Calling the `select` function can select from OOYE's own SQLite database. If you want to see what's in the database, look at `ooye-test-data.sql` for test data, or open `ooye.db` for real data from your own bridge.
+
+The comments `// @ts-check`, `/** @type ... */`, and `/** @param ... */` provide type-based autosuggestions when editing in Visual Studio Code.
+
+Here's the code I haven't yet discussed:
+
+```js
+function pinsToList(pins) {
+	const result = []
+	for (const message of pins) {
+		const eventID = select("event_message", "event_id", {message_id: message.id}).pluck().get()
+		if (eventID) result.push(eventID)
+	}
+	return result
+}
+```
+
+It will go through each `message` in `pins`. For each message, it will look up the corresponding Matrix event in the database, and if found, it will add it to `result`.
+
+The `select` line will run this SQL: `SELECT event_id FROM event_message WHERE message_id = {the message ID}` and will return the event ID as a string or null.
+
+For any database experts worried about an SQL query inside a loop, the N+1 problem does not apply to SQLite because the queries are executed in the same process rather than crossing a process (and network) boundary. https://www.sqlite.org/np1queryprob.html
+
 ## Test case for the d2m converter
 
-There's not much room for bugs in this function. A single manual test that it works would be good enough for me. But since this is an example of how you can add your own, let's add a test case for this. We'll take the data we just prepared and process it through the function we just wrote:
+There's not much room for bugs in this function. A single manual test that it works would be good enough for me. But since this is an example of how you can add your own, let's add a test case for this. The testing code will take the data we just prepared and process it through the `pinsToList` function we just wrote. Then, it will check the result is what we expected.
 
 ```diff
 diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js
@@ -177,6 +206,18 @@ index 5cc851e..280503d 100644
 
 Good to go.
 
+### Explaining the code
+
+`require("supertape")` is a library that helps with testing and printing test results. `data = require("../../test/data")` is the file we edited earlier in the "Test data for the d2m converter" section. `require("./pins-to-list")` is the function we want to test.
+
+Here is how you declare a test: `test("pins2list: converts known IDs, ignores unknown IDs", t => {` The string describes what you are trying to test and it will be displayed if the test fails.
+
+`result = pinsToList(data.pins.faked)` is calling the implementation function we wrote.
+
+`t.deepEqual(actual, expected)` will check whether the `actual` result value is the same as our `expected` result value. If it's not, it'll mark that as a failed test.
+
+### Run the test!
+
 ```
 ><> $ npm t
 
@@ -209,7 +250,11 @@ Oh no! (I promise I didn't make it fail for demonstration purposes, this was act
 ('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1),
 ```
 
-Explanation: This Discord message `1141501302736695316` is actually part of 2 different Matrix events, `$mtR...` and `$51f...`. This often happens when a Discord user uploads an image with a caption. Matrix doesn't support combined image+text events, so the image and the text have to be bridged to separate events. We should consider the text to be the primary part, and pin that, and consider the image to be the secondary part, and not pin that.
+Explanation: This Discord message `1141501302736695316` is actually part of 2 different Matrix events, `$mtR...` and `$51f...`. This often happens when a Discord user uploads an image with a caption. Matrix doesn't support combined image+text events, so the image and the text have to be bridged to separate events.
+
+In the current code, `pinsToList` is picking ALL the associated event IDs, and then `.get` is forcing it to limit that list to 1. It doesn't care which, so it's essentially random which event it wants to pin.
+
+We should make a decision on which event is more important. You can make whatever decision you want - you could even make it pin every event associated with a message - but I've decided that the text should be the primary part and be pinned, and the image should be considered a secondary part and left unpinned.
 
 We already have a column `part` in the `event_message` table for this reason! When `part = 0`, that's the primary part. I'll edit the converter to actually use that column:
 
@@ -229,6 +274,8 @@ index e4107be..f401de2 100644
         return result
 ```
 
+As long as the database is consistent, this new `select` will return at most 1 event, always choosing the primary part.
+
 ```
 ><> $ npm t
 
@@ -341,6 +388,8 @@ I try to keep as much logic as possible out of the actions and in the converters
 
 ## See if it works
 
+Since the automated tests pass, let's start up the bridge and run our nice new code:
+
 ```
 node start.js
 ```
@@ -359,7 +408,7 @@ I expected that to be the end of the guide, but after some time, I noticed a new
 
 [After some investigation,](https://gitdab.com/cadence/out-of-your-element/issues/16) it turns out Discord puts the most recently pinned message at the start of the array and displays the array in forwards order, while Matrix puts the most recently pinned message at the end of the array and displays the array in reverse order.
 
-We'll fix this by reversing the order of the list of pins before we store it. I'll do this in the converter.
+We can fix this by reversing the order of the list of pins before we store it. The converter can do this:
 
 ```diff
 diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js
@@ -405,7 +454,7 @@ index c2e3774..92e5678 100644
   Pass!
 ```
 
-Next time a message is pinned or unpinned on Discord, the order should be updated on Matrix.
+Next time a message is pinned or unpinned on Discord, OOYE should update the order of all the pins on Matrix.
 
 ## Notes on missed events
 

docs/pluralkit-notetaking.md

@@ -85,7 +85,9 @@ OOYE's speedbump will prevent the edit command appearing at all on Matrix-side,
 
 ## Database schema
 
-TBD
+* channel_room
+	+ speedbump_id - the ID of the webhook that may be proxying in this channel
+	+ speedbump_checked - time in unix seconds when the webhooks were last queried
 
 ## Unsolved problems
 

m2d/actions/channel-webhook.js

@@ -57,7 +57,7 @@ async function withWebhook(channelID, callback) {
  */
 async function sendMessageWithWebhook(channelID, data, threadID) {
 	const result = await withWebhook(channelID, async webhook => {
-		return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID, disableEveryone: true})
+		return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID})
 	})
 	return result
 }

m2d/actions/emoji-sheet.js

@@ -0,0 +1,36 @@
+// @ts-check
+
+const assert = require("assert")
+const fetch = require("node-fetch").default
+
+const utils = require("../converters/utils")
+const {sync} = require("../../passthrough")
+
+/** @type {import("../converters/emoji-sheet")} */
+const emojiSheetConverter = sync.require("../converters/emoji-sheet")
+
+/**
+ * Downloads the emoji from the web and converts to uncompressed PNG data.
+ * @param {string} mxc a single mxc:// URL
+ * @returns {Promise<Buffer | undefined>} uncompressed PNG data, or undefined if the downloaded emoji is not valid
+ */
+async function getAndConvertEmoji(mxc) {
+	const abortController = new AbortController()
+
+	const url = utils.getPublicUrlForMxc(mxc)
+	assert(url)
+
+	/** @type {import("node-fetch").Response} */
+	// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
+	// If we were using connection pooling, we would be forced to download the entire GIF.
+	// So we set no agent to ensure we are not connection pooling.
+	// @ts-ignore the signal is slightly different from the type it wants (still works fine)
+	const res = await fetch(url, {agent: false, signal: abortController.signal})
+	return emojiSheetConverter.convertImageStream(res.body, () => {
+		abortController.abort()
+		res.body.pause()
+		res.body.emit("end")
+	})
+}
+
+module.exports.getAndConvertEmoji = getAndConvertEmoji

m2d/actions/send-event.js

@@ -17,6 +17,10 @@ const eventToMessage = sync.require("../converters/event-to-message")
 const api = sync.require("../../matrix/api")
 /** @type {import("../../d2m/actions/register-user")} */
 const registerUser = sync.require("../../d2m/actions/register-user")
+/** @type {import("../../d2m/actions/edit-message")} */
+const editMessage = sync.require("../../d2m/actions/edit-message")
+/** @type {import("../actions/emoji-sheet")} */
+const emojiSheet = sync.require("../actions/emoji-sheet")
 
 /**
  * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message
@@ -75,7 +79,7 @@ async function sendEvent(event) {
 
 	// no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it
 
-	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow})
+	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch, mxcDownloader: emojiSheet.getAndConvertEmoji})
 
 	messagesToEdit = await Promise.all(messagesToEdit.map(async e => {
 		e.message = await resolvePendingFiles(e.message)
@@ -86,6 +90,7 @@ async function sendEvent(event) {
 	}))
 
 	let eventPart = 0 // 0 is primary, 1 is supporting
+	const pendingEdits = []
 
 	/** @type {DiscordTypes.APIMessage[]} */
 	const messageResponses = []
@@ -109,12 +114,33 @@ async function sendEvent(event) {
 
 		eventPart = 1
 		messageResponses.push(messageResponse)
+
+		/*
+			If the Discord system has a cached link preview embed for one of the links just sent,
+			it will be instantly added as part of `embeds` and there won't be a MESSAGE_UPDATE.
+			To reflect the generated embed back to Matrix, we pretend the message was updated right away.
+		*/
+		const sentEmbedsCount = message.embeds?.length || 0
+		if (messageResponse.embeds.length > sentEmbedsCount) {
+			// not awaiting here because requests to Matrix shouldn't block requests to Discord
+			pendingEdits.push(() =>
+				// @ts-ignore this is a valid message edit payload
+				editMessage.editMessage({
+					id: messageResponse.id,
+					channel_id: messageResponse.channel_id,
+					guild_id: guild.id,
+					embeds: messageResponse.embeds
+				}, guild, null)
+			)
+		}
 	}
 
 	for (const user of ensureJoined) {
 		registerUser.ensureSimJoined(user, event.room_id)
 	}
 
+	await Promise.all(pendingEdits.map(f => f())) // `await` will propagate any errors during editing
+
 	return messageResponses
 }
 

m2d/converters/emoji-sheet.js

@@ -4,8 +4,7 @@ const assert = require("assert").strict
 const {pipeline} = require("stream").promises
 const sharp = require("sharp")
 const {GIFrame} = require("giframe")
-const utils = require("./utils")
-const fetch = require("node-fetch").default
+const {PNG} = require("pngjs")
 const streamMimeType = require("stream-mime-type")
 
 const SIZE = 48
@@ -15,64 +14,11 @@ const IMAGES_ACROSS = Math.floor(RESULT_WIDTH / SIZE)
 /**
  * Composite a bunch of Matrix emojis into a kind of spritesheet image to upload to Discord.
  * @param {string[]} mxcs mxc URLs, in order
+ * @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
  * @returns {Promise<Buffer>} PNG image
  */
-async function compositeMatrixEmojis(mxcs) {
-	let buffers = await Promise.all(mxcs.map(async mxc => {
-		const abortController = new AbortController()
-
-		try {
-			const url = utils.getPublicUrlForMxc(mxc)
-			assert(url)
-
-			/** @type {import("node-fetch").Response} res */
-			// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
-			// If we were using connection pooling, we would be forced to download the entire GIF.
-			// So we set no agent to ensure we are not connection pooling.
-			// @ts-ignore the signal is slightly different from the type it wants (still works fine)
-			const res = await fetch(url, {agent: false, signal: abortController.signal})
-			const {stream, mime} = await streamMimeType.getMimeType(res.body)
-
-			if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") {
-				/** @type {{info: sharp.OutputInfo, buffer: Buffer}} */
-				const result = await new Promise((resolve, reject) => {
-					const transformer = sharp()
-						.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
-						.png({compressionLevel: 0})
-						.toBuffer((err, buffer, info) => {
-							/* c8 ignore next */
-							if (err) return reject(err)
-							resolve({info, buffer})
-						})
-					pipeline(
-						stream,
-						transformer
-					)
-				})
-				return result.buffer
-
-			} else if (mime === "image/gif") {
-				const giframe = new GIFrame(0)
-				stream.on("data", chunk => {
-					giframe.feed(chunk)
-				})
-				const frame = await giframe.getFrame()
-
-				const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}})
-					.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
-					.png({compressionLevel: 0})
-					.toBuffer({resolveWithObject: true})
-				return buffer.data
-
-			} else {
-				// unsupported mime type
-				console.error(`I don't know what a ${mime} emoji is.`)
-				return null
-			}
-		} finally {
-			abortController.abort()
-		}
-	}))
+async function compositeMatrixEmojis(mxcs, mxcDownloader) {
+	const buffers = await Promise.all(mxcs.map(mxcDownloader))
 
 	// Calculate the size of the final composited image
 	const totalWidth = Math.min(buffers.length, IMAGES_ACROSS) * SIZE
@@ -101,4 +47,67 @@ async function compositeMatrixEmojis(mxcs) {
 	return output.data
 }
 
+/**
+ * @param {import("node-fetch").Response["body"]} streamIn
+ * @param {() => any} stopStream
+ * @returns {Promise<Buffer | undefined>} Uncompressed PNG image
+ */
+async function convertImageStream(streamIn, stopStream) {
+	const {stream, mime} = await streamMimeType.getMimeType(streamIn)
+	assert(["image/png", "image/jpeg", "image/webp", "image/gif", "image/apng"].includes(mime), `Mime type ${mime} is impossible for emojis`)
+
+	try {
+		if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") {
+			/** @type {{info: sharp.OutputInfo, buffer: Buffer}} */
+			const result = await new Promise((resolve, reject) => {
+				const transformer = sharp()
+					.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
+					.png({compressionLevel: 0})
+					.toBuffer((err, buffer, info) => {
+						/* c8 ignore next */
+						if (err) return reject(err)
+						resolve({info, buffer})
+					})
+				pipeline(
+					stream,
+					transformer
+				)
+			})
+			return result.buffer
+
+		} else if (mime === "image/gif") {
+			const giframe = new GIFrame(0)
+			stream.on("data", chunk => {
+				giframe.feed(chunk)
+			})
+			const frame = await giframe.getFrame()
+			stopStream()
+
+			const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}})
+				.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
+				.png({compressionLevel: 0})
+				.toBuffer({resolveWithObject: true})
+			return buffer.data
+
+		} else if (mime === "image/apng") {
+			const png = new PNG({maxFrames: 1})
+			// @ts-ignore
+			stream.pipe(png)
+			/** @type {Buffer} */ // @ts-ignore
+			const frame = await new Promise(resolve => png.on("parsed", resolve))
+			stopStream()
+
+			const buffer = await sharp(frame, {raw: {width: png.width, height: png.height, channels: 4}})
+				.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
+				.png({compressionLevel: 0})
+				.toBuffer({resolveWithObject: true})
+			return buffer.data
+
+		}
+	} finally {
+		stopStream()
+	}
+}
+
 module.exports.compositeMatrixEmojis = compositeMatrixEmojis
+module.exports.convertImageStream = convertImageStream

m2d/converters/emoji-sheet.test.js

@@ -0,0 +1,58 @@
+const {test} = require("supertape")
+const {convertImageStream} = require("./emoji-sheet")
+const fs = require("fs")
+const {Transform} = require("stream").Transform
+
+/* c8 ignore next 7 */
+function slow() {
+	if (process.argv.includes("--slow")) {
+		return test
+	} else {
+		return test.skip
+	}
+}
+
+class Meter extends Transform {
+	bytes = 0
+
+	_transform(chunk, encoding, cb) {
+		this.bytes += chunk.length
+		this.push(chunk)
+		cb()
+	}
+}
+
+/**
+ * @param {import("supertape").Test} t
+ * @param {string} path
+ * @param {number} totalSize
+ * @param {number => boolean} sizeCheck
+ */
+async function runSingleTest(t, path, totalSize, sizeCheck) {
+	const file = fs.createReadStream(path)
+	const meter = new Meter()
+	const p = file.pipe(meter)
+	const result = await convertImageStream(p, () => {
+		file.pause()
+		file.emit("end")
+	})
+	t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `test that this is a PNG file: ${result.toString("base64").slice(0, 100)}`)
+	/* c8 ignore next 5 */
+	if (sizeCheck(meter.bytes)) {
+		t.pass("read the correct amount of the file")
+	} else {
+		t.fail(`read too much or too little of the file, read: ${meter.bytes}, total: ${totalSize}`)
+	}
+}
+
+slow()("emoji-sheet: only partial file is read for APNG", async t => {
+	await runSingleTest(t, "test/res/butterfly.png", 2438998, n => n < 2438998 / 4) // should download less than 25% of the file
+})
+
+slow()("emoji-sheet: only partial file is read for GIF", async t => {
+	await runSingleTest(t, "test/res/butterfly.gif", 781223, n => n < 781223 / 4) // should download less than 25% of the file
+})
+
+slow()("emoji-sheet: entire file is read for static PNG", async t => {
+	await runSingleTest(t, "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png", 11301, n => n === 11301) // should download entire file
+})

m2d/converters/event-to-message.js

@@ -5,6 +5,7 @@ const DiscordTypes = require("discord-api-types/v10")
 const {Readable} = require("stream")
 const chunk = require("chunk-text")
 const TurndownService = require("turndown")
+const domino = require("domino")
 const assert = require("assert").strict
 const entities = require("entities")
 
@@ -32,13 +33,19 @@ const markdownEscapes = [
 	[/^>/g, '\\>'],
 	[/_/g, '\\_'],
 	[/^(\d+)\. /g, '$1\\. ']
- ]
+	/*
+		Strikethrough is deliberately not escaped. Usually when Matrix users type ~~ it's not because they wanted to send ~~,
+		it's because they wanted strikethrough and it didn't work because their client doesn't support it.
+		As bridge developers, we can choose between "messages should look as similar as possible" vs "it was most likely intended to be strikethrough".
+		I went with the latter. Even though the appearance doesn't match, I'd rather it displayed as originally intended for 80% of the readers than for 0%.
+	*/
+]
 
 const turndownService = new TurndownService({
 	hr: "----",
 	headingStyle: "atx",
 	preformattedCode: true,
-	codeBlockStyle: "fenced",
+	codeBlockStyle: "fenced"
 })
 
 /**
@@ -86,11 +93,15 @@ turndownService.addRule("blockquote", {
 
 turndownService.addRule("spoiler", {
 	filter: function (node, options) {
-		return node.hasAttribute("data-mx-spoiler")
+		return node.tagName === "SPAN" && node.hasAttribute("data-mx-spoiler")
 	},
 
 	replacement: function (content, node) {
-		return "||" + content + "||"
+		if (node.getAttribute("data-mx-spoiler")) {
+			// escape parentheses so it can't become a link
+			return `\\(${node.getAttribute("data-mx-spoiler")}\\) ||${content}||`
+		}
+		return `||${content}||`
 	}
 })
 
@@ -103,13 +114,22 @@ turndownService.addRule("inlineLink", {
 	},
 
 	replacement: function (content, node) {
-		if (node.getAttribute("data-user-id")) return `<@${node.getAttribute("data-user-id")}>`
+		if (node.getAttribute("data-user-id")) {
+			const user_id = node.getAttribute("data-user-id")
+			const row = select("sim_proxy", ["displayname", "proxy_owner_id"], {user_id}).get()
+			if (row) {
+				return `**@${row.displayname}** (<@${row.proxy_owner_id}>)`
+			} else {
+				return `<@${user_id}>`
+			}
+		}
 		if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}`
 		if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>`
 		const href = node.getAttribute("href")
-		let brackets = ["", ""]
-		if (href.startsWith("https://matrix.to")) brackets = ["<", ">"]
-		return "[" + content + "](" + brackets[0] + href + brackets[1] + ")"
+		content = content.replace(/ @.*/, "")
+		if (href === content) return href
+		if (href.startsWith("https://matrix.to/#/@") && content[0] !== "@") content = "@" + content
+		return "[" + content + "](" + href + ")"
 	}
 })
 
@@ -141,32 +161,8 @@ turndownService.addRule("emoji", {
 
 	replacement: function (content, node) {
 		const mxcUrl = node.getAttribute("src")
-		// Get the known emoji from the database. (We may not be able to actually use this if it was from another server.)
-		const row = select("emoji", ["emoji_id", "name", "animated"], {mxc_url: mxcUrl}).get()
-		// Also guess a suitable emoji based on the ID (if available) or name
-		let guess = null
 		const guessedName = node.getAttribute("title").replace(/^:|:$/g, "")
-		for (const guild of discord.guilds.values()) {
-			/** @type {{name: string, id: string, animated: number}[]} */
-			// @ts-ignore
-			const emojis = guild.emojis
-			const match = emojis.find(e => e.id === row?.emoji_id) || emojis.find(e => e.name === guessedName) || emojis.find(e => e.name?.toLowerCase() === guessedName.toLowerCase())
-			if (match) {
-				guess = match
-				break
-			}
-		}
-		if (guess) {
-			// We know an emoji, and we can use it
-			const animatedChar = guess.animated ? "a" : ""
-			return `<${animatedChar}:${guess.name}:${guess.id}>`
-		} else if (endOfMessageEmojis.includes(mxcUrl)) {
-			// We can't locate or use a suitable emoji. After control returns, it will rewind over this, delete this section, and upload the emojis as a sprite sheet.
-			return `<::>`
-		} else {
-			// We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle.
-			return `[${node.getAttribute("title")}](${mxUtils.getPublicUrlForMxc(mxcUrl)})`
-		}
+		return convertEmoji(mxcUrl, guessedName, true, true)
 	}
 })
 
@@ -183,7 +179,7 @@ turndownService.addRule("fencedCodeBlock", {
 		const className = node.firstChild.getAttribute("class") || ""
 		const language = (className.match(/language-(\S+)/) || [null, ""])[1]
 		const code = node.firstChild
-		const visibleCode = code.childNodes.map(c => c.nodeName === "BR" ? "\n" : c.textContent).join("").replace(/\n*$/g, "")
+		const visibleCode = getCodeContent(code)
 
 		var fence = "```"
 
@@ -195,6 +191,62 @@ turndownService.addRule("fencedCodeBlock", {
 	}
 })
 
+/** @param {{ childNodes: Node[]; }} preCode the <code> directly inside the <pre> */
+function getCodeContent(preCode) {
+	return preCode.childNodes.map(c => c.nodeName === "BR" ? "\n" : c.textContent).join("").replace(/\n*$/g, "")
+}
+
+/**
+ * @param {string | null} mxcUrl
+ * @param {string | null} nameForGuess without colons
+ * @param {boolean} allowSpriteSheetIndicator
+ * @param {boolean} allowLink
+ * @returns {string} discord markdown that represents the custom emoji in some form
+ */
+function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink) {
+	// Get the known emoji from the database.
+	let row
+	if (mxcUrl) row = select("emoji", ["emoji_id", "name", "animated"], {mxc_url: mxcUrl}).get()
+	// Now we have to search all servers to see if we're able to send this emoji.
+	if (row) {
+		const found = [...discord.guilds.values()].find(g => g.emojis.find(e => e.id === row.id))
+		if (!found) row = null
+	}
+	// Or, if we don't have an emoji right now, we search for the name instead.
+	if (!row && nameForGuess) {
+		const nameForGuessLower = nameForGuess.toLowerCase()
+		for (const guild of discord.guilds.values()) {
+			/** @type {{name: string, id: string, animated: number}[]} */
+			// @ts-ignore
+			const emojis = guild.emojis
+			const found = emojis.find(e => e.id === row?.id || e.name?.toLowerCase() === nameForGuessLower)
+			if (found) {
+				row = {
+					animated: found.animated,
+					emoji_id: found.id,
+					name: found.name
+				}
+				break
+			}
+		}
+	}
+	if (row) {
+		// We know an emoji, and we can use it
+		const animatedChar = row.animated ? "a" : ""
+		return `<${animatedChar}:${row.name}:${row.emoji_id}>`
+	} else if (allowSpriteSheetIndicator && mxcUrl && endOfMessageEmojis.includes(mxcUrl)) {
+		// We can't locate or use a suitable emoji. After control returns, it will rewind over this, delete this section, and upload the emojis as a sprite sheet.
+		return `<::>`
+	} else if (allowLink && mxcUrl && nameForGuess) {
+		// We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle.
+		return `[:${nameForGuess}:](${mxUtils.getPublicUrlForMxc(mxcUrl)})`
+	} else if (nameForGuess) {
+		return `:${nameForGuess}:`
+	} else {
+		return ""
+	}
+}
+
 /**
  * @param {string} roomID
  * @param {string} mxid
@@ -212,8 +264,8 @@ async function getMemberFromCacheOrHomeserver(roomID, mxid, api) {
 }
 
 /**
- * Splits a display name into one chunk containing <=80 characters, and another chunk containing the rest of the characters. Splits on
- * whitespace if possible.
+ * Splits a display name into one chunk containing <=80 characters (80 being how many characters Discord allows for the name of a webhook),
+ * and another chunk containing the rest of the characters. Splits on whitespace if possible.
  * These chunks, respectively, go in the display name, and at the top of the message.
  * If the second part isn't empty, it'll also contain boldening markdown and a line break at the end, so that regardless of its value it
  * can be prepended to the message content as-is.
@@ -236,14 +288,25 @@ function splitDisplayName(displayName) {
 	}
 }
 
+/**
+ * Convert a Matrix user ID into a Discord user ID for mentioning, where if the user is a PK proxy, it will mention the proxy owner.
+ * @param {string} mxid
+ */
+function getUserOrProxyOwnerID(mxid) {
+	const row = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({mxid}).get()
+	if (!row) return null
+	return row.proxy_owner_id || row.user_id
+}
+
 /**
  * At the time of this executing, we know what the end of message emojis are, and we know that at least one of them is unknown.
  * This function will strip them from the content and generate the correct pending file of the sprite sheet.
  * @param {string} content
  * @param {{id: string, name: string}[]} attachments
  * @param {({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} pendingFiles
+ * @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
  */
-async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles) {
+async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, mxcDownloader) {
 	if (!content.includes("<::>")) return content // No unknown emojis, nothing to do
 	// Remove known and unknown emojis from the end of the message
 	const r = /<a?:[a-zA-Z0-9_]*:[0-9]*>\s*$/
@@ -251,10 +314,10 @@ async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
 		content = content.replace(r, "")
 	}
 	// Create a sprite sheet of known and unknown emojis from the end of the message
-	const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis)
+	const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis, mxcDownloader)
 	// Attach it
 	const name = "emojis.png"
-	attachments.push({id: "0", name})
+	attachments.push({id: String(attachments.length), name})
 	pendingFiles.push({name, buffer})
 	return content
 }
@@ -301,8 +364,12 @@ async function handleRoomOrMessageLinks(input, di) {
 				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, messageID)
 			} else {
 				// 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp
-				const originalEvent = await di.api.getEvent(roomID, eventID)
-				if (!originalEvent) continue
+				let originalEvent
+				try {
+					originalEvent = await di.api.getEvent(roomID, eventID)
+				} catch (e) {
+					continue // Our homeserver doesn't know about the event, so can't resolve it to a Discord link
+				}
 				const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts)
 				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, guessedMessageID)
 			}
@@ -315,17 +382,68 @@ async function handleRoomOrMessageLinks(input, di) {
 	return input
 }
 
+/**
+ * @param {string} content
+ * @param {string} senderMxid
+ * @param {string} roomID
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di
+ */
+async function checkWrittenMentions(content, senderMxid, roomID, guild, di) {
+	let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+
+	if (writtenMentionMatch) {
+		if (writtenMentionMatch[1] === "room") { // convert @room to @everyone
+			const powerLevels = await di.api.getStateEvent(roomID, "m.room.power_levels", "")
+			const userPower = powerLevels.users?.[senderMxid] || 0
+			if (userPower >= powerLevels.notifications?.room) {
+				return {
+					// @ts-ignore - typescript doesn't know about indices yet
+					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `@everyone` + content.slice(writtenMentionMatch.indices[1][1]),
+					ensureJoined: [],
+					allowedMentionsParse: ["everyone"]
+				}
+			}
+		} else {
+			const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]})
+			if (results[0]) {
+				assert(results[0].user)
+				return {
+					// @ts-ignore - typescript doesn't know about indices yet
+					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]),
+					ensureJoined: [results[0].user],
+					allowedMentionsParse: []
+				}
+			}
+		}
+	}
+}
+
+/**
+ * @param {Element} node
+ * @param {string[]} tagNames allcaps tag names
+ * @returns {any | undefined} the node you were checking for, or undefined
+ */
+function nodeIsChildOf(node, tagNames) {
+	// @ts-ignore
+	for (; node; node = node.parentNode) if (tagNames.includes(node.tagName)) return node
+}
+
+const attachmentEmojis = new Map([
+	["m.image", "🖼️"],
+	["m.video", "🎞️"],
+	["m.audio", "🎶"],
+	["m.file", "📄"]
+])
+
 /**
  * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event
  * @param {import("discord-api-types/v10").APIGuild} guild
- * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer}} di simple-as-nails dependency injection for the matrix API
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"], mxcDownloader: (mxc: string) => Promise<Buffer | undefined>}} di simple-as-nails dependency injection for the matrix API
  */
 async function eventToMessage(event, guild, di) {
-	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
-	let messages = []
-
 	let displayName = event.sender
 	let avatarURL = undefined
+	const allowedMentionsParse = ["users", "roles"]
 	/** @type {string[]} */
 	let messageIDsToEdit = []
 	let replyLine = ""
@@ -356,12 +474,10 @@ async function eventToMessage(event, guild, di) {
 		// Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events.
 		// this event ---is an edit of--> original event ---is a reply to--> past event
 		await (async () => {
-			if (!event.content["m.new_content"]) return
+			// Check if there is an edit
 			const relatesTo = event.content["m.relates_to"]
-			if (!relatesTo) return
+			if (!event.content["m.new_content"] || !relatesTo || relatesTo.rel_type !== "m.replace") return
 			// Check if we have a pointer to what was edited
-			const relType = relatesTo.rel_type
-			if (relType !== "m.replace") return
 			const originalEventId = relatesTo.event_id
 			if (!originalEventId) return
 			messageIDsToEdit = select("event_message", "message_id", {event_id: originalEventId}, "ORDER BY part").pluck().all()
@@ -393,8 +509,43 @@ async function eventToMessage(event, guild, di) {
 		await (async () => {
 			const repliedToEventId = event.content["m.relates_to"]?.["m.in_reply_to"]?.event_id
 			if (!repliedToEventId) return
-			let repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId)
-			if (!repliedToEvent) return
+			let repliedToEvent
+			try {
+				repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId)
+			} catch (e) {
+				// Original event isn't on our homeserver, so we'll *partially* trust the client's reply fallback.
+				// We'll trust the fallback's quoted content and put it in the reply preview, but we won't trust the authorship info on it.
+
+				// (But if the fallback's quoted content doesn't exist, we give up. There's nothing for us to quote.)
+				if (event.content["format"] !== "org.matrix.custom.html" || typeof event.content["formatted_body"] !== "string") {
+					const lines = event.content.body.split("\n")
+					let stage = 0
+					for (let i = 0; i < lines.length; i++) {
+						if (stage >= 0 && lines[i][0] === ">") stage = 1
+						if (stage >= 1 && lines[i].trim() === "") stage = 2
+						if (stage === 2 && lines[i].trim() !== "") {
+							event.content.body = lines.slice(i).join("\n")
+							break
+						}
+					}
+					return
+				}
+				const mxReply = event.content["formatted_body"]
+				const quoted = mxReply.match(/^<mx-reply><blockquote>.*?In reply to.*?<br>(.*)<\/blockquote><\/mx-reply>/)?.[1]
+				if (!quoted) return
+				const contentPreviewChunks = chunk(
+					entities.decodeHTML5Strict( // Remove entities like &amp; &quot;
+						quoted.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
+							.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+							.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+							.replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting.
+					), 50)
+				replyLine = "> " + contentPreviewChunks[0]
+				if (contentPreviewChunks.length > 1) replyLine = replyLine.replace(/[,.']$/, "") + "..."
+				replyLine += "\n"
+				return
+			}
+
 			// @ts-ignore
 			const autoEmoji = new Map(select("auto_emoji", ["name", "emoji_id"], {}, "WHERE name = 'L1' OR name = 'L2'").raw().all())
 			replyLine = `<:L1:${autoEmoji.get("L1")}><:L2:${autoEmoji.get("L2")}>`
@@ -403,12 +554,16 @@ async function eventToMessage(event, guild, di) {
 				replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} `
 			}
 			const sender = repliedToEvent.sender
-			const authorID = select("sim", "user_id", {mxid: repliedToEvent.sender}).pluck().get()
+			const authorID = getUserOrProxyOwnerID(sender)
 			if (authorID) {
 				replyLine += `<@${authorID}>`
 			} else {
-				let senderName = select("member_cache", "displayname", {mxid: repliedToEvent.sender}).pluck().get()
-				if (!senderName) senderName = sender.match(/@([^:]*)/)?.[1] || sender
+				let senderName = select("member_cache", "displayname", {mxid: sender}).pluck().get()
+				if (!senderName) {
+					const match = sender.match(/@([^:]*)/)
+					assert(match)
+					senderName = match[1]
+				}
 				replyLine += `Ⓜ️**${senderName}**`
 			}
 			// If the event has been edited, the homeserver will include the relation in `unsigned`.
@@ -417,26 +572,34 @@ async function eventToMessage(event, guild, di) {
 				repliedToEvent.content = repliedToEvent.content["m.new_content"]
 			}
 			let contentPreview
-			const fileReplyContentAlternative =
-				( repliedToEvent.content.msgtype === "m.image" ? "🖼️"
-				: repliedToEvent.content.msgtype === "m.video" ? "🎞️"
-				: repliedToEvent.content.msgtype === "m.audio" ? "🎶"
-				: repliedToEvent.content.msgtype === "m.file" ? "📄"
-				: null)
+			const fileReplyContentAlternative = attachmentEmojis.get(repliedToEvent.content.msgtype)
 			if (fileReplyContentAlternative) {
 				contentPreview = " " + fileReplyContentAlternative
+			} else if (repliedToEvent.unsigned?.redacted_because) {
+				contentPreview = " (in reply to a deleted message)"
 			} else {
-				const repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body
-				const contentPreviewChunks = chunk(
-					entities.decodeHTML5Strict( // Remove entities like &amp; &quot;
-						repliedToContent.replace(/.*<\/mx-reply>/s, "") // Remove everything before replies, so just use the actual message body
-						.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
-						.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
-						.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
-						.replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting.
-					), 50)
-				contentPreview = ":\n> " + contentPreviewChunks[0]
-				if (contentPreviewChunks.length > 1) contentPreview = contentPreview.replace(/[,.']$/, "") + "..."
+				// Generate a reply preview for a standard message
+				/** @type {string} */
+				let repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body
+				repliedToContent = repliedToContent.replace(/.*<\/mx-reply>/s, "") // Remove everything before replies, so just use the actual message body
+				repliedToContent = repliedToContent.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
+				repliedToContent = repliedToContent.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+				repliedToContent = repliedToContent.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+				repliedToContent = repliedToContent.replace(/<img([^>]*)>/g, (_, att) => { // Convert Matrix emoji images into Discord emoji markdown
+					const mxcUrlMatch = att.match(/\bsrc="(mxc:\/\/[^"]+)"/)
+					const titleTextMatch = att.match(/\btitle=":?([^:"]+)/)
+					return convertEmoji(mxcUrlMatch?.[1], titleTextMatch?.[1], false, false)
+				})
+				repliedToContent = repliedToContent.replace(/<[^:>][^>]*>/g, "") // Completely strip all HTML tags and formatting.
+				repliedToContent = entities.decodeHTML5Strict(repliedToContent) // Remove entities like &amp; &quot;
+				const contentPreviewChunks = chunk(repliedToContent, 50)
+				if (contentPreviewChunks.length) {
+					contentPreview = ":\n> " + contentPreviewChunks[0]
+					if (contentPreviewChunks.length > 1) contentPreview = contentPreview.replace(/[,.']$/, "") + "..."
+				} else {
+					console.log("Unable to generate reply preview for this replied-to event because we stripped all of it:", repliedToEvent)
+					contentPreview = ""
+				}
 			}
 			replyLine = `> ${replyLine}${contentPreview}\n`
 		})()
@@ -452,9 +615,9 @@ async function eventToMessage(event, guild, di) {
 				mxid = decodeURIComponent(mxid)
 				if (mxUtils.eventSenderIsFromDiscord(mxid)) {
 					// Handle mention of an OOYE sim user by their mxid
-					const userID = select("sim", "user_id", {mxid: mxid}).pluck().get()
-					if (!userID) return whole
-					return `${attributeValue} data-user-id="${userID}">`
+					const id = select("sim", "user_id", {mxid}).pluck().get()
+					if (!id) return whole
+					return `${attributeValue} data-user-id="${id}">`
 				} else {
 					// Handle mention of a Matrix user by their mxid
 					// Check if this Matrix user is actually the sim user from another old bridge in the room?
@@ -498,6 +661,52 @@ async function eventToMessage(event, guild, di) {
 			// input = input.replace(/ /g, "&nbsp;")
 			// There is also a corresponding test to uncomment, named "event2message: whitespace is retained"
 
+			// Handling written @mentions: we need to look for candidate Discord members to join to the room
+			// This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here.
+			// We're using the domino parser because Turndown uses the same and can reuse this tree.
+			const doc = domino.createDocument(
+				// DOM parsers arrange elements in the <head> and <body>. Wrapping in a custom element ensures elements are reliably arranged in a single element.
+				'<x-turndown id="turndown-root">' + input + '</x-turndown>'
+			);
+			const root = doc.getElementById("turndown-root");
+			async function forEachNode(node) {
+				for (; node; node = node.nextSibling) {
+					// Check written mentions
+					if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) {
+						const result = await checkWrittenMentions(node.nodeValue, event.sender, event.room_id, guild, di)
+						if (result) {
+							node.nodeValue = result.content
+							ensureJoined.push(...result.ensureJoined)
+							allowedMentionsParse.push(...result.allowedMentionsParse)
+						}
+					}
+					// Check for incompatible backticks in code blocks
+					let preNode
+					if (node.nodeType === 3 && node.nodeValue.includes("```") && (preNode = nodeIsChildOf(node, ["PRE"]))) {
+						if (preNode.firstChild?.nodeName === "CODE") {
+							const ext = (preNode.firstChild.className.match(/language-(\S+)/) || [null, "txt"])[1]
+							const filename = `inline_code.${ext}`
+							// Build the replacement <code> node
+							const replacementCode = doc.createElement("code")
+							replacementCode.textContent = `[${filename}]`
+							// Build its containing <span> node
+							const replacement = doc.createElement("span")
+							replacement.appendChild(doc.createTextNode(" "))
+							replacement.appendChild(replacementCode)
+							replacement.appendChild(doc.createTextNode(" "))
+							// Replace the code block with the <span>
+							preNode.replaceWith(replacement)
+							// Upload the code as an attachment
+							const content = getCodeContent(preNode.firstChild)
+							attachments.push({id: String(attachments.length), filename})
+							pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")})
+						}
+					}
+					await forEachNode(node.firstChild)
+				}
+			}
+			await forEachNode(root)
+
 			// SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet.
 			// First we need to determine which emojis are at the end.
 			endOfMessageEmojis = []
@@ -507,18 +716,24 @@ async function eventToMessage(event, guild, di) {
 				if (!match[0].includes("data-mx-emoticon")) break
 				const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/)
 				if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1])
-				if (typeof match.index !== "number") break
+				assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec")
 				last = match.index
 			}
 
 			// @ts-ignore bad type from turndown
-			content = turndownService.turndown(input)
+			content = turndownService.turndown(root)
+
+			// Put < > around any surviving matrix.to links to hide the URL previews
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^ )]*/g, "<$&>")
 
 			// It's designed for commonmark, we need to replace the space-space-newline with just newline
 			content = content.replace(/  \n/g, "\n")
 
+			// If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated
+			if (replyLine && content.startsWith("> ")) content = "\n" + content
+
 			// SPRITE SHEET EMOJIS FEATURE:
-			content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
+			content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader)
 		} else {
 			// Looks like we're using the plaintext body!
 			content = event.content.body
@@ -527,7 +742,15 @@ async function eventToMessage(event, guild, di) {
 				content = `* ${displayName} ${content}`
 			}
 
-			content = await handleRoomOrMessageLinks(content, di)
+			content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^ )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews
+
+			const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di)
+			if (result) {
+				content = result.content
+				ensureJoined.push(...result.ensureJoined)
+				allowedMentionsParse.push(...result.allowedMentionsParse)
+			}
 
 			// Markdown needs to be escaped, though take care not to escape the middle of links
 			// @ts-ignore bad type from turndown
@@ -563,8 +786,11 @@ async function eventToMessage(event, guild, di) {
 			if (event.content.info?.mimetype?.includes("/")) {
 				mimetype = event.content.info.mimetype
 			} else {
-				const res = await fetch(url, {method: "HEAD"})
-				mimetype = res.headers.get("content-type") || "image/webp"
+				const res = await di.fetch(url, {method: "HEAD"})
+				if (res.status === 200) {
+					mimetype = res.headers.get("content-type")
+				}
+				if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`)
 			}
 			filename += "." + mimetype.split("/")[1]
 		}
@@ -574,25 +800,17 @@ async function eventToMessage(event, guild, di) {
 
 	content = displayNameRunoff + replyLine + content
 
-	// Handling written @mentions: we need to look for candidate Discord members to join to the room
-	let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+
-	if (writtenMentionMatch) {
-		const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]})
-		if (results[0]) {
-			assert(results[0].user)
-			// @ts-ignore - typescript doesn't know about indices yet
-			content = content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1])
-			ensureJoined.push(results[0].user)
-		}
-	}
-
 	// Split into 2000 character chunks
 	const chunks = chunk(content, 2000)
-	messages = messages.concat(chunks.map(content => ({
+	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
+	const messages = chunks.map(content => ({
 		content,
+		allowed_mentions: {
+			parse: allowedMentionsParse
+		},
 		username: displayNameShortened,
 		avatar_url: avatarURL
-	})))
+	}))
 
 	if (attachments.length) {
 		// If content is empty (should be the case when uploading a file) then chunk-text will create 0 messages.

m2d/converters/utils.js

@@ -72,7 +72,7 @@ class MatrixStringBuilder {
 
 	/**
 	 * @param {string} body
-	 * @param {string} formattedBody
+	 * @param {string} [formattedBody]
 	 * @param {any} [condition]
 	 */
 	add(body, formattedBody, condition = true) {
@@ -127,8 +127,96 @@ class MatrixStringBuilder {
 	}
 }
 
+/**
+ * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers.
+ * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation.
+ * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged.
+ * https://spec.matrix.org/v1.9/appendices/#routing
+ * https://gitdab.com/cadence/out-of-your-element/issues/11
+ * @param {string} roomID
+ * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api
+ */
+async function getViaServers(roomID, api) {
+	const candidates = []
+	const {joined} = await api.getJoinedMembers(roomID)
+	// Candidate 0: The bot's own server name
+	candidates.push(reg.ooye.server_name)
+	// Candidate 1: Highest joined non-sim non-bot power level user in the room
+	// https://github.com/matrix-org/matrix-react-sdk/blob/552c65db98b59406fb49562e537a2721c8505517/src/utils/permalinks/Permalinks.ts#L172
+	try {
+		/** @type {{users?: {[mxid: string]: number}}} */
+		const powerLevels = await api.getStateEvent(roomID, "m.room.power_levels", "")
+		if (powerLevels.users) {
+			const sorted = Object.entries(powerLevels.users).sort((a, b) => b[1] - a[1]) // Highest...
+			for (const power of sorted) {
+				const mxid = power[0]
+				if (!(mxid in joined)) continue // joined...
+				if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot...
+				const match = mxid.match(/:(.*)/)
+				assert(match)
+				if (!candidates.includes(match[1])) {
+					candidates.push(match[1])
+					break
+				}
+			}
+		}
+	} catch (e) {
+		// power levels event not found
+	}
+	// Candidates 2-3: Most popular servers in the room
+	/** @type {Map<string, number>} */
+	const servers = new Map()
+	// We can get the most popular servers if we know the members, so let's process those...
+	Object.keys(joined)
+		.filter(mxid => !mxid.startsWith("@_")) // Quick check
+		.filter(mxid => !userRegex.some(r => mxid.match(r))) // Full check
+		.slice(0, 1000) // Just sample the first thousand real members
+		.map(mxid => {
+			const match = mxid.match(/:(.*)/)
+			assert(match)
+			return match[1]
+		})
+		.filter(server => !server.match(/([a-f0-9:]+:+)+[a-f0-9]+/)) // No IPv6 servers
+		.filter(server => !server.match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)) // No IPv4 servers
+		// I don't care enough to check ACLs
+		.forEach(server => {
+			const existing = servers.get(server)
+			if (!existing) servers.set(server, 1)
+			else servers.set(server, existing + 1)
+		})
+	const serverList = [...servers.entries()].sort((a, b) => b[1] - a[1])
+	for (const server of serverList) {
+		if (!candidates.includes(server[0])) {
+			candidates.push(server[0])
+			if (candidates.length >= 4) break // Can have at most 4 candidate via servers
+		}
+	}
+	return candidates
+}
+
+/**
+ * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers.
+ * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation.
+ * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged.
+ * https://spec.matrix.org/v1.9/appendices/#routing
+ * https://gitdab.com/cadence/out-of-your-element/issues/11
+ * @param {string} roomID
+ * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api
+ * @returns {Promise<URLSearchParams>}
+ */
+async function getViaServersQuery(roomID, api) {
+	const list = await getViaServers(roomID, api)
+	const qs = new URLSearchParams()
+	for (const server of list) {
+		qs.append("via", server)
+	}
+	return qs
+}
+
 module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS
 module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord
 module.exports.getPublicUrlForMxc = getPublicUrlForMxc
 module.exports.getEventIDHash = getEventIDHash
 module.exports.MatrixStringBuilder = MatrixStringBuilder
+module.exports.getViaServers = getViaServers
+module.exports.getViaServersQuery = getViaServersQuery

m2d/converters/utils.test.js

@@ -1,7 +1,23 @@
 // @ts-check
 
+const e = new Error("Custom error")
+
 const {test} = require("supertape")
-const {eventSenderIsFromDiscord, getEventIDHash} = require("./utils")
+const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers} = require("./utils")
+const util = require("util")
+
+/** @param {string[]} mxids */
+function joinedList(mxids) {
+	/** @type {{[mxid: string]: {display_name: null, avatar_url: null}}} */
+	const joined = {}
+	for (const mxid of mxids) {
+		joined[mxid] = {
+			display_name: null,
+			avatar_url: null
+		}
+	}
+	return {joined}
+}
 
 test("sender type: matrix user", t => {
 	t.notOk(eventSenderIsFromDiscord("@cadence:cadence.moe"))
@@ -23,3 +39,140 @@ test("event hash: hash is the same each time", t => {
 test("event hash: hash is different for different inputs", t => {
 	t.notEqual(getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe1"), getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe2"))
 })
+
+test("MatrixStringBuilder: add, addLine, add same text", t => {
+	const gatewayMessage = {t: "MY_MESSAGE", d: {display: "Custom message data"}}
+	let stackLines = e.stack?.split("\n")
+
+	const builder = new MatrixStringBuilder()
+	builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 <strong>Bridged event from Discord not delivered</strong>")
+	builder.addLine(`Gateway event: ${gatewayMessage.t}`)
+	builder.addLine(e.toString())
+	if (stackLines) {
+		stackLines = stackLines.slice(0, 2)
+		stackLines[1] = stackLines[1].replace(/\\/g, "/").replace(/(\s*at ).*(\/m2d\/)/, "$1.$2")
+		builder.addLine(`Error trace:`, `<details><summary>Error trace</summary>`)
+		builder.add(`\n${stackLines.join("\n")}`, `<pre>${stackLines.join("\n")}</pre></details>`)
+	}
+	builder.addLine("", `<details><summary>Original payload</summary><pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`)
+
+	t.deepEqual(builder.get(), {
+		msgtype: "m.text",
+		body: "\u26a0 Bridged event from Discord not delivered"
+			+ "\nGateway event: MY_MESSAGE"
+			+ "\nError: Custom error"
+			+ "\nError trace:"
+			+ "\nError: Custom error"
+			+ "\n    at ./m2d/converters/utils.test.js:3:11)\n",
+		format: "org.matrix.custom.html",
+		formatted_body: "\u26a0 <strong>Bridged event from Discord not delivered</strong>"
+			+ "<br>Gateway event: MY_MESSAGE"
+			+ "<br>Error: Custom error"
+			+ "<br><details><summary>Error trace</summary><pre>Error: Custom error\n    at ./m2d/converters/utils.test.js:3:11)</pre></details>"
+			+ `<details><summary>Original payload</summary><pre>{ display: 'Custom message data' }</pre></details>`
+	})
+})
+
+test("MatrixStringBuilder: complete code coverage", t => {
+	const builder = new MatrixStringBuilder()
+	builder.add("Line 1")
+	builder.addParagraph("Line 2")
+	builder.add("Line 3")
+	builder.addParagraph("Line 4")
+
+	t.deepEqual(builder.get(), {
+		msgtype: "m.text",
+		body: "Line 1\n\nLine 2Line 3\n\nLine 4",
+		format: "org.matrix.custom.html",
+		formatted_body: "Line 1<p>Line 2</p>Line 3<p>Line 4</p>"
+	})
+})
+
+test("getViaServers: returns the server name if the room only has sim users", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe"])
+	})
+	t.deepEqual(result, ["cadence.moe"])
+})
+
+test("getViaServers: also returns the most popular servers in order", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "thecollective.invalid", "selfhosted.invalid"])
+})
+
+test("getViaServers: does not return IP address servers", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:45.77.232.172:8443", "@cadence:[::1]:8443", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "123example.456example.invalid"])
+})
+
+test("getViaServers: also returns the highest power level user (100)", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@singleuser:selfhosted.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@moderator:tractor.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "selfhosted.invalid", "thecollective.invalid", "tractor.invalid"])
+})
+
+test("getViaServers: also returns the highest power level user (50)", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"])
+})
+
+test("getViaServers: returns at most 4 results", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@singleuser:selfhosted.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result.length, 4)
+})
+
+test("getViaServers: returns results even when power levels can't be fetched", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => {
+			throw new Error("event not found or something")
+		},
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result.length, 4)
+})
+
+test("getViaServers: only considers power levels of currently joined members", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@former_moderator:missing.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"])
+})

matrix/api.js

@@ -115,12 +115,25 @@ function getStateEvent(roomID, type, key) {
 /**
  * "Any of the AS's users must be in the room. This API is primarily for Application Services and should be faster to respond than /members as it can be implemented more efficiently on the server."
  * @param {string} roomID
- * @returns {Promise<{joined: {[mxid: string]: {avatar_url?: string, display_name?: string}}}>}
+ * @returns {Promise<{joined: {[mxid: string]: {avatar_url: string?, display_name: string?}}}>}
  */
 function getJoinedMembers(roomID) {
 	return mreq.mreq("GET", `/client/v3/rooms/${roomID}/joined_members`)
 }
 
+/**
+ * @param {string} roomID
+ * @param {{from?: string, limit?: any}} pagination
+ * @returns {Promise<Ty.HierarchyPagination<Ty.R.Hierarchy>>}
+ */
+function getHierarchy(roomID, pagination) {
+	let path = `/client/v1/rooms/${roomID}/hierarchy`
+	if (!pagination.from) delete pagination.from
+	if (!pagination.limit) pagination.limit = 50
+	path += `?${new URLSearchParams(pagination)}`
+	return mreq.mreq("GET", path)
+}
+
 /**
  * @param {string} roomID
  * @param {string} eventID
@@ -239,6 +252,7 @@ module.exports.getEventForTimestamp = getEventForTimestamp
 module.exports.getAllState = getAllState
 module.exports.getStateEvent = getStateEvent
 module.exports.getJoinedMembers = getJoinedMembers
+module.exports.getHierarchy = getHierarchy
 module.exports.getRelations = getRelations
 module.exports.sendState = sendState
 module.exports.sendEvent = sendEvent

matrix/kstate.js

@@ -2,7 +2,7 @@
 
 const assert = require("assert").strict
 const mixin = require("mixin-deep")
-const deepEqual = require("deep-equal")
+const {isDeepStrictEqual} = require("util")
 
 /** Mutates the input. */
 function kstateStripConditionals(kstate) {
@@ -51,14 +51,14 @@ function diffKState(actual, target) {
 			// Special handling for power levels, we want to deep merge the actual and target into the final state.
 			if (!(key in actual)) throw new Error(`want to apply a power levels diff, but original power level data is missing\nstarted with:  ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`)
 			const temp = mixin({}, actual[key], target[key])
-			if (!deepEqual(actual[key], temp, {strict: true})) {
+			if (!isDeepStrictEqual(actual[key], temp)) {
 				// they differ. use the newly prepared object as the diff.
 				diff[key] = temp
 			}
 
 		} else if (key in actual) {
 			// diff
-			if (!deepEqual(actual[key], target[key], {strict: true})) {
+			if (!isDeepStrictEqual(actual[key], target[key])) {
 				// they differ. use the target as the diff.
 				diff[key] = target[key]
 			}

matrix/kstate.test.js

@@ -1,3 +1,4 @@
+const assert = require("assert")
 const {kstateToState, stateToKState, diffKState, kstateStripConditionals} = require("./kstate")
 const {test} = require("supertape")
 
@@ -162,3 +163,29 @@ test("diffKState: power levels are mixed together", t => {
 	})
 	t.notDeepEqual(original, result)
 })
+
+test("diffKState: cannot merge power levels if original power levels are missing", t => {
+	const original = {}
+	assert.throws(() =>
+		diffKState(original, {
+			"m.room.power_levels/": {
+				"events": {
+					"m.room.avatar": 0
+				}
+			}
+		})
+	, /original power level data is missing/)
+	t.pass()
+})
+
+test("diffKState: kstate keys must contain a slash separator", t => {
+	assert.throws(() =>
+		diffKState({
+			"m.room.name/": {name: "test name"},
+		}, {
+			"m.room.name/": {name: "test name"},
+			"new": {a: 2}
+		})
+	, /does not contain a slash separator/)
+	t.pass()
+})

package.json

@@ -18,12 +18,11 @@
     "@chriscdn/promise-semaphore": "^2.0.1",
     "better-sqlite3": "^9.0.0",
     "chunk-text": "^2.0.1",
-    "cloudstorm": "^0.9.5",
-    "deep-equal": "^2.2.3",
-    "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8",
+    "cloudstorm": "^0.10.8",
+    "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#2881b447954fcea10510f212fa4c1dbbdc0a57a3",
     "entities": "^4.5.0",
     "get-stream": "^6.0.1",
-    "giframe": "github:cloudrac3r/giframe#v0.4.1",
+    "giframe": "github:cloudrac3r/giframe#v0.4.2",
     "heatsync": "^2.4.1",
     "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0",
     "js-yaml": "^4.1.0",
@@ -31,10 +30,10 @@
     "minimist": "^1.2.8",
     "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
     "node-fetch": "^2.6.7",
-    "pngjs": "^7.0.0",
+    "pngjs": "github:cloudrac3r/pngjs#v7.0.2",
     "prettier-bytes": "^1.0.4",
     "sharp": "^0.32.6",
-    "snowtransfer": "^0.9.0",
+    "snowtransfer": "^0.10.5",
     "stream-mime-type": "^1.0.2",
     "try-to-catch": "^3.0.1",
     "turndown": "^7.1.2",
@@ -44,15 +43,16 @@
     "@types/node": "^18.16.0",
     "@types/node-fetch": "^2.6.3",
     "c8": "^8.0.1",
+    "colorette": "^1.4.0",
     "cross-env": "^7.0.3",
     "discord-api-types": "^0.37.60",
-    "supertape": "^8.3.0",
+    "supertape": "^10.4.0",
     "tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d"
   },
   "scripts": {
     "addbot": "node addbot.js",
     "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap test/test.js | tap-dot",
-    "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap test/test.js -- --slow | tap-dot",
-    "cover": "c8 --skip-full -x db/migrations -x matrix/file.js -x matrix/api.js -x matrix/mreq.js -r html -r text supertape --no-check-assertions-count --format fail test/test.js -- --slow"
+    "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot",
+    "cover": "c8 --skip-full -x db/migrations -x matrix/file.js -x matrix/api.js -x matrix/mreq.js -x d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow"
   }
 }

readme.md

@@ -2,9 +2,9 @@
 
 <img src="docs/img/icon.png" height="128" width="128">
 
-Modern Matrix-to-Discord appservice bridge.
+Modern Matrix-to-Discord appservice bridge, created by [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe)
 
-Created by [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) // Discuss in [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe)
+[![Releases](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=plastic&color=green)](https://gitdab.com/cadence/out-of-your-element/releases) [![Discuss on Matrix](https://img.shields.io/badge/discuss-%23out--of--your--element-white?style=plastic)](https://matrix.to/#/#out-of-your-element:cadence.moe)
 
 ## Docs
 
@@ -76,7 +76,8 @@ Follow these steps:
 
 1. [Get Node.js version 18 or later](https://nodejs.org/en/download/releases) (the version is required by the better-sqlite3 and matrix-appservice dependencies)
 
-1. Clone this repo and checkout a specific tag. (Development happens on main. Stabler versions are tagged.)
+1. Clone this repo and checkout a specific tag. (Development happens on main. Stable versions are tagged.)
+	* The latest release tag is ![](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=flat-square&label=%20&color=black).
 
 1. Install dependencies: `npm install --save-dev` (omit --save-dev if you will not run the automated tests)
 
@@ -166,7 +167,6 @@ To get into the rooms on your Matrix account, either add yourself to `invite` in
 * (1) chunk-text: It does what I want.
 * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
 * (8) snowtransfer: Discord API library with bring-your-own-caching that I trust.
-* (0) deep-equal: It's already pulled in by supertape.
 * (1) discord-markdown: This is my fork!
 * (0) get-stream: Only needed if content_length_workaround is true.
 * (0) giframe: This is my fork!

scripts/capture-message-update-events.js

@@ -37,7 +37,7 @@ passthrough.discord = discord
 })()
 
 const events = new sqlite("scripts/events.db")
-const sql = "INSERT INTO \"update\" (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")"
+const sql = "INSERT INTO update_event (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")"
 console.log(sql)
 const prepared = events.prepare(sql)
 

scripts/seed.js

@@ -53,17 +53,19 @@ async function uploadAutoEmoji(guild, name, filename) {
 	const mxid = `@${reg.sender_localpart}:${reg.ooye.server_name}`
 
 	// ensure registration is correctly set...
-	assert(reg.sender_localpart.startsWith(reg.ooye.namespace_prefix)) // appservice's localpart must be in the namespace it controls
-	assert(utils.eventSenderIsFromDiscord(mxid)) // appservice's mxid must be in the namespace it controls
-	assert(reg.ooye.server_origin.match(/^https?:\/\//)) // must start with http or https
-	assert.notEqual(reg.ooye.server_origin.slice(-1), "/") // must not end in slash
+	assert(reg.sender_localpart.startsWith(reg.ooye.namespace_prefix), "appservice's localpart must be in the namespace it controls")
+	assert(utils.eventSenderIsFromDiscord(mxid), "appservice's mxid must be in the namespace it controls")
+	assert(reg.ooye.server_origin.match(/^https?:\/\//), "server origin must start with http or https")
+	assert.notEqual(reg.ooye.server_origin.slice(-1), "/", "server origin must not end in slash")
+	const botID = Buffer.from(config.discordToken.split(".")[0], "base64").toString()
+	assert(botID.match(/^[0-9]{10,}$/), "discord token must follow the correct format")
 	console.log("✅ Configuration looks good...")
 
 	// database ddl...
 	await migrate.migrate(db)
 
 	// add initial rows to database, like adding the bot to sim...
-	db.prepare("INSERT OR IGNORE INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run("0", reg.sender_localpart.slice(reg.ooye.namespace_prefix.length), reg.sender_localpart, mxid)
+	db.prepare("INSERT OR IGNORE INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(botID, reg.sender_localpart.slice(reg.ooye.namespace_prefix.length), reg.sender_localpart, mxid)
 
 	console.log("✅ Database is ready...")
 

stdin.js

@@ -17,6 +17,7 @@ const file = sync.require("./matrix/file")
 const sendEvent = sync.require("./m2d/actions/send-event")
 const eventDispatcher = sync.require("./d2m/event-dispatcher")
 const updatePins = sync.require("./d2m/actions/update-pins")
+const speedbump = sync.require("./d2m/actions/speedbump")
 const ks = sync.require("./matrix/kstate")
 const guildID = "112760669178241024"
 

test/ooye-test-data.sql

@@ -11,7 +11,9 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom
 ('1162005314908999790', '!FuDZhlOAtqswlyxzeR:cadence.moe', 'Hey.', NULL, '1100319550446252084', NULL),
 ('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL),
 ('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL),
-('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS');
+('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS'),
+('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL),
+('1160894080998461480', '!TqlyQmifxGUggEmdBN:cadence.moe', 'ooyexperiment', NULL, NULL, NULL);
 
 INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES
 ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'),
@@ -20,13 +22,19 @@ INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES
 ('112890272819507200', '.wing.', '_ooye_.wing.', '@_ooye_.wing.:cadence.moe'),
 ('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'),
 ('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe'),
-('1109360903096369153', 'amanda', '_ooye_amanda', '@_ooye_amanda:cadence.moe');
+('1109360903096369153', 'amanda', '_ooye_amanda', '@_ooye_amanda:cadence.moe'),
+('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '_pk_zoego', '_ooye__pk_zoego', '@_ooye__pk_zoego:cadence.moe');
+
+INSERT INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES
+('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '196188877885538304', 'Azalea &flwr; 🌺');
 
 INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES
 ('@_ooye_bojack_horseman:cadence.moe', '!hYnGGlPHlbujVVfktC:cadence.moe', NULL);
 
 INSERT INTO message_channel (message_id, channel_id) VALUES
 ('1106366167788044450', '122155380120748034'),
+('1106366167788044451', '122155380120748034'),
+('1106366167788044452', '122155380120748034'),
 ('1126786462646550579', '112760669178241024'),
 ('1128084748338741392', '112760669178241024'),
 ('1128084851279536279', '112760669178241024'),
@@ -42,7 +50,11 @@ INSERT INTO message_channel (message_id, channel_id) VALUES
 ('1145688633186193481', '1100319550446252084'),
 ('1162005526675193909', '1162005314908999790'),
 ('1162625810109317170', '497161350934560778'),
-('1158842413025071135', '176333891320283136');
+('1158842413025071135', '176333891320283136'),
+('1197612733600895076', '112760669178241024'),
+('1202543413652881428', '1160894080998461480'),
+('1207486471489986620', '1160894080998461480'),
+('1210387798297682020', '112760669178241024');
 
 INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES
 ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1),
@@ -59,6 +71,12 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
 ('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1, 1),
 ('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 1, 0),
 ('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0, 0),
+('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999', 'm.room.message', 'm.text', '1106366167788044451', 0, 0, 1),
+('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZI999', 'm.room.message', 'm.image', '1106366167788044451', 0, 0, 1),
+('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd999', 'm.sticker', NULL, '1106366167788044451', 0, 0, 1),
+('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111', 'm.room.message', 'm.text', '1106366167788044452', 1, 1, 1),
+('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZI111', 'm.room.message', 'm.image', '1106366167788044452', 1, 1, 1),
+('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd111', 'm.sticker', NULL, '1106366167788044452', 1, 1, 1),
 ('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 0, 1),
 ('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0, 0),
 ('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0, 0),
@@ -67,7 +85,11 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
 ('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0, 0),
 ('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1, 1),
 ('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 0, 1),
-('$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo', 'm.room.message', 'm.text', '1158842413025071135', 0, 0, 1);
+('$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo', 'm.room.message', 'm.text', '1158842413025071135', 0, 0, 1),
+('$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw', 'm.room.message', 'm.text', '1197612733600895076', 0, 0, 1),
+('$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU', 'm.room.message', 'm.text', '1202543413652881428', 0, 0, 0),
+('$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y', 'm.room.message', 'm.image', '1207486471489986620', 0, 0, 0),
+('$mPSzglkCu-6cZHbYro0RW2u5mHvbH9aXDjO5FCzosc0', 'm.room.message', 'm.text', '1210387798297682020', 0, 0, 1);
 
 INSERT INTO file (discord_url, mxc_url) VALUES
 ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),
@@ -84,11 +106,15 @@ INSERT INTO file (discord_url, mxc_url) VALUES
 ('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'),
 ('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'),
 ('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX'),
-('https://cdn.discordapp.com/emojis/288858540888686602.png', 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO');
+('https://cdn.discordapp.com/emojis/288858540888686602.png', 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO'),
+('https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4', 'mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU'),
+('https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg', 'mxc://cadence.moe/MRRPDggXQMYkrUjTpxQbmcxB'),
+('https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml', 'mxc://cadence.moe/HnQIYQmmlIKwOQsbFsIGpzPP');
 
 INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES
 ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'),
 ('393635038903926784', 'hipposcope', 1, 'mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc'),
+('457898385297815911', 'emoji_from_unreachable_server', 0, 'mxc://cadence.moe/bZFuuUSEebJYXUMSxuuSuLTa'),
 ('362741439211503616', 'bn_re', 0, 'mxc://cadence.moe/OIpqpfxTnHKokcsYqDusxkBT'),
 ('551636841284108289', 'ae_botrac4r', 0, 'mxc://cadence.moe/skqfuItqxNmBYekzmVKyoLzs'),
 ('975572106295259148', 'brillillillilliant_move', 0, 'mxc://cadence.moe/scfRIDOGKWFDEBjVXocWYQHik'),
@@ -101,7 +127,14 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES
 ('!fGgIymcYWOqjbSRUdV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'),
 ('!BnKuBPCvyfOkhcUjEu:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'),
 ('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'),
-('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU');
+('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'),
+('!cBxtVRxDlZvSVhJXVK:cadence.moe', '@Milan:tchncs.de', 'Milan', NULL),
+('!TqlyQmifxGUggEmdBN:cadence.moe', '@ampflower:matrix.org', 'Ampflower 🌺', 'mxc://cadence.moe/PRfhXYBTOalvgQYtmCLeUXko'),
+('!TqlyQmifxGUggEmdBN:cadence.moe', '@aflower:syndicated.gay', 'Rose', 'mxc://syndicated.gay/ZkBUPXCiXTjdJvONpLJmcbKP'),
+('!TqlyQmifxGUggEmdBN:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', NULL);
+
+INSERT INTO lottie (sticker_id, mxc_url) VALUES
+('860171525772279849', 'mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR');
 
 INSERT INTO "auto_emoji" ("name","emoji_id","guild_id") VALUES
 ('L1','1144820033948762203','529176156398682115'),

test/res/lottie-bee.license.txt

@@ -0,0 +1,16 @@
+# Source
+
+Flying Bee by Afif Ridwan
+https://lottiefiles.com/animations/flying-bee-WkXvUiWkZ1
+
+# License
+
+Lottie Simple License (FL 9.13.21) https://lottiefiles.com/page/license
+
+Copyright © 2021 Design Barn Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of the public animation files available for download at the LottieFiles site (“Files”) to download, reproduce, modify, publish, distribute, publicly display, and publicly digitally perform such Files, including for commercial purposes, provided that any display, publication, performance, or distribution of Files must contain (and be subject to) the same terms and conditions of this license. Modifications to Files are deemed derivative works and must also be expressly distributed under the same terms and conditions of this license. You may not purport to impose any additional or different terms or conditions on, or apply any technical measures that restrict exercise of, the rights granted under this license. This license does not include the right to collect or compile Files from LottieFiles to replicate or develop a similar or competing service.
+
+Use of Files without attributing the creator(s) of the Files is permitted under this license, though attribution is strongly encouraged. If attributions are included, such attributions should be visible to the end user.
+
+FILES ARE PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL THE CREATOR(S) OF FILES OR DESIGN BARN, INC. BE LIABLE ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE, OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF SUCH FILES.

test/test.js

@@ -2,11 +2,16 @@
 
 const fs = require("fs")
 const {join} = require("path")
+const stp = require("stream").promises
 const sqlite = require("better-sqlite3")
 const migrate = require("../db/migrate")
 const HeatSync = require("heatsync")
 const {test} = require("supertape")
 const data = require("./data")
+/** @type {import("node-fetch").default} */
+// @ts-ignore
+const fetch = require("node-fetch")
+const {green} = require("colorette")
 
 const config = require("../config")
 const passthrough = require("../passthrough")
@@ -29,6 +34,10 @@ const discord = {
 	channels: new Map([
 		["497161350934560778", {
 			guild_id: "497159726455455754"
+		}],
+		["498323546729086986", {
+			guild_id: "497159726455455754",
+			name: "bad-boots-prison"
 		}]
 	])
 }
@@ -40,16 +49,66 @@ passthrough.from = orm.from
 passthrough.select = orm.select
 
 const file = sync.require("../matrix/file")
+/* c8 ignore next */
 file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) }
 
 ;(async () => {
+	/* c8 ignore start - maybe download some more test files in slow mode */
+	if (process.argv.includes("--slow")) {
+		test("test files: download", async t => {
+			/** @param {{url: string, to: string}[]} files */
+			async function allReporter(files) {
+				return new Promise(resolve => {
+					let resolved = 0
+					const report = files.map(file => file.to.split("/").slice(-1)[0][0])
+					files.map(download).forEach((p, i) => {
+						p.then(() => {
+							report[i] = green(".")
+							process.stderr.write("\r" + report.join(""))
+							if (++resolved === files.length) resolve(null)
+						})
+					})
+				})
+			}
+			async function download({url, to}) {
+				if (await fs.existsSync(to)) return
+				const res = await fetch(url)
+				await stp.pipeline(res.body, fs.createWriteStream(to, {encoding: "binary"}))
+			}
+			await allReporter([
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/RLMgJGfgTPjIQtvvWZsYjhjy", to: "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/bZFuuUSEebJYXUMSxuuSuLTa", to: "test/res/bZFuuUSEebJYXUMSxuuSuLTa.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/qWmbXeRspZRLPcjseyLmeyXC", to: "test/res/qWmbXeRspZRLPcjseyLmeyXC.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/wcouHVjbKJJYajkhJLsyeJAA", to: "test/res/wcouHVjbKJJYajkhJLsyeJAA.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/WbYqNlACRuicynBfdnPYtmvc", to: "test/res/WbYqNlACRuicynBfdnPYtmvc.gif"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/HYcztccFIPgevDvoaWNsEtGJ", to: "test/res/HYcztccFIPgevDvoaWNsEtGJ.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/lHfmJpzgoNyNtYHdAmBHxXix", to: "test/res/lHfmJpzgoNyNtYHdAmBHxXix.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/MtRdXixoKjKKOyHJGWLsWLNU", to: "test/res/MtRdXixoKjKKOyHJGWLsWLNU.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/HXfFuougamkURPPMflTJRxGc", to: "test/res/HXfFuougamkURPPMflTJRxGc.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ikYKbkhGhMERAuPPbsnQzZiX", to: "test/res/ikYKbkhGhMERAuPPbsnQzZiX.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/AYPpqXzVJvZdzMQJGjioIQBZ", to: "test/res/AYPpqXzVJvZdzMQJGjioIQBZ.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/UVuzvpVUhqjiueMxYXJiFEAj", to: "test/res/UVuzvpVUhqjiueMxYXJiFEAj.png"},
+				{url: "https://ezgif.com/images/format-demo/butterfly.gif", to: "test/res/butterfly.gif"},
+				{url: "https://ezgif.com/images/format-demo/butterfly.png", to: "test/res/butterfly.png"},
+			])
+		}, {timeout: 60000})
+	}
+	/* c8 ignore end */
+
 	const p = migrate.migrate(db)
 	test("migrate: migration works", async t => {
 		await p
 		t.pass("it did not throw an error")
 	})
 	await p
+
+	test("migrate: migration works the second time", async t => {
+		await migrate.migrate(db)
+		t.pass("it did not throw an error")
+	})
+
 	db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8"))
+
 	require("../db/orm.test")
 	require("../discord/utils.test")
 	require("../matrix/kstate.test")
@@ -61,12 +120,15 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
 	require("../d2m/actions/register-user.test")
 	require("../d2m/converters/edit-to-changes.test")
 	require("../d2m/converters/emoji-to-key.test")
+	require("../d2m/converters/lottie.test")
 	require("../d2m/converters/message-to-event.test")
 	require("../d2m/converters/message-to-event.embeds.test")
+	require("../d2m/converters/message-to-event.pk.test")
 	require("../d2m/converters/pins-to-list.test")
 	require("../d2m/converters/remove-reaction.test")
 	require("../d2m/converters/thread-to-announcement.test")
 	require("../d2m/converters/user-to-mxid.test")
 	require("../m2d/converters/event-to-message.test")
 	require("../m2d/converters/utils.test")
+	require("../m2d/converters/emoji-sheet.test")
 })()

types.d.ts

@@ -35,6 +35,45 @@ export type WebhookCreds = {
 	token: string
 }
 
+export type PkSystem = {
+	id: string
+	uuid: string
+	name: string | null
+	description: string | null
+	tag: string | null
+	pronouns: string | null
+	avatar_url: string | null
+	banner: string | null
+	color: string | null
+	created: string | null
+}
+
+export type PkMember = {
+	id: string
+	uuid: string
+	name: string
+	display_name: string | null
+	color: string | null
+	birthday: string | null
+	pronouns: string | null
+	avatar_url: string | null
+	webhook_avatar_url: string | null
+	banner: string | null
+	description: string | null
+	created: string | null
+	keep_proxy: boolean
+	tts: boolean
+	autoproxy_enabled: boolean | null
+	message_count: number | null
+	last_message_timestamp: string
+}
+
+export type PkMessage = {
+	system: PkSystem
+	member: PkMember
+	sender: string
+}
+
 export namespace Event {
 	export type Outer<T> = {
 		type: string
@@ -218,6 +257,18 @@ export namespace R {
 	export type EventRedacted = {
 		event_id: string
 	}
+
+	export type Hierarchy = {
+		avatar_url?: string
+		canonical_alias?: string
+		children_state: {}
+		guest_can_join: boolean
+		join_rule?: string
+		name?: string
+		num_joined_members: number
+		room_id: string
+		room_type?: string
+	}
 }
 
 export type Pagination<T> = {
@@ -225,3 +276,8 @@ export type Pagination<T> = {
 	next_batch?: string
 	prev_match?: string
 }
+
+export type HierarchyPagination<T> = {
+	rooms: T[]
+	next_batch?: string
+}