Look up thread IDs if channels cannot be found

The sim_member cache was getting stuck, so OOYE thought it was already
in the room when it actually wasn't.

scripts/backfill.js

@@ -51,10 +51,24 @@ const preparedInsert = backfill.prepare("INSERT INTO backfill (channel_id, messa
 
 async function event(event) {
 	if (event.t !== "GUILD_CREATE") return
-	const channel = event.d.channels.find(c => c.id === channelID)
-	if (!channel) return
+	let channel = event.d.channels.find(c => c.id === channelID) || (event.d.threads || []).find(c => c.id === channelID)
 	const guild_id = event.d.id
 
+	if (!channel) {
+		// May be an archived thread not present in GUILD_CREATE data - try fetching via API
+		try {
+			const fetched = await discord.snow.channel.getChannel(channelID)
+			if (!fetched.guild_id || fetched.guild_id !== guild_id) return
+			fetched.guild_id = guild_id
+			discord.channels.set(fetched.id, fetched)
+			channel = fetched
+		} catch (e) {
+			return
+		}
+	}
+
+	if (!channel) return
+
 	try {
 		await createRoom.syncRoom(channelID)
 		let last = backfill.prepare("SELECT cast(max(message_id) as TEXT) FROM backfill WHERE channel_id = ?").pluck().get(channelID) || "0"

scripts/remove-uncached-bridged-users.js

@@ -0,0 +1,36 @@
+// @ts-check
+
+const HeatSync = require("heatsync")
+const sync = new HeatSync({watchFS: false})
+
+const sqlite = require("better-sqlite3")
+const db = new sqlite("ooye.db", {fileMustExist: true})
+
+const passthrough = require("../src/passthrough")
+Object.assign(passthrough, {db, sync})
+
+const api = require("../src/matrix/api")
+const utils = require("../src/matrix/utils")
+const {reg} = require("../src/matrix/read-registration")
+
+const rooms = db.prepare("select room_id, name, nick from channel_room").all()
+
+;(async () => {
+	// Search for members starting with @_ooye_ and kick them if they are not in sim_member cache
+	for (const room of rooms) {
+		try {
+			const members = await api.getJoinedMembers(room.room_id)
+			for (const mxid of Object.keys(members.joined)) {
+				if (!mxid.startsWith("@" + reg.sender_localpart) && utils.eventSenderIsFromDiscord(mxid) && !db.prepare("select mxid from sim_member where mxid = ? and room_id = ?").get(mxid, room.room_id)) {
+					await api.leaveRoom(room.room_id, mxid)
+				}
+			}
+		} catch (e) {
+			if (e.message.includes("Appservice not in room")) {
+				// ok
+			} else {
+				throw e
+			}
+		}
+	}
+})()

src/d2m/actions/create-room.js

@@ -256,7 +256,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 
 /**
  * Handling power levels separately. The spec doesn't specify what happens, Dendrite differs,
- * and Synapse does an absolutely insane *shallow merge* of what I provide on top of what it creates.
+ * and Synapse does a very poorly thought out *shallow merge* of what I provide on top of what it creates.
  * We don't want the `events` key to be overridden completely.
  * https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/room.py#L1170-L1210
  * https://github.com/matrix-org/matrix-spec/issues/492

src/d2m/actions/register-user.js

@@ -206,14 +206,16 @@ function _hashProfileContent(content, powerLevel) {
  * 3. Calculate the power level the user should get based on their Discord permissions
  * 4. Compare against the previously known state content, which is helpfully stored in the database
  * 5. If the state content or power level have changed, send them to Matrix and update them in the database for next time
+ * 6. If the sim is for a user-installed app, check which user it was added by
  * @param {DiscordTypes.APIUser} user
  * @param {Omit<DiscordTypes.APIGuildMember, "user"> | undefined} member
  * @param {DiscordTypes.APIGuildChannel} channel
  * @param {DiscordTypes.APIGuild} guild
  * @param {string} roomID
+ * @param {DiscordTypes.APIMessageInteractionMetadata} [interactionMetadata]
  * @returns {Promise<string>} mxid of the updated sim
  */
-async function syncUser(user, member, channel, guild, roomID) {
+async function syncUser(user, member, channel, guild, roomID, interactionMetadata) {
 	const mxid = await ensureSimJoined(user, roomID)
 	const content = await memberToStateContent(user, member, guild.id)
 	const powerLevel = memberToPowerLevel(user, member, guild, channel)
@@ -222,6 +224,12 @@ async function syncUser(user, member, channel, guild, roomID) {
 		allowOverwrite: !!member,
 		globalProfile: await userToGlobalProfile(user)
 	})
+
+	const appInstalledByUser = user.bot && interactionMetadata?.authorizing_integration_owners?.[DiscordTypes.ApplicationIntegrationType.UserInstall]
+	if (appInstalledByUser) {
+		db.prepare("INSERT OR IGNORE INTO app_user_install (app_bot_id, user_id, guild_id) VALUES (?, ?, ?)").run(user.id, appInstalledByUser, guild.id)
+	}
+
 	return mxid
 }
 

src/d2m/actions/remove-member.js

@@ -0,0 +1,37 @@
+// @ts-check
+
+const passthrough = require("../../passthrough")
+const {sync, db, select, from} = passthrough
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+/** @type {import("../converters/remove-member-mxids")} */
+const removeMemberMxids = sync.require("../converters/remove-member-mxids")
+
+/**
+ * @param {string} userID discord user ID that left
+ * @param {string} guildID discord guild ID that they left
+ */
+async function removeMember(userID, guildID) {
+	const {userAppDeletions, membership} = removeMemberMxids.removeMemberMxids(userID, guildID)
+	db.transaction(() => {
+		for (const d of userAppDeletions) {
+			db.prepare("DELETE FROM app_user_install WHERE guild_id = ? and user_id = ?").run(guildID, d)
+		}
+	})()
+	for (const m of membership) {
+		try {
+			await api.leaveRoom(m.room_id, m.mxid)
+		} catch (e) {
+			if (String(e).includes("not in room")) {
+				// no further action needed
+			} else {
+				throw e
+			}
+		}
+		// Update cache to say that the member isn't in the room any more
+		// You'd think this would happen automatically when the leave event arrives at Matrix's event dispatcher, but that isn't 100% reliable.
+		db.prepare("DELETE FROM sim_member WHERE room_id = ? AND mxid = ?").run(m.room_id, m.mxid)
+	}
+}
+
+module.exports.removeMember = removeMember

src/d2m/actions/send-message.js

@@ -51,7 +51,7 @@ async function sendMessage(message, channel, guild, row) {
 		if (message.author.id === discord.application.id) {
 			// no need to sync the bot's own user
 		} else {
-			senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID)
+			senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID, message.interaction_metadata)
 		}
 	}
 

src/d2m/actions/set-presence.js

@@ -1,5 +1,7 @@
 // @ts-check
 
+const assert = require("assert").strict
+
 const passthrough = require("../../passthrough")
 const {sync, select} = passthrough
 /** @type {import("../../matrix/api")} */
@@ -26,7 +28,7 @@ const presenceLoopInterval = 28e3
 
 // Cache the list of enabled guilds rather than accessing it like multiple times per second when any user changes presence
 const guildPresenceSetting = new class {
-	/** @private @type {Set<string>} */ guilds
+	/** @private @type {Set<string>} */ guilds = new Set()
 	constructor() {
 		this.update()
 	}
@@ -40,7 +42,7 @@ const guildPresenceSetting = new class {
 
 class Presence extends sync.reloadClassMethods(() => Presence) {
 	/** @type {string} */ userID
-	/** @type {{presence: "online" | "offline" | "unavailable", status_msg?: string}} */ data
+	/** @type {{presence: "online" | "offline" | "unavailable", status_msg?: string} | undefined} */ data
 	/** @private @type {?string | undefined} */ mxid
 	/** @private @type {number} */ delay = Math.random()
 
@@ -66,6 +68,7 @@ class Presence extends sync.reloadClassMethods(() => Presence) {
 		// I haven't tried, but I assume Synapse explodes if you try to update too many presences at the same time.
 		// This random delay will space them out over the whole 28 second cycle.
 		setTimeout(() => {
+			assert(this.data)
 			api.setPresence(this.data, mxid).catch(() => {})
 		}, this.delay * presenceLoopInterval).unref()
 	}

src/d2m/converters/message-to-event.js

@@ -35,10 +35,10 @@ function getDiscordParseCallbacks(message, guild, useHTML, spoilers = []) {
 		/** @param {{id: string, type: "discordUser"}} node */
 		user: node => {
 			const mxid = select("sim", "mxid", {user_id: node.id}).pluck().get()
-			const interaction = message.interaction_metadata || message.interaction
+			const interactionMetadata = message.interaction_metadata
 			const username = message.mentions?.find(ment => ment.id === node.id)?.username
 				|| message.referenced_message?.mentions?.find(ment => ment.id === node.id)?.username
-				|| (interaction?.user.id === node.id ? interaction.user.username : null)
+				|| (interactionMetadata?.user.id === node.id ? interactionMetadata.user.username : null)
 				|| (message.author?.id === node.id ? message.author.username : null)
 				|| "unknown-user"
 			if (mxid && useHTML) {
@@ -357,9 +357,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 		}]
 	}
 
-	const interaction = message.interaction_metadata || message.interaction
-	const isInteraction = message.type === DiscordTypes.MessageType.ChatInputCommand && !!interaction && "name" in interaction
-	const isThinkingInteraction = isInteraction && !!((message.flags || 0) & DiscordTypes.MessageFlags.Loading)
+	let isInteraction = (message.type === DiscordTypes.MessageType.ChatInputCommand || message.type === DiscordTypes.MessageType.ContextMenuCommand) && message.interaction && "name" in message.interaction
+	let isThinkingInteraction = isInteraction && !!((message.flags || 0) & DiscordTypes.MessageFlags.Loading)
 
 	/**
 	   @type {{room?: boolean, user_ids?: string[]}}
@@ -400,6 +399,16 @@ async function messageToEvent(message, guild, options = {}, di) {
 		} else if (message.referenced_message) {
 			repliedToUnknownEvent = true
 		}
+	} else if (message.type === DiscordTypes.MessageType.ContextMenuCommand && message.interaction && message.message_reference?.message_id) {
+		// It could be a /plu/ral emulated reply
+		if (message.interaction.name.startsWith("Reply ") && message.content.startsWith("-# [↪](")) {
+			const row = await getHistoricalEventRow(message.message_reference?.message_id)
+			if (row && "event_id" in row) {
+				repliedToEventRow = Object.assign(row, {channel_id: row.reference_channel_id})
+				message.content = message.content.replace(/^.*\n/, "")
+				isInteraction = false // declutter
+			}
+		}
 	} else if (dUtils.isWebhookMessage(message) && message.embeds[0]?.author?.name?.endsWith("↩️")) {
 		// It could be a PluralKit emulated reply, let's see if it has a message link
 		const isEmulatedReplyToText = message.embeds[0].description?.startsWith("**[Reply to:]")
@@ -685,8 +694,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 			}
 		}
 
-		if (isInteraction && !isThinkingInteraction && events.length === 0) {
-			const formattedInteraction = getFormattedInteraction(interaction, false)
+		if (isInteraction && !isThinkingInteraction && message.interaction && events.length === 0) {
+			const formattedInteraction = getFormattedInteraction(message.interaction, false)
 			body = `${formattedInteraction.body}\n${body}`
 			html = `${formattedInteraction.html}${html}`
 		}
@@ -782,8 +791,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 		events.push(...forwardedEvents)
 	}
 
-	if (isThinkingInteraction) {
-		const formattedInteraction = getFormattedInteraction(interaction, true)
+	if (isInteraction && isThinkingInteraction && message.interaction) {
+		const formattedInteraction = getFormattedInteraction(message.interaction, true)
 		await addTextEvent(formattedInteraction.body, formattedInteraction.html, "m.notice")
 	}
 

src/d2m/converters/message-to-event.test.components.js

@@ -65,7 +65,7 @@ test("message2event components: pk question mark output", async t => {
 			+ "<hr>"
 			+ "<blockquote><p><strong>System:</strong> INX (<code>xffgnx</code>)"
 			+ "<br><strong>Member:</strong> Lillith (<code>pphhoh</code>)"
-			+ "<br><strong>Sent by:</strong> infinidoge1337 (@unknown-user:)"
+			+ "<br><strong>Sent by:</strong> infinidoge1337 (<a href=\"https://matrix.to/#/@_ooye_infinidoge1337:cadence.moe\">@unknown-user</a>)"
 			+ "<br><br><strong>Account Roles (7)</strong>"
 			+ "<br>§b, !, ‼, Ears Port Ping, Ears Update Ping, Yttr Ping, unsup Ping</p>"
 			+ `🖼️ <a href="https://files.inx.moe/p/cdn/lillith.webp">https://files.inx.moe/p/cdn/lillith.webp</a>`

src/d2m/converters/message-to-event.test.embeds.js

@@ -125,8 +125,8 @@ test("message2event embeds: blockquote in embed", async t => {
 	t.equal(called, 1, "should call getJoinedMembers once")
 })
 
-test("message2event embeds: crazy html is all escaped", async t => {
-	const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general)
+test("message2event embeds: extreme html is all escaped", async t => {
+	const events = await messageToEvent(data.message_with_embeds.extreme_html_escaping, data.guild.general)
 	t.deepEqual(events, [{
 		$type: "m.room.message",
 		msgtype: "m.notice",

src/d2m/converters/remove-member-mxids.js

@@ -0,0 +1,38 @@
+// @ts-check
+
+const passthrough = require("../../passthrough")
+const {db, select, from} = passthrough
+
+/**
+ * @param {string} userID discord user ID that left
+ * @param {string} guildID discord guild ID that they left
+ */
+function removeMemberMxids(userID, guildID) {
+	// Get sims for user and remove
+	let membership = from("sim").join("sim_member", "mxid").join("channel_room", "room_id")
+		.select("room_id", "mxid").where({user_id: userID, guild_id: guildID}).and("ORDER BY room_id, mxid").all()
+	membership = membership.concat(from("sim_proxy").join("sim", "user_id").join("sim_member", "mxid").join("channel_room", "room_id")
+		.select("room_id", "mxid").where({proxy_owner_id: userID, guild_id: guildID}).and("ORDER BY room_id, mxid").all())
+
+	// Get user installed apps and remove
+	/** @type {string[]} */
+	let userAppDeletions = []
+	// 1. Select apps that have 1 user remaining
+	/** @type {Set<string>} */
+	const appsWithOneUser = new Set(db.prepare("SELECT app_bot_id FROM app_user_install WHERE guild_id = ? GROUP BY app_bot_id HAVING count(*) = 1").pluck().all(guildID))
+	// 2. Select apps installed by this user
+	const appsFromThisUser = new Set(select("app_user_install", "app_bot_id", {guild_id: guildID, user_id: userID}).pluck().all())
+	if (appsFromThisUser.size) userAppDeletions.push(userID)
+	// Then remove user installed apps if this was the last user with them
+	const appsToRemove = appsWithOneUser.intersection(appsFromThisUser)
+	for (const botID of appsToRemove) {
+		// Remove sims for user installed app
+		const appRemoval = removeMemberMxids(botID, guildID)
+		membership = membership.concat(appRemoval.membership)
+		userAppDeletions = userAppDeletions.concat(appRemoval.userAppDeletions)
+	}
+
+	return {membership, userAppDeletions}
+}
+
+module.exports.removeMemberMxids = removeMemberMxids

src/d2m/converters/remove-member-mxids.test.js

@@ -0,0 +1,43 @@
+// @ts-check
+
+const {test} = require("supertape")
+const {removeMemberMxids} = require("./remove-member-mxids")
+
+test("remove member mxids: would remove mxid for all rooms in this server", t => {
+	t.deepEqual(removeMemberMxids("772659086046658620", "112760669178241024"), {
+		userAppDeletions: [],
+		membership: [{
+			mxid: "@_ooye_cadence:cadence.moe",
+			room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe"
+		}, {
+			mxid: "@_ooye_cadence:cadence.moe",
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe"
+		}]
+	})
+})
+
+test("remove member mxids: removes sims too", t => {
+	t.deepEqual(removeMemberMxids("196188877885538304", "112760669178241024"), {
+		userAppDeletions: [],
+		membership: [{
+			mxid: '@_ooye_ampflower:cadence.moe',
+			room_id: '!qzDBLKlildpzrrOnFZ:cadence.moe'
+		}, {
+			mxid: '@_ooye__pk_zoego:cadence.moe',
+			room_id: '!qzDBLKlildpzrrOnFZ:cadence.moe'
+		}]
+	})
+})
+
+test("remove member mxids: removes apps too", t => {
+	t.deepEqual(removeMemberMxids("197126718400626689", "66192955777486848"), {
+		userAppDeletions: ["197126718400626689"],
+		membership: [{
+			mxid: '@_ooye_infinidoge1337:cadence.moe',
+			room_id: '!BnKuBPCvyfOkhcUjEu:cadence.moe'
+		}, {
+			mxid: '@_ooye_evil_lillith_sheher:cadence.moe',
+			room_id: '!BnKuBPCvyfOkhcUjEu:cadence.moe'
+		}]
+	})
+})

src/d2m/discord-packets.js

@@ -49,8 +49,9 @@ const utils = {
 			if (listen === "full") {
 				try {
 					await eventDispatcher.checkMissedExpressions(message.d)
-					await eventDispatcher.checkMissedPins(client, message.d)
 					await eventDispatcher.checkMissedMessages(client, message.d)
+					await eventDispatcher.checkMissedPins(client, message.d)
+					await eventDispatcher.checkMissedLeaves(client, message.d)
 				} catch (e) {
 					console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:")
 					console.error(e)

src/d2m/event-dispatcher.js

@@ -32,6 +32,8 @@ const speedbump = sync.require("./actions/speedbump")
 const retrigger = sync.require("./actions/retrigger")
 /** @type {import("./actions/set-presence")} */
 const setPresence = sync.require("./actions/set-presence")
+/** @type {import("./actions/remove-member")} */
+const removeMember = sync.require("./actions/remove-member")
 /** @type {import("./actions/poll-vote")} */
 const vote = sync.require("./actions/poll-vote")
 /** @type {import("../m2d/event-dispatcher")} */
@@ -123,6 +125,7 @@ module.exports = {
 			// Send in order
 			for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) {
 				const message = messages[i]
+				if (message.type === DiscordTypes.MessageType.UserJoin) continue // since join announcements don't become events, it would be a repetition to act on them during backfill
 
 				if (!members.has(message.author.id)) members.set(message.author.id, await client.snow.guild.getGuildMember(guild.id, message.author.id).catch(() => undefined))
 				await module.exports.MESSAGE_CREATE(client, {
@@ -172,6 +175,31 @@ module.exports = {
 		await createSpace.syncSpaceExpressions(data, true)
 	},
 
+	/**
+	 * When logging back in, check if any members left while we were gone.
+	 * Do this by getting the member list from Discord and seeing who we still have locally that isn't there in the response.
+	 * @param {import("./discord-client")} client
+	 * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild
+	 */
+	async checkMissedLeaves(client, guild) {
+		const maxLimit = 1000
+		if (guild.member_count >= maxLimit) return // too large to want to scan
+		const discordMembers = await client.snow.guild.getGuildMembers(guild.id, {limit: maxLimit})
+		if (discordMembers.length >= maxLimit) return // response was maxed out, there are guild members that weren't listed, can't act safely
+		const discordMembersSet = new Set(discordMembers.map(m => m.user.id))
+		// no indexes on this one but I'll cope
+		const membersAddedOnMatrix = new Set(from("sim").join("sim_member", "mxid").join("channel_room", "room_id")
+			.pluck("user_id").selectUnsafe("DISTINCT user_id").where({guild_id: guild.id}).and("AND user_id not like '%-%' and user_id not like '%\\_%' escape '\\'").all())
+		const userInstalledAppIDs = new Set(from("app_user_install").pluck("app_bot_id").selectUnsafe("DISTINCT app_bot_id").where({guild_id: guild.id}).all())
+		// loop over members added on matrix and if the member does not exist on discord-side then they should be removed
+		for (const userID of membersAddedOnMatrix) {
+			if (userInstalledAppIDs.has(userID)) continue // skip user installed apps here since they're never true members - they'll be removed by removeMember when the associated user is removed
+			if (!discordMembersSet.has(userID)) {
+				await removeMember.removeMember(userID, guild.id)
+			}
+		}
+	},
+
 	/**
 	 * Announces to the parent room that the thread room has been created.
 	 * See notes.md, "Ignore MESSAGE_UPDATE and bridge THREAD_CREATE as the announcement"
@@ -211,6 +239,14 @@ module.exports = {
 		}
 	},
 
+	/**
+	 * @param {import("./discord-client")} client
+	 * @param {DiscordTypes.GatewayGuildMemberRemoveDispatchData} data
+	 */
+	async GUILD_MEMBER_REMOVE(client, data) {
+		await removeMember.removeMember(data.user.id, data.guild_id)
+	},
+
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {DiscordTypes.GatewayChannelUpdateDispatchData} channelOrThread

src/db/migrations/0035-role-default.sql

@@ -4,6 +4,6 @@ CREATE TABLE "role_default" (
 	"guild_id"	TEXT NOT NULL,
 	"role_id"	TEXT NOT NULL,
 	PRIMARY KEY ("guild_id", "role_id")
-);
+) WITHOUT ROWID;
 
 COMMIT;

src/db/migrations/0036-app-user-install.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE "app_user_install" (
+	"guild_id"	TEXT NOT NULL,
+	"app_bot_id"	TEXT NOT NULL,
+	"user_id"	TEXT NOT NULL,
+	PRIMARY KEY ("guild_id", "app_bot_id", "user_id")
+) WITHOUT ROWID;
+
+COMMIT;

src/db/orm-defs.d.ts

@@ -1,4 +1,10 @@
 export type Models = {
+	app_user_install: {
+		guild_id: string
+		app_bot_id: string
+		user_id: string
+	}
+
 	auto_emoji: {
 		name: string
 		emoji_id: string

src/discord/utils.js

@@ -114,7 +114,7 @@ function hasAllPermissions(resolvedPermissions, permissionsToCheckFor) {
  * @param {DiscordTypes.APIMessage} message
  */
 function isWebhookMessage(message) {
-	return message.webhook_id && message.type !== DiscordTypes.MessageType.ChatInputCommand
+	return message.webhook_id && message.type !== DiscordTypes.MessageType.ChatInputCommand && message.type !== DiscordTypes.MessageType.ContextMenuCommand
 }
 
 /**

src/m2d/converters/event-to-message.js

@@ -471,7 +471,8 @@ async function checkWrittenMentions(content, senderMxid, roomID, guild, di) {
 					// @ts-ignore - typescript doesn't know about indices yet
 					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `@everyone` + content.slice(writtenMentionMatch.indices[1][1]),
 					ensureJoined: [],
-					allowedMentionsParse: ["everyone"]
+					allowedMentionsParse: ["everyone"],
+					allowedMentionsUsers: []
 				}
 			}
 		} else if (writtenMentionMatch[1].length < 40) { // the API supports up to 100 characters, but really if you're searching more than 40, something messed up
@@ -482,7 +483,8 @@ async function checkWrittenMentions(content, senderMxid, roomID, guild, di) {
 					// @ts-ignore - typescript doesn't know about indices yet
 					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]),
 					ensureJoined: [results[0].user],
-					allowedMentionsParse: []
+					allowedMentionsParse: [],
+					allowedMentionsUsers: [results[0].user.id]
 				}
 			}
 		}
@@ -544,6 +546,7 @@ async function eventToMessage(event, guild, channel, di) {
 	let displayName = event.sender
 	let avatarURL = undefined
 	const allowedMentionsParse = ["users", "roles"]
+	const allowedMentionsUsers = []
 	/** @type {string[]} */
 	let messageIDsToEdit = []
 	let replyLine = ""
@@ -763,7 +766,7 @@ async function eventToMessage(event, guild, channel, di) {
 				// Generate a reply preview for a standard message
 				repliedToContent = repliedToContent.replace(/.*<\/mx-reply>/s, "") // Remove everything before replies, so just use the actual message body
 				repliedToContent = repliedToContent.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
-				repliedToContent = repliedToContent.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+				repliedToContent = repliedToContent.replace(/(?:\n|<br ?\/?>)+/g, " ") // Should all be on one line
 				repliedToContent = repliedToContent.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
 				repliedToContent = repliedToContent.replace(/<img([^>]*)>/g, (_, att) => { // Convert Matrix emoji images into Discord emoji markdown
 					const mxcUrlMatch = att.match(/\bsrc="(mxc:\/\/[^"]+)"/)
@@ -986,16 +989,34 @@ async function eventToMessage(event, guild, channel, di) {
 		}
 	}
 
+	// Complete content
 	content = displayNameRunoff + replyLine + content
-
 	// Split into 2000 character chunks
 	const chunks = chunk(content, 2000)
+
+	// If m.mentions is specified and valid, overwrite allowedMentionsParse with a converted m.mentions
+	let allowed_mentions = {parse: allowedMentionsParse}
+	if (event.content["m.mentions"]) {
+		// Combine requested mentions with detected written mentions to get the full list
+		if (Array.isArray(event.content["m.mentions"].user_ids)) {
+			for (const mxid of event.content["m.mentions"].user_ids) {
+				const user_id = select("sim", "user_id", {mxid}).pluck().get()
+				if (!user_id) continue
+				allowedMentionsUsers.push(
+					select("sim_proxy", "proxy_owner_id", {user_id}).pluck().get() || user_id
+				)
+			}
+		}
+		// Specific mentions were requested, so do not parse users
+		allowed_mentions.parse = allowed_mentions.parse.filter(x => x !== "users")
+		allowed_mentions.users = allowedMentionsUsers
+	}
+
+	// Assemble chunks into Discord messages content
 	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | stream.Readable}[]})[]} */
 	const messages = chunks.map(content => ({
 		content,
-		allowed_mentions: {
-			parse: allowedMentionsParse
-		},
+		allowed_mentions,
 		username: displayNameShortened,
 		avatar_url: avatarURL
 	}))

src/m2d/converters/event-to-message.test.js

@@ -266,7 +266,8 @@ test("event2message: markdown in link text does not attempt to be escaped becaus
 				content: "hey [@mario sports mix [she/her]](<https://matrix.to/#/%40cadence%3Acadence.moe>), is it possible to listen on a unix socket?",
 				avatar_url: undefined,
 				allowed_mentions: {
-					parse: ["users", "roles"]
+					parse: ["roles"],
+					users: []
 				}
 			}]
 		}
@@ -547,7 +548,8 @@ test("event2message: links don't have angle brackets added by accident", async t
 				content: "Wanted to automate WG→AWG config enrichment and ended up basically coding a batch INI processor.\nhttps://github.com/Erquint/wgcbp",
 				avatar_url: undefined,
 				allowed_mentions: {
-					parse: ["users", "roles"]
+					parse: ["roles"],
+					users: []
 				}
 			}]
 		}
@@ -1296,7 +1298,8 @@ test("event2message: lists have appropriate line breaks", async t => {
 				content: `i am not certain what you mean by "already exists with as discord". my goals are\n\n* bridgeing specific channels with existing matrix rooms\n  * optionally maybe entire "servers"\n* offering the bridge as a public service`,
 				avatar_url: undefined,
 				allowed_mentions: {
-					parse: ["users", "roles"]
+					parse: ["roles"],
+					users: []
 				}
 			}]
 		}
@@ -1337,7 +1340,8 @@ test("event2message: ordered list start attribute works", async t => {
 				content: `i am not certain what you mean by "already exists with as discord". my goals are\n\n1. bridgeing specific channels with existing matrix rooms\n  2. optionally maybe entire "servers"\n2. offering the bridge as a public service`,
 				avatar_url: undefined,
 				allowed_mentions: {
-					parse: ["users", "roles"]
+					parse: ["roles"],
+					users: []
 				}
 			}]
 		}
@@ -1463,6 +1467,118 @@ test("event2message: rich reply to a sim user", async t => {
 	)
 })
 
+test("event2message: rich reply to a sim user, explicitly enabling mentions in client", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			"type": "m.room.message",
+			"sender": "@cadence:cadence.moe",
+			"content": {
+				"msgtype": "m.text",
+				"body": "> <@_ooye_kyuugryphon:cadence.moe> Slow news day.\n\nTesting this reply, ignore",
+				"format": "org.matrix.custom.html",
+				"formatted_body": "<mx-reply><blockquote><a href=\"https://matrix.to/#/!fGgIymcYWOqjbSRUdV:cadence.moe/$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04?via=cadence.moe&via=feather.onl\">In reply to</a> <a href=\"https://matrix.to/#/@_ooye_kyuugryphon:cadence.moe\">@_ooye_kyuugryphon:cadence.moe</a><br>Slow news day.</blockquote></mx-reply>Testing this reply, ignore",
+				"m.relates_to": {
+					"m.in_reply_to": {
+						"event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04"
+					}
+				},
+				"m.mentions": {
+					user_ids: ["@_ooye_kyuugryphon:cadence.moe"]
+				}
+			},
+			"origin_server_ts": 1693029683016,
+			"unsigned": {
+				"age": 91,
+				"transaction_id": "m1693029682894.510"
+			},
+			"event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8",
+			"room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe"
+		}, data.guild.general, data.channel.general, {
+			api: {
+				getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", {
+					type: "m.room.message",
+					content: {
+						msgtype: "m.text",
+						body: "Slow news day."
+					},
+					sender: "@_ooye_kyuugryphon:cadence.moe"
+				})
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "-# > <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 <@111604486476181504>:"
+					+ " Slow news day."
+					+ "\nTesting this reply, ignore",
+				avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU",
+				allowed_mentions: {
+					parse: ["roles"],
+					users: ["111604486476181504"]
+				}
+			}]
+		}
+	)
+})
+
+test("event2message: rich reply to a sim user, explicitly disabling mentions in client", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			"type": "m.room.message",
+			"sender": "@cadence:cadence.moe",
+			"content": {
+				"msgtype": "m.text",
+				"body": "> <@_ooye_kyuugryphon:cadence.moe> Slow news day.\n\nTesting this reply, ignore",
+				"format": "org.matrix.custom.html",
+				"formatted_body": "<mx-reply><blockquote><a href=\"https://matrix.to/#/!fGgIymcYWOqjbSRUdV:cadence.moe/$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04?via=cadence.moe&via=feather.onl\">In reply to</a> <a href=\"https://matrix.to/#/@_ooye_kyuugryphon:cadence.moe\">@_ooye_kyuugryphon:cadence.moe</a><br>Slow news day.</blockquote></mx-reply>Testing this reply, ignore",
+				"m.relates_to": {
+					"m.in_reply_to": {
+						"event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04"
+					}
+				},
+				"m.mentions": {}
+			},
+			"origin_server_ts": 1693029683016,
+			"unsigned": {
+				"age": 91,
+				"transaction_id": "m1693029682894.510"
+			},
+			"event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8",
+			"room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe"
+		}, data.guild.general, data.channel.general, {
+			api: {
+				getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", {
+					type: "m.room.message",
+					content: {
+						msgtype: "m.text",
+						body: "Slow news day."
+					},
+					sender: "@_ooye_kyuugryphon:cadence.moe"
+				})
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "-# > <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 <@111604486476181504>:"
+					+ " Slow news day."
+					+ "\nTesting this reply, ignore",
+				avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/azCAhThKTojXSZJRoWwZmhvU",
+				allowed_mentions: {
+					parse: ["roles"],
+					users: []
+				}
+			}]
+		}
+	)
+})
+
 test("event2message: rich reply to a rich reply to a multi-line message should correctly strip reply fallback", async t => {
 	t.deepEqual(
 		await eventToMessage({
@@ -1827,9 +1943,9 @@ test("event2message: should suppress embeds for links in reply preview", async t
 			sender: "@rnl:cadence.moe",
 			content: {
 				msgtype: "m.text",
-				body: `> <@cadence:cadence.moe> https://www.youtube.com/watch?v=uX32idb1jMw\n\nEveryone in the comments is like "you're so ahead of the curve" and "crazy that this came out before the scandal" but I thought Mr Beast sucking was a common sentiment`,
+				body: `> <@cadence:cadence.moe> https://www.youtube.com/watch?v=uX32idb1jMw\n\nEveryone in the comments is like "you're so ahead of the curve" and "can't believe this came out before the scandal" but I thought Mr Beast sucking was a common sentiment`,
 				format: "org.matrix.custom.html",
-				formatted_body: `<mx-reply><blockquote><a href="https://matrix.to/#/!fGgIymcYWOqjbSRUdV:cadence.moe/$qmyjr-ISJtnOM5WTWLI0fT7uSlqRLgpyin2d2NCglCU?via=cadence.moe">In reply to</a> <a href="https://matrix.to/#/@cadence:cadence.moe">@cadence:cadence.moe</a><br>https://www.youtube.com/watch?v=uX32idb1jMw</blockquote></mx-reply>Everyone in the comments is like "you're so ahead of the curve" and "crazy that this came out before the scandal" but I thought Mr Beast sucking was a common sentiment`,
+				formatted_body: `<mx-reply><blockquote><a href="https://matrix.to/#/!fGgIymcYWOqjbSRUdV:cadence.moe/$qmyjr-ISJtnOM5WTWLI0fT7uSlqRLgpyin2d2NCglCU?via=cadence.moe">In reply to</a> <a href="https://matrix.to/#/@cadence:cadence.moe">@cadence:cadence.moe</a><br>https://www.youtube.com/watch?v=uX32idb1jMw</blockquote></mx-reply>Everyone in the comments is like "you're so ahead of the curve" and "can't believe this came out before the scandal" but I thought Mr Beast sucking was a common sentiment`,
 				"m.relates_to": {
 					"m.in_reply_to": {
 						event_id: "$qmyjr-ISJtnOM5WTWLI0fT7uSlqRLgpyin2d2NCglCU"
@@ -1859,7 +1975,7 @@ test("event2message: should suppress embeds for links in reply preview", async t
 				username: "RNL",
 				content: "-# > <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1273204543739396116 **Ⓜcadence [they]**:"
 					+ " <https://www.youtube.com/watch?v=uX32idb1jMw>"
-					+ `\nEveryone in the comments is like "you're so ahead of the curve" and "crazy that this came out before the scandal" but I thought Mr Beast sucking was a common sentiment`,
+					+ `\nEveryone in the comments is like "you're so ahead of the curve" and "can't believe this came out before the scandal" but I thought Mr Beast sucking was a common sentiment`,
 				avatar_url: undefined,
 				allowed_mentions: {
 						parse: ["users", "roles"]

src/m2d/event-dispatcher.js

@@ -423,7 +423,10 @@ async event => {
 
 	if (event.content.membership === "leave" || event.content.membership === "ban") {
 		// Member is gone
+		// if Matrix member, data was cached in member_cache
 		db.prepare("DELETE FROM member_cache WHERE room_id = ? and mxid = ?").run(event.room_id, event.state_key)
+		// if Discord member (so kicked/banned by Matrix user), data was cached in sim_member
+		db.prepare("DELETE FROM sim_member WHERE room_id = ? and mxid = ?").run(event.room_id, event.state_key)
 
 		// Unregister room's use as a direct chat and/or an invite target if the bot itself left
 		if (event.state_key === utils.bot) {

src/web/pug/includes/template.pug

@@ -88,6 +88,12 @@ html(lang="en")
         --_ts-multiple-bg: var(--green-400);
         --_ts-multiple-fc: var(--white);
       }
+      .s-avatar {
+        --_av-bg: var(--white);
+      }
+      .s-avatar .s-avatar--letter {
+        color: var(--white);
+      }
       .s-btn__dropdown:has(+ :popover-open) {
         background-color: var(--theme-topbar-item-background-hover, var(--black-200)) !important;
       }

src/web/server.js

@@ -83,7 +83,13 @@ function tryStatic(event, fallthrough) {
 			// Everything else
 			else {
 				const mime = mimeTypes.lookup(id)
-				if (typeof mime === "string") defaultContentType(event, mime)
+				if (typeof mime === "string") {
+					if (mime.startsWith("text/")) {
+						defaultContentType(event, mime + "; charset=utf-8") // usually wise
+					} else {
+						defaultContentType(event, mime)
+					}
+				}
 				return {
 					size: stats.size
 				}
@@ -94,7 +100,7 @@ function tryStatic(event, fallthrough) {
 				const path = join(publicDir, id)
 				return pugSync.renderPath(event, path, {})
 			} else {
-				return fs.promises.readFile(join(publicDir, id))
+				return fs.createReadStream(join(publicDir, id))
 			}
 		}
 	})

test/data.js

@@ -4617,7 +4617,7 @@ module.exports = {
 			flags: 0,
 			components: []
 		},
-		escaping_crazy_html_tags: {
+		extreme_html_escaping: {
 			id: "1158894131322552391",
 			type: 0,
 			content: "",

test/ooye-test-data.sql

@@ -38,15 +38,28 @@ INSERT INTO sim (user_id, username, sim_name, mxid) VALUES
 ('1109360903096369153', 'Amanda', 'amanda', '@_ooye_amanda:cadence.moe'),
 ('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '_pk_zoego', '_pk_zoego', '@_ooye__pk_zoego:cadence.moe'),
 ('320067006521147393', 'papiophidian', 'papiophidian', '@_ooye_papiophidian:cadence.moe'),
-('772659086046658620', 'cadence.worm', 'cadence', '@_ooye_cadence:cadence.moe');
+('772659086046658620', 'cadence.worm', 'cadence', '@_ooye_cadence:cadence.moe'),
+('196188877885538304', 'ampflower', 'ampflower', '@_ooye_ampflower:cadence.moe'),
+('1458668878107381800', 'Evil Lillith (she/her)', 'evil_lillith_sheher', '@_ooye_evil_lillith_sheher:cadence.moe'),
+('197126718400626689', 'infinidoge1337', 'infinidoge1337', '@_ooye_infinidoge1337:cadence.moe');
+
 
 INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES
 ('@_ooye_bojack_horseman:cadence.moe', '!hYnGGlPHlbujVVfktC:cadence.moe', NULL),
-('@_ooye_cadence:cadence.moe', '!BnKuBPCvyfOkhcUjEu:cadence.moe', NULL);
+('@_ooye_cadence:cadence.moe', '!BnKuBPCvyfOkhcUjEu:cadence.moe', NULL),
+('@_ooye_cadence:cadence.moe', '!kLRqKKUQXcibIMtOpl:cadence.moe', NULL),
+('@_ooye_cadence:cadence.moe', '!fGgIymcYWOqjbSRUdV:cadence.moe', NULL),
+('@_ooye_ampflower:cadence.moe', '!qzDBLKlildpzrrOnFZ:cadence.moe', NULL),
+('@_ooye__pk_zoego:cadence.moe', '!qzDBLKlildpzrrOnFZ:cadence.moe', NULL),
+('@_ooye_infinidoge1337:cadence.moe', '!BnKuBPCvyfOkhcUjEu:cadence.moe', NULL),
+('@_ooye_evil_lillith_sheher:cadence.moe', '!BnKuBPCvyfOkhcUjEu:cadence.moe', NULL);
 
 INSERT INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES
 ('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '196188877885538304', 'Azalea &flwr; 🌺');
 
+INSERT INTO app_user_install (guild_id, app_bot_id, user_id) VALUES
+('66192955777486848', '1458668878107381800', '197126718400626689');
+
 INSERT INTO message_room (message_id, historical_room_index)
 WITH a (message_id, channel_id) AS (VALUES
 ('1106366167788044450', '122155380120748034'),

test/test.js

@@ -152,6 +152,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
 	require("../src/d2m/converters/message-to-event.test.embeds")
 	require("../src/d2m/converters/message-to-event.test.pk")
 	require("../src/d2m/converters/pins-to-list.test")
+	require("../src/d2m/converters/remove-member-mxids.test")
 	require("../src/d2m/converters/remove-reaction.test")
 	require("../src/d2m/converters/thread-to-announcement.test")
 	require("../src/d2m/converters/user-to-mxid.test")