Bridge forums as spaces

I should probably change this to use kstate.

.c8rc.json

@@ -0,0 +1,8 @@
+{
+	"watermarks": {
+		"statements": [60, 100],
+		"lines": [60, 100],
+		"functions": [60, 100],
+		"branches": [60, 100]
+	}
+}

.gitignore

@@ -3,4 +3,5 @@ config.js
 registration.yaml
 coverage
 db/ooye.db*
-test/res/butterfly*
+test/res/*
+!test/res/lottie*

d2m/actions/create-room.js

@@ -12,6 +12,8 @@ const file = sync.require("../../matrix/file")
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/kstate")} */
 const ks = sync.require("../../matrix/kstate")
+/** @type {import("../../discord/utils")} */
+const utils = sync.require("../../discord/utils")
 /** @type {import("./create-space")}) */
 const createSpace = sync.require("./create-space") // watch out for the require loop
 
@@ -57,13 +59,16 @@ function applyKStateDiffToRoom(roomID, kstate) {
 }
 
 /**
- * @param {{id: string, name: string, topic?: string?, type: number}} channel
+ * @param {{id: string, name: string, topic?: string?, type: number, parent_id?: string?}} channel
  * @param {{id: string}} guild
  * @param {string | null | undefined} customName
  */
 function convertNameAndTopic(channel, guild, customName) {
+	// @ts-ignore
+	const parentChannel = discord.channels.get(channel.parent_id)
 	let channelPrefix =
-		( channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] "
+		( parentChannel?.type === DiscordTypes.ChannelType.GuildForum ? ""
+		: channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] "
 		: channel.type === DiscordTypes.ChannelType.PrivateThread ? "[🔒⛓️] "
 		: channel.type === DiscordTypes.ChannelType.GuildVoice ? "[🔊] "
 		: "")
@@ -86,9 +91,24 @@ function convertNameAndTopic(channel, guild, customName) {
  * @param {DiscordTypes.APIGuild} guild
  */
 async function channelToKState(channel, guild) {
-	const spaceID = await createSpace.ensureSpace(guild)
-	assert(typeof spaceID === "string")
-	const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get()
+	// @ts-ignore
+	const parentChannel = discord.channels.get(channel.parent_id)
+	/** Used for membership/permission checks. */
+	let guildSpaceID
+	/** Used as the literal parent on Matrix, for categorisation. Will be the same as `guildSpaceID` unless it's a forum channel's thread, in which case a different space is used to group those threads. */
+	let parentSpaceID
+	let privacyLevel
+	if (parentChannel?.type === DiscordTypes.ChannelType.GuildForum) { // it's a forum channel's thread, so use a different space to group those threads
+		guildSpaceID = await createSpace.ensureSpace(guild)
+		parentSpaceID = await ensureRoom(channel.parent_id)
+		privacyLevel = select("guild_space", "privacy_level", {space_id: guildSpaceID}).pluck().get()
+	} else { // otherwise use the guild's space like usual
+		parentSpaceID = await createSpace.ensureSpace(guild)
+		guildSpaceID = parentSpaceID
+		privacyLevel = select("guild_space", "privacy_level", {space_id: parentSpaceID}).pluck().get()
+	}
+	assert(typeof parentSpaceID === "string")
+	assert(typeof guildSpaceID === "string")
 	assert(typeof privacyLevel === "number")
 
 	const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get()
@@ -112,20 +132,23 @@ async function channelToKState(channel, guild) {
 		join_rule: "restricted",
 		allow: [{
 			type: "m.room_membership",
-			room_id: spaceID
+			room_id: guildSpaceID
 		}]
 	}
 	if (PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel] !== "restricted") {
 		join_rules = {join_rule: PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel]}
 	}
 
+	const everyonePermissions = utils.getPermissions([], guild.roles, undefined, channel.permission_overwrites)
+	const everyoneCanMentionEveryone = utils.hasAllPermissions(everyonePermissions, ["MentionEveryone"])
+
 	const channelKState = {
 		"m.room.name/": {name: convertedName},
 		"m.room.topic/": {topic: convertedTopic},
 		"m.room.avatar/": avatarEventContent,
 		"m.room.guest_access/": {guest_access: PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
 		"m.room.history_visibility/": {history_visibility},
-		[`m.space.parent/${spaceID}`]: {
+		[`m.space.parent/${parentSpaceID}`]: {
 			via: [reg.ooye.server_name],
 			canonical: true
 		},
@@ -135,6 +158,9 @@ async function channelToKState(channel, guild) {
 			events: {
 				"m.room.avatar": 0
 			},
+			notifications: {
+				room: everyoneCanMentionEveryone ? 0 : 20
+			},
 			users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})
 		},
 		"chat.schildi.hide_ui/read_receipts": {
@@ -159,7 +185,7 @@ async function channelToKState(channel, guild) {
 		}
 	}
 
-	return {spaceID, privacyLevel, channelKState}
+	return {spaceID: parentSpaceID, privacyLevel, channelKState}
 }
 
 /**
@@ -175,6 +201,9 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 	let threadParent = null
 	if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id
 
+	let spaceCreationContent = {}
+	if (channel.type === DiscordTypes.ChannelType.GuildForum) spaceCreationContent = {creation_content: {type: "m.space"}}
+
 	// Name and topic can be done earlier in room creation rather than in initial_state
 	// https://spec.matrix.org/latest/client-server-api/#creation
 	const name = kstate["m.room.name/"].name
@@ -191,7 +220,8 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 			preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
 			visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
 			invite: [],
-			initial_state: ks.kstateToState(kstate)
+			initial_state: ks.kstateToState(kstate),
+			...spaceCreationContent
 		})
 
 		db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent)

d2m/actions/create-room.test.js

@@ -1,5 +1,6 @@
 // @ts-check
 
+const mixin = require("mixin-deep")
 const {channelToKState, _convertNameAndTopic} = require("./create-room")
 const {kstateStripConditionals} = require("../../matrix/kstate")
 const {test} = require("supertape")
@@ -39,6 +40,16 @@ test("channel2room: invite-only privacy room", async t => {
 	)
 })
 
+test("channel2room: room where limited people can mention everyone", async t => {
+	const limitedGuild = mixin({}, testData.guild.general)
+	limitedGuild.roles[0].permissions = (BigInt(limitedGuild.roles[0].permissions) - 131072n).toString()
+	const limitedRoom = mixin({}, testData.room.general, {"m.room.power_levels/": {notifications: {room: 20}}})
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, limitedGuild).then(x => x.channelKState)),
+		limitedRoom
+	)
+})
+
 test("convertNameAndTopic: custom name and topic", t => {
 	t.deepEqual(
 		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, "hauntings"),

d2m/actions/create-space.js

@@ -1,8 +1,9 @@
 // @ts-check
 
 const assert = require("assert").strict
+const {isDeepStrictEqual} = require("util")
 const DiscordTypes = require("discord-api-types/v10")
-const deepEqual = require("deep-equal")
+const Ty = require("../../types")
 const reg = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
@@ -181,9 +182,16 @@ async function syncSpaceFully(guildID) {
 	const spaceDiff = ks.diffKState(spaceKState, guildKState)
 	await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff)
 
-	const childRooms = ks.kstateToState(spaceKState).filter(({type, content}) => {
-		return type === "m.space.child" && "via" in content
-	}).map(({state_key}) => state_key)
+	/** @type {string[]} room IDs */
+	let childRooms = []
+	/** @type {string | undefined} */
+	let nextBatch = undefined
+	do {
+		/** @type {Ty.HierarchyPagination<Ty.R.Hierarchy>} */
+		const res = await api.getHierarchy(spaceID, {from: nextBatch})
+		childRooms.push(...res.rooms.map(room => room.room_id))
+		nextBatch = res.next_batch
+	} while (nextBatch)
 
 	for (const roomID of childRooms) {
 		const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
@@ -226,7 +234,7 @@ async function syncSpaceExpressions(data, checkBeforeSync) {
 				// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
 				existing = fn([])
 			}
-			if (deepEqual(existing, content, {strict: true})) return
+			if (isDeepStrictEqual(existing, content)) return
 		}
 		api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content)
 	}

d2m/actions/register-pk-user.js

@@ -131,7 +131,7 @@ async function syncUser(author, pkMessage, roomID) {
 	db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username)
 	// Sync the member state
 	const content = await memberToStateContent(pkMessage, author)
-	const currentHash = registerUser._hashProfileContent(content)
+	const currentHash = registerUser._hashProfileContent(content, 0)
 	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
 	// only do the actual sync if the hash has changed since we last looked
 	if (existingHash !== currentHash) {

d2m/actions/register-user.js

@@ -1,7 +1,9 @@
 // @ts-check
 
-const assert = require("assert")
+const assert = require("assert").strict
 const reg = require("../../matrix/read-registration")
+const DiscordTypes = require("discord-api-types/v10")
+const mixin = require("mixin-deep")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
@@ -9,6 +11,8 @@ const {discord, sync, db, select} = passthrough
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
+/** @type {import("../../discord/utils")} */
+const utils = sync.require("../../discord/utils")
 /** @type {import("../converters/user-to-mxid")} */
 const userToMxid = sync.require("../converters/user-to-mxid")
 /** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
@@ -18,7 +22,7 @@ require("xxhash-wasm")().then(h => hasher = h)
 
 /**
  * A sim is an account that is being simulated by the bridge to copy events from the other side.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @returns mxid
  */
 async function createSim(user) {
@@ -46,7 +50,7 @@ async function createSim(user) {
 /**
  * Ensure a sim is registered for the user.
  * If there is already a sim, use that one. If there isn't one yet, register a new sim.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @returns {Promise<string>} mxid
  */
 async function ensureSim(user) {
@@ -62,7 +66,7 @@ async function ensureSim(user) {
 
 /**
  * Ensure a sim is registered for the user and is joined to the room.
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {DiscordTypes.APIUser} user
  * @param {string} roomID
  * @returns {Promise<string>} mxid
  */
@@ -92,8 +96,8 @@ async function ensureSimJoined(user, roomID) {
 }
 
 /**
- * @param {import("discord-api-types/v10").APIUser} user
- * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
  */
 async function memberToStateContent(user, member, guildID) {
 	let displayname = user.username
@@ -123,8 +127,46 @@ async function memberToStateContent(user, member, guildID) {
 	return content
 }
 
-function _hashProfileContent(content) {
-	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
+/**
+ * https://gitdab.com/cadence/out-of-your-element/issues/9
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @returns {number} 0 to 100
+ */
+function memberToPowerLevel(user, member, guild, channel) {
+	const permissions = utils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites)
+	/*
+	 * PL 100 = Administrator = People who can brick the room. RATIONALE:
+	 * 	- Administrator.
+	 * 	- Manage Webhooks: People who remove the webhook can break the room.
+	 * 	- Manage Guild: People who can manage guild can add bots.
+	 * 	- Manage Channels: People who can manage the channel can delete it.
+	 * (Setting sim users to PL 100 is safe because even though we can't demote the sims we can use code to make the sims demote themselves.)
+	 */
+	if (guild.owner_id === user.id || utils.hasSomePermissions(permissions, ["Administrator", "ManageWebhooks", "ManageGuild", "ManageChannels"])) return 100
+	/*
+	 * PL 50 = Moderator = People who can manage people and messages in many ways. RATIONALE:
+	 * 	- Manage Messages: Can moderate by pinning or deleting the conversation.
+	 * 	- Manage Nicknames: Can moderate by removing inappropriate nicknames.
+	 * 	- Manage Threads: Can moderate by deleting conversations.
+	 * 	- Kick Members & Ban Members: Can moderate by removing disruptive people.
+	 * 	- Mute Members & Deafen Members: Can moderate by silencing disruptive people in ways they can't undo.
+	 * 	- Moderate Members.
+	 */
+	if (utils.hasSomePermissions(permissions, ["ManageMessages", "ManageNicknames", "ManageThreads", "KickMembers", "BanMembers", "MuteMembers", "DeafenMembers", "ModerateMembers"])) return 50
+	/* PL 20 = Mention Everyone for technical reasons. */
+	if (utils.hasSomePermissions(permissions, ["MentionEveryone"])) return 20
+	return 0
+}
+
+/**
+ * @param {any} content
+ * @param {number} powerLevel
+ */
+function _hashProfileContent(content, powerLevel) {
+	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}\u0000${powerLevel}`)
 	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
 	return signedHash
 }
@@ -133,48 +175,65 @@ function _hashProfileContent(content) {
  * Sync profile data for a sim user. This function follows the following process:
  * 1. Join the sim to the room if needed
  * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
- * 3. Compare against the previously known state content, which is helpfully stored in the database
- * 4. If the state content has changed, send it to Matrix and update it in the database for next time
- * @param {import("discord-api-types/v10").APIUser} user
- * @param {Omit<import("discord-api-types/v10").APIGuildMember, "user">} member
+ * 3. Calculate the power level the user should get based on their Discord permissions
+ * 4. Compare against the previously known state content, which is helpfully stored in the database
+ * 5. If the state content or power level have changed, send them to Matrix and update them in the database for next time
+ * @param {DiscordTypes.APIUser} user
+ * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {string} roomID
  * @returns {Promise<string>} mxid of the updated sim
  */
-async function syncUser(user, member, guildID, roomID) {
+async function syncUser(user, member, channel, guild, roomID) {
 	const mxid = await ensureSimJoined(user, roomID)
-	const content = await memberToStateContent(user, member, guildID)
-	const currentHash = _hashProfileContent(content)
+	const content = await memberToStateContent(user, member, guild.id)
+	const powerLevel = memberToPowerLevel(user, member, guild, channel)
+	const currentHash = _hashProfileContent(content, powerLevel)
 	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
 	// only do the actual sync if the hash has changed since we last looked
 	if (existingHash !== currentHash) {
+		// Update room member state
 		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+		// Update power levels
+		const powerLevelsStateContent = await api.getStateEvent(roomID, "m.room.power_levels", "")
+		mixin(powerLevelsStateContent, {users: {[mxid]: powerLevel}})
+		if (powerLevel === 0) delete powerLevelsStateContent.users[mxid] // keep the event compact
+		await api.sendState(roomID, "m.room.power_levels", "", powerLevelsStateContent)
+		// Update cached hash
 		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
 	}
 	return mxid
 }
 
+/**
+ * @param {string} roomID
+ */
 async function syncAllUsersInRoom(roomID) {
 	const mxids = select("sim_member", "mxid", {room_id: roomID}).pluck().all()
 
 	const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
 	assert.ok(typeof channelID === "string")
 
-	/** @ts-ignore @type {import("discord-api-types/v10").APIGuildChannel} */
+	/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
 	const channel = discord.channels.get(channelID)
 	const guildID = channel.guild_id
 	assert.ok(typeof guildID === "string")
+	/** @ts-ignore @type {DiscordTypes.APIGuild} */
+	const guild = discord.guilds.get(guildID)
 
 	for (const mxid of mxids) {
 		const userID = select("sim", "user_id", {mxid}).pluck().get()
 		assert.ok(typeof userID === "string")
 
-		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIGuildMember>} */
+		/** @ts-ignore @type {Required<DiscordTypes.APIGuildMember>} */
 		const member = await discord.snow.guild.getGuildMember(guildID, userID)
-		/** @ts-ignore @type {Required<import("discord-api-types/v10").APIUser>} user */
+		/** @ts-ignore @type {Required<DiscordTypes.APIUser>} user */
 		const user = member.user
 		assert.ok(user)
 
 		console.log(`[user sync] to matrix: ${user.username} in ${channel.name}`)
-		await syncUser(user, member, guildID, roomID)
+		await syncUser(user, member, channel, guild, roomID)
 	}
 }
 

d2m/actions/send-message.js

@@ -1,6 +1,7 @@
 // @ts-check
 
-const assert = require("assert")
+const assert = require("assert").strict
+const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
 const { discord, sync, db } = passthrough
@@ -18,17 +19,20 @@ const createRoom = sync.require("../actions/create-room")
 const dUtils = sync.require("../../discord/utils")
 
 /**
- * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
- * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
+ * @param {DiscordTypes.APIGuildChannel} channel
+ * @param {DiscordTypes.APIGuild} guild
  * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
-async function sendMessage(message, guild, row) {
+async function sendMessage(message, channel, guild, row) {
 	const roomID = await createRoom.ensureRoom(message.channel_id)
 
 	let senderMxid = null
 	if (!dUtils.isWebhookMessage(message)) {
-		if (message.member) { // available on a gateway message create event
-			senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID)
+		if (message.author.id === discord.application.id) {
+			// no need to sync the bot's own user
+		} else if (message.member) { // available on a gateway message create event
+			senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID)
 		} else { // well, good enough...
 			senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
 		}

d2m/converters/edit-to-changes.js

@@ -3,13 +3,24 @@
 const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select, from} = passthrough
+const {sync, select, from} = passthrough
 /** @type {import("./message-to-event")} */
 const messageToEvent = sync.require("../converters/message-to-event")
-/** @type {import("../actions/register-user")} */
-const registerUser = sync.require("../actions/register-user")
-/** @type {import("../actions/create-room")} */
-const createRoom = sync.require("../actions/create-room")
+/** @type {import("../../m2d/converters/utils")} */
+const utils = sync.require("../../m2d/converters/utils")
+
+function eventCanBeEdited(ev) {
+	// Discord does not allow files, images, attachments, or videos to be edited.
+	if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
+		return false
+	}
+	// Discord does not allow stickers to be edited.
+	if (ev.old.event_type === "m.sticker") {
+		return false
+	}
+	// Anything else is fair game.
+	return true
+}
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
@@ -19,12 +30,27 @@ const createRoom = sync.require("../actions/create-room")
  * @param {import("../../matrix/api")} api simple-as-nails dependency injection for the matrix API
  */
 async function editToChanges(message, guild, api) {
+	// If it is a user edit, allow deleting old messages (e.g. they might have removed text from an image). If it is the system adding a generated embed to a message, don't delete old messages since the system only sends partial data.
+
+	const isGeneratedEmbed = !("content" in message)
+
 	// Figure out what events we will be replacing
 
 	const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
 	assert(roomID)
 	/** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */
-	const senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
+	let senderMxid = null
+	if (message.author) {
+		senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
+	} else {
+		// Should be a system generated embed. We want the embed to be sent by the same user who sent the message, so that the messages get grouped in most clients.
+		const eventID = select("event_message", "event_id", {message_id: message.id}).pluck().get()
+		assert(eventID) // this should have been checked earlier in a calling function
+		const event = await api.getEvent(roomID, eventID)
+		if (utils.eventSenderIsFromDiscord(event.sender)) {
+			senderMxid = event.sender
+		}
+	}
 
 	const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all()
 
@@ -48,7 +74,8 @@ async function editToChanges(message, guild, api) {
 	let eventsToRedact = []
 	/** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */
 	let eventsToSend = []
-	//  4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. This is represented as nothing.
+	/**  4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. */
+	let unchangedEvents = []
 
 	function shift() {
 		newFallbackContent.shift()
@@ -81,19 +108,36 @@ async function editToChanges(message, guild, api) {
 		shift()
 	}
 	// Anything remaining in oldEventRows is present in the old version only and should be redacted.
-	eventsToRedact = oldEventRows
+	eventsToRedact = oldEventRows.map(e => ({old: e}))
+
+	// If this is a generated embed update, only allow the embeds to be updated, since the system only sends data about events. Ignore changes to other things.
+	if (isGeneratedEmbed) {
+		unchangedEvents.push(...eventsToRedact.filter(e => e.old.event_subtype !== "m.notice")) // Move them from eventsToRedact to unchangedEvents.
+		eventsToRedact = eventsToRedact.filter(e => e.old.event_subtype === "m.notice")
+	}
+
+	// Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed!
+	// (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.)
+	// So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed.
+	unchangedEvents.push(...eventsToReplace.filter(ev => !eventCanBeEdited(ev))) // Move them from eventsToRedact to unchangedEvents.
+	eventsToReplace = eventsToReplace.filter(eventCanBeEdited)
 
 	// We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times.
 	/** @type {({column: string, eventID: string} | {column: string, nextEvent: true})[]} */
 	const promotions = []
 	for (const column of ["part", "reaction_part"]) {
+		const candidatesForParts = unchangedEvents.concat(eventsToReplace)
 		// If no events with part = 0 exist (or will exist), we need to do some management.
-		if (!eventsToReplace.some(e => e.old[column] === 0)) {
-			if (eventsToReplace.length) {
+		if (!candidatesForParts.some(e => e.old[column] === 0)) {
+			if (candidatesForParts.length) {
 				// We can choose an existing event to promote. Bigger order is better.
-				const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text")
-				eventsToReplace.sort((a, b) => order(b) - order(a))
-				promotions.push({column, eventID: eventsToReplace[0].old.event_id})
+				const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.old.event_subtype === "m.text")
+				candidatesForParts.sort((a, b) => order(b) - order(a))
+				if (column === "part") {
+					promotions.push({column, eventID: candidatesForParts[0].old.event_id}) // part should be the first one
+				} else {
+					promotions.push({column, eventID: candidatesForParts[candidatesForParts.length - 1].old.event_id}) // reaction_part should be the last one
+				}
 			} else {
 				// No existing events to promote, but new events are being sent. Whatever gets sent will be the next part = 0.
 				promotions.push({column, nextEvent: true})
@@ -101,24 +145,8 @@ async function editToChanges(message, guild, api) {
 		}
 	}
 
-	// Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed!
-	// (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.)
-	// So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed.
-	eventsToReplace = eventsToReplace.filter(ev => {
-		// Discord does not allow files, images, attachments, or videos to be edited.
-		if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
-			return false
-		}
-		// Discord does not allow stickers to be edited.
-		if (ev.old.event_type === "m.sticker") {
-			return false
-		}
-		// Anything else is fair game.
-		return true
-	})
-
 	// Removing unnecessary properties before returning
-	eventsToRedact = eventsToRedact.map(e => e.event_id)
+	eventsToRedact = eventsToRedact.map(e => e.old.event_id)
 	eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)}))
 
 	return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions}

d2m/converters/edit-to-changes.test.js

@@ -175,3 +175,98 @@ test("edit2changes: edit of reply to skull webp attachment with content", async
 		}
 	}])
 })
+
+test("edit2changes: edits the text event when multiple rows have part = 0 (should never happen in real life, but make sure the safety net works)", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_0, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* only the content can be edited",
+			"m.mentions": {},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "only the content can be edited",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999"
+			}
+		}
+	}])
+})
+
+test("edit2changes: promotes the text event when multiple rows have part = 1 (should never happen in real life, but make sure the safety net works)", async t => {
+	const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_1, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* only the content can be edited",
+			"m.mentions": {},
+			// *** Replaced With: ***
+			"m.new_content": {
+				msgtype: "m.text",
+				body: "only the content can be edited",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111"
+			}
+		}
+	}])
+	t.deepEqual(promotions, [
+		{
+			column: "part",
+			eventID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111"
+		},
+		{
+			column: "reaction_part",
+			eventID: "$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd111"
+		}
+	])
+})
+
+test("edit2changes: generated embed", async t => {
+	let called = 0
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.embed_generated_social_media_image, data.guild.general, {
+		async getEvent(roomID, eventID) {
+			called++
+			t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+			t.equal(eventID, "$mPSzglkCu-6cZHbYro0RW2u5mHvbH9aXDjO5FCzosc0")
+			return {sender: "@_ooye_cadence:cadence.moe"}
+		}
+	})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToReplace, [])
+	t.deepEqual(eventsToSend, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via hthrflwrs on cohost"
+			+ "\n| \n| ## This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO https://cohost.org/jkap/post/4794219-empty"
+			+ "\n| \n| 1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:"
+			+ "\n| \n|  * Both players draw eight cards"
+			+ "\n|  * Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand"
+			+ "\n|  * Both players present their best five-or-less-card pok...",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub>hthrflwrs on cohost</sub>`
+			+ `</p><p><strong><a href="https://cohost.org/jkap/post/4794219-empty">This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO</a></strong>`
+			+ `</p><p>1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:`
+			+ `<br><br><ul><li>Both players draw eight cards`
+			+ `</li><li>Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand`
+			+ `</li><li>Both players present their best five-or-less-card pok...</li></ul></p></blockquote>`,
+		"m.mentions": {}
+	}])
+	t.deepEqual(promotions, []) // TODO: it would be ideal to promote this to reaction_part = 0. this is OK to do because the main message won't have had any reactions yet.
+	t.equal(senderMxid, "@_ooye_cadence:cadence.moe")
+	t.equal(called, 1)
+})

d2m/converters/lottie.js

@@ -1,5 +1,6 @@
 // @ts-check
 
+const assert = require("assert")
 const stream = require("stream")
 const {PNG} = require("pngjs")
 
@@ -27,7 +28,7 @@ async function convert(text) {
 	/** @type RlottieWasm */
 	const rh = new r.RlottieWasm()
 	const status = rh.load(text)
-	if (!status) throw new Error(`Rlottie unable to load ${text.length} byte data file.`)
+	assert(status, `Rlottie unable to load ${text.length} byte data file.`)
 	const rendered = rh.render(0, SIZE, SIZE)
 	let png = new PNG({
 		width: SIZE,
@@ -38,11 +39,9 @@ async function convert(text) {
 		inputHasAlpha: true,
 	})
 	png.data = Buffer.from(rendered)
-	// The transform stream is necessary because PNG requires me to pipe it somewhere before this event loop ends
-	const resultStream = png.pack()
-	const p = new stream.PassThrough()
-	resultStream.pipe(p)
-	return p
+	// png.pack() is a bad stream and will throw away any data it sends if it's not connected to a destination straight away.
+	// We use Duplex.from to convert it into a good stream.
+	return stream.Duplex.from(png.pack())
 }
 
 module.exports.convert = convert

d2m/converters/message-to-event.embeds.test.js

@@ -27,7 +27,6 @@ test("message2event embeds: reply with just an embed", async t => {
 		msgtype: "m.notice",
 		"m.mentions": {},
 		body: "| ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
-			+ "\n| \n| ## https://twitter.com/i/status/1707484191963648161"
 			+ "\n| \n| does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
 			+ "\n| \n| ### Retweets"
 			+ "\n| 119"
@@ -35,8 +34,7 @@ test("message2event embeds: reply with just an embed", async t => {
 			+ "\n| 5581"
 			+ "\n| — Twitter",
 		format: "org.matrix.custom.html",
-		formatted_body: '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong></p>'
-			+ '<p><strong><a href="https://twitter.com/i/status/1707484191963648161">https://twitter.com/i/status/1707484191963648161</a></strong>'
+		formatted_body: '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong>'
 			+ '</p><p>does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
 			+ '</p><p><strong>Retweets</strong><br>119</p><p><strong>Likes</strong><br>5581</p>— Twitter</blockquote>'
 	}])
@@ -141,3 +139,180 @@ test("message2event embeds: crazy html is all escaped", async t => {
 		"m.mentions": {}
 	}])
 })
+
+test("message2event embeds: title without url", async t => {
+	const events = await messageToEvent(data.message_with_embeds.title_without_url, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## Hi, I'm Amanda!\n| \n| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><strong>Hi, I'm Amanda!</strong></p><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: url without title", async t => {
+	const events = await messageToEvent(data.message_with_embeds.url_without_title, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: author without url", async t => {
+	const events = await messageToEvent(data.message_with_embeds.author_without_url, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| ## Amanda\n| \n| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><strong>Amanda</strong></p><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: author url without name", async t => {
+	const events = await messageToEvent(data.message_with_embeds.author_url_without_name, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| I condone pirating music!",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p>I condone pirating music!</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: vx image", async t => {
+	const events = await messageToEvent(data.message_with_embeds.vx_image, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://vxtwitter.com/TomorrowCorp/status/1760330671074287875 we got a release date!!!",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://vxtwitter.com/TomorrowCorp/status/1760330671074287875">https://vxtwitter.com/TomorrowCorp/status/1760330671074287875</a> we got a release date!!!',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via vxTwitter / fixvx https://github.com/dylanpdx/BetterTwitFix"
+			+ "\n| "
+			+ "\n| ## Twitter https://twitter.com/tomorrowcorp/status/1760330671074287875"
+			+ "\n| "
+			+ "\n| ## Tomorrow Corporation (@TomorrowCorp) https://vxtwitter.com/TomorrowCorp/status/1760330671074287875"
+			+ "\n| "
+			+ "\n| Mark your calendar with a wet black stain! World of Goo 2 releases on May 23, 2024 on Nintendo Switch, Epic Games Store (Win/Mac), and http://WorldOfGoo2.com (Win/Mac/Linux)."
+			+ "\n| "
+			+ "\n| https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms"
+			+ "\n| "
+			+ "\n| 💖 123 🔁 36"
+			+ "\n| "
+			+ "\n| 📸 https://pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://github.com/dylanpdx/BetterTwitFix">vxTwitter / fixvx</a></sub>`
+			+ `</p><p><strong><a href="https://twitter.com/tomorrowcorp/status/1760330671074287875">Twitter</a></strong>`
+			+ `</p><p><strong><a href="https://vxtwitter.com/TomorrowCorp/status/1760330671074287875">Tomorrow Corporation (@TomorrowCorp)</a></strong>`
+			+ `</p><p>Mark your calendar with a wet black stain! World of Goo 2 releases on May 23, 2024 on Nintendo Switch, Epic Games Store (Win/Mac), and <a href="http://WorldOfGoo2.com">http://WorldOfGoo2.com</a> (Win/Mac/Linux).`
+			+ `<br><br><a href="https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms">https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms</a>`
+			+ `<br><br>💖 123 🔁 36`
+			+ `</p><p>📸 https://pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: vx video", async t => {
+	const events = await messageToEvent(data.message_with_embeds.vx_video, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://vxtwitter.com/McDonalds/status/1759971752254341417",
+		format: "org.matrix.custom.html",
+		formatted_body: '<a href="https://vxtwitter.com/McDonalds/status/1759971752254341417">https://vxtwitter.com/McDonalds/status/1759971752254341417</a>',
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via vxTwitter / fixvx https://github.com/dylanpdx/BetterTwitFix"
+			+ "\n| \n| ## McDonald’s🤝@studiopierrot"
+			+ "\n| \n| 💖 89 🔁 21 https://twitter.com/McDonalds/status/1759971752254341417"
+			+ "\n| \n| ## McDonald's (@McDonalds) https://vxtwitter.com/McDonalds/status/1759971752254341417"
+			+ "\n| \n| 🎞️ https://video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4?tag=12",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://github.com/dylanpdx/BetterTwitFix">vxTwitter / fixvx</a></sub>`
+			+ `</p><p><strong><a href="https://twitter.com/McDonalds/status/1759971752254341417">McDonald’s🤝@studiopierrot\n\n💖 89 🔁 21</a></strong>`
+			+ `</p><p><strong><a href="https://vxtwitter.com/McDonalds/status/1759971752254341417">McDonald's (@McDonalds)</a></strong>`
+			+ `</p><p>🎞️ https://video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4?tag=12</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: youtube video", async t => {
+	const events = await messageToEvent(data.message_with_embeds.youtube_video, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E\n\n\nJutomi I'm gonna make these sounds in your walls tonight",
+		format: "org.matrix.custom.html",
+		formatted_body: `<a href="https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E">https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E</a><br><br><br>Jutomi I'm gonna make these sounds in your walls tonight`,
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "| via YouTube https://www.youtube.com"
+			+ "\n| \n| ## Happy O Funny https://www.youtube.com/channel/UCEpQ9aEb1NafpvWp5Aoizrg"
+      	+ "\n| \n| ## Shoebill stork clattering sounds like machine guun~!! (Japan Matsue... https://www.youtube.com/watch?v=kDMHHw8JqLE"
+			+ "\n| \n| twitter"
+			+ "\n| https://twitter.com/matsuevogelpark"
+			+ "\n| \n| The shoebill (Balaeniceps rex) also known as whalehead, whale-headed stork, or shoe-billed stork, is a very large stork-like bird. It derives its name from its enormous shoe-shaped bill"
+			+ "\n| some people also called them the living dinosaur~~"
+			+ "\n| \n| #shoebill #livingdinosaur #happyofunny #weirdcreature #weirdsoun..."
+			+ "\n| \n| 🎞️ https://www.youtube.com/embed/kDMHHw8JqLE",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote><p><sub><a href="https://www.youtube.com">YouTube</a></sub></p>`
+			+ `<p><strong><a href="https://www.youtube.com/channel/UCEpQ9aEb1NafpvWp5Aoizrg">Happy O Funny</a></strong>`
+			+ `</p><p><strong><a href="https://www.youtube.com/watch?v=kDMHHw8JqLE">Shoebill stork clattering sounds like machine guun~!! (Japan Matsue...</a></strong>`
+			+ `</p><p>twitter<br><a href="https://twitter.com/matsuevogelpark">https://twitter.com/matsuevogelpark</a><br><br>The shoebill (Balaeniceps rex) also known as whalehead, whale-headed stork, or shoe-billed stork, is a very large stork-like bird. It derives its name from its enormous shoe-shaped bill<br>some people also called them the living dinosaur~~<br><br>#shoebill #livingdinosaur #happyofunny #weirdcreature #weirdsoun...`
+			+ `</p><p>🎞️ https://www.youtube.com/embed/kDMHHw8JqLE`
+			+ `</p></blockquote>`,
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: if discord creates an embed preview for a discord channel link, don't copy that embed", async t => {
+	const events = await messageToEvent(data.message_with_embeds.discord_server_included_punctuation_bad_discord, data.guild.general, {}, {
+		api: {
+			async getStateEvent(roomID, type, key) {
+				t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe")
+				t.equal(type, "m.room.power_levels")
+				t.equal(key, "")
+				return {
+					users: {
+						"@_ooye_bot:cadence.moe": 100
+					}
+				}
+			},
+			async getJoinedMembers(roomID) {
+				t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe")
+				return {
+					joined: {
+						"@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null},
+						"@user:matrix.org": {display_name: null, avatar_url: null}
+					}
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "(test https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&via=matrix.org)",
+		format: "org.matrix.custom.html",
+		formatted_body: `(test <a href="https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&amp;via=matrix.org">https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&amp;via=matrix.org</a>)`,
+		"m.mentions": {}
+	}])
+})

d2m/converters/message-to-event.js

@@ -78,10 +78,14 @@ function getDiscordParseCallbacks(message, guild, useHTML) {
 				return `@${role.name}:`
 			}
 		},
-		everyone: node =>
-			"@room",
-		here: node =>
-			"@here"
+		everyone: () => {
+			if (message.mention_everyone) return "@room"
+			return "@everyone"
+		},
+		here: () => {
+			if (message.mention_everyone) return "@room"
+			return "@here"
+		}
 	}
 }
 
@@ -199,6 +203,7 @@ async function attachmentToEvent(mentions, attachment) {
 async function messageToEvent(message, guild, options = {}, di) {
 	const events = []
 
+	/* c8 ignore next 7 */
 	if (message.type === DiscordTypes.MessageType.ThreadCreated) {
 		// This is the kind of message that appears when somebody makes a thread which isn't close enough to the message it's based off.
 		// It lacks the lines and the pill, so it looks kind of like a member join message, and it says:
@@ -244,6 +249,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 	let repliedToEventRow = null
 	let repliedToEventSenderMxid = null
 
+	if (message.mention_everyone) mentions.room = true
+
 	function addMention(mxid) {
 		if (!mentions.user_ids) mentions.user_ids = []
 		if (!mentions.user_ids.includes(mxid)) mentions.user_ids.push(mxid)
@@ -473,32 +480,35 @@ async function messageToEvent(message, guild, options = {}, di) {
 		message.content = "changed the channel name to **" + message.content + "**"
 	}
 
-	// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
-	const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
-	if (matches.length && matches.some(m => m[1].match(/[a-z]/i))) {
-		const writtenMentionsText = matches.map(m => m[1].toLowerCase())
-		const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
-		assert(roomID)
-		const {joined} = await di.api.getJoinedMembers(roomID)
-		for (const [mxid, member] of Object.entries(joined)) {
-			if (!userRegex.some(rx => mxid.match(rx))) {
-				const localpart = mxid.match(/@([^:]*)/)
-				assert(localpart)
-				const displayName = member.display_name || localpart[1]
-				if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+
+	if (message.content) {
+		// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
+		const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
+		if (matches.length && matches.some(m => m[1].match(/[a-z]/i) && m[1] !== "everyone" && m[1] !== "here")) {
+			const writtenMentionsText = matches.map(m => m[1].toLowerCase())
+			const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
+			assert(roomID)
+			const {joined} = await di.api.getJoinedMembers(roomID)
+			for (const [mxid, member] of Object.entries(joined)) {
+				if (!userRegex.some(rx => mxid.match(rx))) {
+					const localpart = mxid.match(/@([^:]*)/)
+					assert(localpart)
+					const displayName = member.display_name || localpart[1]
+					if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+				}
 			}
 		}
-	}
 
-	// Text content appears first
-	if (message.content) {
+		// Text content appears first
 		const {body, html} = await transformContent(message.content)
 		await addTextEvent(body, html, msgtype, {scanMentions: true})
 	}
 
 	// Then attachments
-	const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions)))
-	events.push(...attachmentEvents)
+	if (message.attachments) {
+		const attachmentEvents = await Promise.all(message.attachments.map(attachmentToEvent.bind(null, mentions)))
+		events.push(...attachmentEvents)
+	}
 
 	// Then embeds
 	for (const embed of message.embeds || []) {
@@ -506,13 +516,26 @@ async function messageToEvent(message, guild, options = {}, di) {
 			continue // Matrix's own URL previews are fine for images.
 		}
 
+		if (embed.url?.startsWith("https://discord.com/")) {
+			continue // If discord creates an embed preview for a discord channel link, don't copy that embed
+		}
+
 		// Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once
 		const rep = new mxUtils.MatrixStringBuilder()
 
+		// Provider
+		if (embed.provider?.name) {
+			if (embed.provider.url) {
+				rep.addParagraph(`via ${embed.provider.name} ${embed.provider.url}`, tag`<sub><a href="${embed.provider.url}">${embed.provider.name}</a></sub>`)
+			} else {
+				rep.addParagraph(`via ${embed.provider.name}`, tag`<sub>${embed.provider.name}</sub>`)
+			}
+		}
+
 		// Author and URL into a paragraph
 		let authorNameText = embed.author?.name || ""
 		if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image
-		if (authorNameText || embed.author?.url) {
+		if (authorNameText) {
 			if (embed.author?.url) {
 				const authorURL = await transformContentMessageLinks(embed.author.url)
 				rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`<strong><a href="${authorURL}">${authorNameText}</a></strong>`)
@@ -529,11 +552,11 @@ async function messageToEvent(message, guild, options = {}, di) {
 			} else {
 				rep.addParagraph(`## ${body}`, `<strong>${html}</strong>`)
 			}
-		} else if (embed.url) {
-			rep.addParagraph(`## ${embed.url}`, tag`<strong><a href="${embed.url}">${embed.url}</a></strong>`)
 		}
 
-		if (embed.description) {
+		let embedTypeShouldShowDescription = embed.type !== "video" // Discord doesn't display descriptions for videos
+		if (embed.provider?.name === "YouTube") embedTypeShouldShowDescription = true // But I personally like showing the descriptions for YouTube videos specifically
+		if (embed.description && embedTypeShouldShowDescription) {
 			const {body, html} = await transformContent(embed.description)
 			rep.addParagraph(body, html)
 		}
@@ -547,7 +570,11 @@ async function messageToEvent(message, guild, options = {}, di) {
 			rep.addParagraph(fieldRep.get().body, fieldRep.get().formatted_body)
 		}
 
-		if (embed.image?.url) rep.addParagraph(`📸 ${embed.image.url}`)
+		let chosenImage = embed.image?.url
+		// the thumbnail seems to be used for "article" type but displayed big at the bottom by discord
+		if (embed.type === "article" && embed.thumbnail?.url && !chosenImage) chosenImage = embed.thumbnail.url
+		if (chosenImage) rep.addParagraph(`📸 ${chosenImage}`)
+
 		if (embed.video?.url) rep.addParagraph(`🎞️ ${embed.video.url}`)
 
 		if (embed.footer?.text) rep.addLine(`— ${embed.footer.text}`, tag`— ${embed.footer.text}`)

d2m/converters/message-to-event.test.js

@@ -789,3 +789,63 @@ test("message2event: crossposted announcements say where they are crossposted fr
 		formatted_body: "🔀 <strong>Chewey Bot Official Server #announcements</strong><br>All text based commands are now inactive on Chewey Bot<br>To continue using commands you'll need to use them as slash commands"
 	}])
 })
+
+test("message2event: @everyone", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@room",
+		"m.mentions": {
+			room: true
+		}
+	}])
+})
+
+test("message2event: @here", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_here)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@room",
+		"m.mentions": {
+			room: true
+		}
+	}])
+})
+
+test("message2event: @everyone without permission", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone_without_permission)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@everyone <-- this is testing that it DOESN'T mention. if this mentions everyone then my apologies.",
+		format: "org.matrix.custom.html",
+		formatted_body: "@everyone &lt;-- this is testing that it DOESN'T mention. if this mentions everyone then my apologies.",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: @here without permission", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_here_without_permission)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "@here <-- this is testing that it DOESN'T mention. if this mentions people then my apologies.",
+		format: "org.matrix.custom.html",
+		formatted_body: "@here &lt;-- this is testing that it DOESN'T mention. if this mentions people then my apologies.",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event: @everyone within a link", async t => {
+	const events = await messageToEvent(data.message_mention_everyone.at_everyone_within_link)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: "https://github.com/@everyone",
+		format: "org.matrix.custom.html",
+		formatted_body: `<a href="https://github.com/@everyone">https://github.com/@everyone</a>`,
+		"m.mentions": {}
+	}])
+})

d2m/converters/remove-reaction.js

@@ -12,7 +12,7 @@ const utils = sync.require("../../m2d/converters/utils")
  * @typedef ReactionRemoveRequest
  * @prop {string} eventID
  * @prop {string | null} mxid
- * @prop {BigInt} [hash]
+ * @prop {bigint} [hash]
  */
 
 /**

d2m/converters/user-to-mxid.js

@@ -60,7 +60,7 @@ function userToSimName(user) {
 
 	// 1. Is sim user already registered?
 	const existing = select("sim", "sim_name", {user_id: user.id}).pluck().get()
-	if (existing) return existing
+	assert.equal(existing, null, "Shouldn't try to create a new name for an existing sim")
 
 	// 2. Register based on username (could be new or old format)
 	// (Unless it's a special user, in which case copy their provided mappings.)

d2m/event-dispatcher.js

@@ -115,8 +115,7 @@ module.exports = {
 			if (!member) return
 			if (!("permission_overwrites" in channel)) continue
 			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
-			if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel
+			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look back in this channel
 
 			/** More recent messages come first. */
 			// console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
@@ -164,8 +163,7 @@ module.exports = {
 
 			// Permissions check
 			const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
-			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
-			if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel
+			if (!dUtils.hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])) continue // We don't have permission to look up the pins in this channel
 
 			const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get()
 			if (!row) continue // Only care about already bridged channels
@@ -250,7 +248,7 @@ module.exports = {
 		if (affected) return
 
 		// @ts-ignore
-		await sendMessage.sendMessage(message, guild, row),
+		await sendMessage.sendMessage(message, channel, guild, row),
 		await discordCommandHandler.execute(message, channel, guild)
 	},
 
@@ -270,7 +268,8 @@ module.exports = {
 
 		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
 		// If the message content is a string then it includes all interesting fields and is meaningful.
-		if (typeof data.content === "string") {
+		// Otherwise, if there are embeds, then the system generated URL preview embeds.
+		if (typeof data.content === "string" || "embeds" in data) {
 			/** @type {DiscordTypes.GatewayMessageCreateDispatchData} */
 			// @ts-ignore
 			const message = data

db/migrations/0011-move-bridge-bot-to-real-id.up.js

@@ -0,0 +1,16 @@
+/*
+	a. If the bridge bot sim already has the correct ID:
+		- No rows updated.
+
+	b. If the bridge bot sim has the wrong ID but there's no duplicate:
+		- One row updated.
+
+	c. If the bridge bot sim has the wrong ID and there's a duplicate:
+		- One row updated (replaces an existing row).
+*/
+
+module.exports = async function(db) {
+	const config = require("../../config")
+	const id = Buffer.from(config.discordToken.split(".")[0], "base64").toString()
+	db.prepare("UPDATE OR REPLACE sim SET user_id = ? WHERE user_id = '0'").run(id)
+}

db/orm-defs.d.ts

@@ -100,7 +100,7 @@ export type Prepared<Row> = {
 	safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
 	raw: () => Prepared<Row[keyof Row][]>
 	all: (..._: any[]) => Row[]
-	get: (..._: any[]) => Row | null
+	get: (..._: any[]) => Row | null | undefined
 }
 
 export type AllKeys<U> = U extends any ? keyof U : never

db/orm.test.js

@@ -51,5 +51,5 @@ test("orm: from: join direction works", t => {
 	const hasNoOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
 	t.deepEqual(hasNoOwner, {user_id: "820865262526005258", proxy_owner_id: null})
 	const hasNoOwnerInner = from("sim").join("sim_proxy", "user_id", "inner").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
-	t.deepEqual(hasNoOwnerInner, null)
+	t.deepEqual(hasNoOwnerInner, undefined)
 })

discord/discord-command-handler.js

@@ -137,7 +137,7 @@ const commands = [{
 			// Check CREATE_INSTANT_INVITE permission
 			assert(message.member)
 			const guildPermissions = utils.getPermissions(message.member.roles, guild.roles)
-			if (!(guildPermissions & BigInt(1))) {
+			if (!(guildPermissions & DiscordTypes.PermissionFlagsBits.CreateInstantInvite)) {
 				return discord.snow.channel.createMessage(channel.id, {
 					...ctx,
 					content: "You don't have permission to invite people to this Discord server."

discord/utils.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 const DiscordTypes = require("discord-api-types/v10")
+const assert = require("assert").strict
 
 const EPOCH = 1420070400000
 
@@ -25,7 +26,7 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) {
 	}
 
 	if (channelOverwrites) {
-		/** @type {((overwrite: Required<DiscordTypes.APIGuildChannel>["permission_overwrites"][0]) => any)[]} */
+		/** @type {((overwrite: Required<DiscordTypes.APIOverwrite>) => any)[]} */
 		const actions = [
 			// Channel @everyone deny
 			overwrite => overwrite.id === everyoneID && (allowed &= ~BigInt(overwrite.deny)),
@@ -49,6 +50,48 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) {
 	return allowed
 }
 
+/**
+ * Note: You can only provide one permission bit to permissionToCheckFor. To check multiple permissions, call `hasAllPermissions` or `hasSomePermissions`.
+ * It is designed like this to avoid developer error with bit manipulations.
+ *
+ * @param {bigint} resolvedPermissions
+ * @param {bigint} permissionToCheckFor
+ * @returns {boolean} whether the user has the requested permission
+ * @example
+ * const permissions = getPermissions(userRoles, guildRoles, userID, channelOverwrites)
+ * hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel)
+ */
+function hasPermission(resolvedPermissions, permissionToCheckFor) {
+	// Make sure permissionToCheckFor has exactly one permission in it
+	assert.equal(permissionToCheckFor.toString(2).match(/1/g)?.length, 1)
+	// Do the actual calculation
+	return (resolvedPermissions & permissionToCheckFor) === permissionToCheckFor
+}
+
+/**
+ * @param {bigint} resolvedPermissions
+ * @param {(keyof DiscordTypes.PermissionFlagsBits)[]} permissionsToCheckFor
+ * @returns {boolean} whether the user has any of the requested permissions
+ * @example
+ * const permissions = getPermissions(userRoles, guildRoles, userID, channelOverwrites)
+ * hasSomePermissions(permissions, ["ViewChannel", "ReadMessageHistory"])
+ */
+function hasSomePermissions(resolvedPermissions, permissionsToCheckFor) {
+	return permissionsToCheckFor.some(x => hasPermission(resolvedPermissions, DiscordTypes.PermissionFlagsBits[x]))
+}
+
+/**
+ * @param {bigint} resolvedPermissions
+ * @param {(keyof DiscordTypes.PermissionFlagsBits)[]} permissionsToCheckFor
+ * @returns {boolean} whether the user has all of the requested permissions
+ * @example
+ * const permissions = getPermissions(userRoles, guildRoles, userID, channelOverwrites)
+ * hasAllPermissions(permissions, ["ViewChannel", "ReadMessageHistory"])
+ */
+function hasAllPermissions(resolvedPermissions, permissionsToCheckFor) {
+	return permissionsToCheckFor.every(x => hasPermission(resolvedPermissions, DiscordTypes.PermissionFlagsBits[x]))
+}
+
 /**
  * Command interaction responses have a webhook_id for some reason, but still have real author info of a real bot user in the server.
  * @param {DiscordTypes.APIMessage} message
@@ -69,6 +112,9 @@ function timestampToSnowflakeInexact(timestamp) {
 }
 
 module.exports.getPermissions = getPermissions
+module.exports.hasPermission = hasPermission
+module.exports.hasSomePermissions = hasSomePermissions
+module.exports.hasAllPermissions = hasAllPermissions
 module.exports.isWebhookMessage = isWebhookMessage
 module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact
 module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact

discord/utils.test.js

@@ -1,3 +1,4 @@
+const DiscordTypes = require("discord-api-types/v10")
 const {test} = require("supertape")
 const data = require("../test/data")
 const utils = require("./utils")
@@ -82,3 +83,27 @@ test("getPermissions: channel overwrite to allow role works", t => {
 	const want = BigInt(1 << 10 | 1 << 16)
 	t.equal((permissions & want), want)
 })
+
+test("hasSomePermissions: detects the permission", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.MentionEveryone | DiscordTypes.PermissionFlagsBits.BanMembers
+	const canRemoveMembers = utils.hasSomePermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, true)
+})
+
+test("hasSomePermissions: doesn't detect not the permission", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.MentionEveryone | DiscordTypes.PermissionFlagsBits.SendMessages
+	const canRemoveMembers = utils.hasSomePermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, false)
+})
+
+test("hasAllPermissions: detects the permissions", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.KickMembers | DiscordTypes.PermissionFlagsBits.BanMembers | DiscordTypes.PermissionFlagsBits.MentionEveryone
+	const canRemoveMembers = utils.hasAllPermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, true)
+})
+
+test("hasAllPermissions: doesn't detect not the permissions", t => {
+	const userPermissions = DiscordTypes.PermissionFlagsBits.MentionEveryone | DiscordTypes.PermissionFlagsBits.SendMessages | DiscordTypes.PermissionFlagsBits.KickMembers
+	const canRemoveMembers = utils.hasAllPermissions(userPermissions, ["KickMembers", "BanMembers"])
+	t.equal(canRemoveMembers, false)
+})

m2d/actions/channel-webhook.js

@@ -57,7 +57,7 @@ async function withWebhook(channelID, callback) {
  */
 async function sendMessageWithWebhook(channelID, data, threadID) {
 	const result = await withWebhook(channelID, async webhook => {
-		return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID, disableEveryone: true})
+		return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID})
 	})
 	return result
 }

m2d/actions/emoji-sheet.js

@@ -0,0 +1,36 @@
+// @ts-check
+
+const assert = require("assert")
+const fetch = require("node-fetch").default
+
+const utils = require("../converters/utils")
+const {sync} = require("../../passthrough")
+
+/** @type {import("../converters/emoji-sheet")} */
+const emojiSheetConverter = sync.require("../converters/emoji-sheet")
+
+/**
+ * Downloads the emoji from the web and converts to uncompressed PNG data.
+ * @param {string} mxc a single mxc:// URL
+ * @returns {Promise<Buffer | undefined>} uncompressed PNG data, or undefined if the downloaded emoji is not valid
+ */
+async function getAndConvertEmoji(mxc) {
+	const abortController = new AbortController()
+
+	const url = utils.getPublicUrlForMxc(mxc)
+	assert(url)
+
+	/** @type {import("node-fetch").Response} */
+	// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
+	// If we were using connection pooling, we would be forced to download the entire GIF.
+	// So we set no agent to ensure we are not connection pooling.
+	// @ts-ignore the signal is slightly different from the type it wants (still works fine)
+	const res = await fetch(url, {agent: false, signal: abortController.signal})
+	return emojiSheetConverter.convertImageStream(res.body, () => {
+		abortController.abort()
+		res.body.pause()
+		res.body.emit("end")
+	})
+}
+
+module.exports.getAndConvertEmoji = getAndConvertEmoji

m2d/actions/send-event.js

@@ -17,6 +17,10 @@ const eventToMessage = sync.require("../converters/event-to-message")
 const api = sync.require("../../matrix/api")
 /** @type {import("../../d2m/actions/register-user")} */
 const registerUser = sync.require("../../d2m/actions/register-user")
+/** @type {import("../../d2m/actions/edit-message")} */
+const editMessage = sync.require("../../d2m/actions/edit-message")
+/** @type {import("../actions/emoji-sheet")} */
+const emojiSheet = sync.require("../actions/emoji-sheet")
 
 /**
  * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message
@@ -75,7 +79,7 @@ async function sendEvent(event) {
 
 	// no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it
 
-	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch})
+	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch, mxcDownloader: emojiSheet.getAndConvertEmoji})
 
 	messagesToEdit = await Promise.all(messagesToEdit.map(async e => {
 		e.message = await resolvePendingFiles(e.message)
@@ -86,6 +90,7 @@ async function sendEvent(event) {
 	}))
 
 	let eventPart = 0 // 0 is primary, 1 is supporting
+	const pendingEdits = []
 
 	/** @type {DiscordTypes.APIMessage[]} */
 	const messageResponses = []
@@ -109,12 +114,33 @@ async function sendEvent(event) {
 
 		eventPart = 1
 		messageResponses.push(messageResponse)
+
+		/*
+			If the Discord system has a cached link preview embed for one of the links just sent,
+			it will be instantly added as part of `embeds` and there won't be a MESSAGE_UPDATE.
+			To reflect the generated embed back to Matrix, we pretend the message was updated right away.
+		*/
+		const sentEmbedsCount = message.embeds?.length || 0
+		if (messageResponse.embeds.length > sentEmbedsCount) {
+			// not awaiting here because requests to Matrix shouldn't block requests to Discord
+			pendingEdits.push(() =>
+				// @ts-ignore this is a valid message edit payload
+				editMessage.editMessage({
+					id: messageResponse.id,
+					channel_id: messageResponse.channel_id,
+					guild_id: guild.id,
+					embeds: messageResponse.embeds
+				}, guild, null)
+			)
+		}
 	}
 
 	for (const user of ensureJoined) {
 		registerUser.ensureSimJoined(user, event.room_id)
 	}
 
+	await Promise.all(pendingEdits.map(f => f())) // `await` will propagate any errors during editing
+
 	return messageResponses
 }
 

m2d/converters/emoji-sheet.js

@@ -5,8 +5,6 @@ const {pipeline} = require("stream").promises
 const sharp = require("sharp")
 const {GIFrame} = require("giframe")
 const {PNG} = require("pngjs")
-const utils = require("./utils")
-const fetch = require("node-fetch").default
 const streamMimeType = require("stream-mime-type")
 
 const SIZE = 48
@@ -16,27 +14,11 @@ const IMAGES_ACROSS = Math.floor(RESULT_WIDTH / SIZE)
 /**
  * Composite a bunch of Matrix emojis into a kind of spritesheet image to upload to Discord.
  * @param {string[]} mxcs mxc URLs, in order
+ * @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
  * @returns {Promise<Buffer>} PNG image
  */
-async function compositeMatrixEmojis(mxcs) {
-	const buffers = await Promise.all(mxcs.map(async mxc => {
-		const abortController = new AbortController()
-
-		const url = utils.getPublicUrlForMxc(mxc)
-		assert(url)
-
-		/** @type {import("node-fetch").Response} */
-		// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
-		// If we were using connection pooling, we would be forced to download the entire GIF.
-		// So we set no agent to ensure we are not connection pooling.
-		// @ts-ignore the signal is slightly different from the type it wants (still works fine)
-		const res = await fetch(url, {agent: false, signal: abortController.signal})
-		return convertImageStream(res.body, () => {
-			abortController.abort()
-			res.body.pause()
-			res.body.emit("end")
-		})
-	}))
+async function compositeMatrixEmojis(mxcs, mxcDownloader) {
+	const buffers = await Promise.all(mxcs.map(mxcDownloader))
 
 	// Calculate the size of the final composited image
 	const totalWidth = Math.min(buffers.length, IMAGES_ACROSS) * SIZE
@@ -128,4 +110,4 @@ async function convertImageStream(streamIn, stopStream) {
 }
 
 module.exports.compositeMatrixEmojis = compositeMatrixEmojis
-module.exports._convertImageStream = convertImageStream
+module.exports.convertImageStream = convertImageStream

m2d/converters/emoji-sheet.test.js

@@ -1,6 +1,5 @@
-const assert = require("assert").strict
 const {test} = require("supertape")
-const {_convertImageStream} = require("./emoji-sheet")
+const {convertImageStream} = require("./emoji-sheet")
 const fs = require("fs")
 const {Transform} = require("stream").Transform
 
@@ -27,28 +26,33 @@ class Meter extends Transform {
  * @param {import("supertape").Test} t
  * @param {string} path
  * @param {number} totalSize
+ * @param {number => boolean} sizeCheck
  */
-async function runSingleTest(t, path, totalSize) {
+async function runSingleTest(t, path, totalSize, sizeCheck) {
 	const file = fs.createReadStream(path)
 	const meter = new Meter()
 	const p = file.pipe(meter)
-	const result = await _convertImageStream(p, () => {
+	const result = await convertImageStream(p, () => {
 		file.pause()
 		file.emit("end")
 	})
-	t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `result was not a PNG file: ${result.toString("base64")}`)
+	t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `test that this is a PNG file: ${result.toString("base64").slice(0, 100)}`)
 	/* c8 ignore next 5 */
-	if (meter.bytes < totalSize / 4) { // should download less than 25% of each file
-		t.pass("intentionally read partial file")
+	if (sizeCheck(meter.bytes)) {
+		t.pass("read the correct amount of the file")
 	} else {
-		t.fail(`read more than 25% of file, read: ${meter.bytes}, total: ${totalSize}`)
+		t.fail(`read too much or too little of the file, read: ${meter.bytes}, total: ${totalSize}`)
 	}
 }
 
 slow()("emoji-sheet: only partial file is read for APNG", async t => {
-	await runSingleTest(t, "test/res/butterfly.png", 2438998)
+	await runSingleTest(t, "test/res/butterfly.png", 2438998, n => n < 2438998 / 4) // should download less than 25% of the file
 })
 
 slow()("emoji-sheet: only partial file is read for GIF", async t => {
-	await runSingleTest(t, "test/res/butterfly.gif", 781223)
+	await runSingleTest(t, "test/res/butterfly.gif", 781223, n => n < 781223 / 4) // should download less than 25% of the file
+})
+
+slow()("emoji-sheet: entire file is read for static PNG", async t => {
+	await runSingleTest(t, "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png", 11301, n => n === 11301) // should download entire file
 })

m2d/converters/event-to-message.js

@@ -126,12 +126,10 @@ turndownService.addRule("inlineLink", {
 		if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}`
 		if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>`
 		const href = node.getAttribute("href")
-		let brackets = ["", ""]
 		content = content.replace(/ @.*/, "")
-		if (href.startsWith("https://matrix.to")) brackets = ["<", ">"]
-		if (href === content) return brackets[0] + href + brackets[1]
+		if (href === content) return href
 		if (href.startsWith("https://matrix.to/#/@") && content[0] !== "@") content = "@" + content
-		return "[" + content + "](" + brackets[0] + href + brackets[1] + ")"
+		return "[" + content + "](" + href + ")"
 	}
 })
 
@@ -266,8 +264,8 @@ async function getMemberFromCacheOrHomeserver(roomID, mxid, api) {
 }
 
 /**
- * Splits a display name into one chunk containing <=80 characters, and another chunk containing the rest of the characters. Splits on
- * whitespace if possible.
+ * Splits a display name into one chunk containing <=80 characters (80 being how many characters Discord allows for the name of a webhook),
+ * and another chunk containing the rest of the characters. Splits on whitespace if possible.
  * These chunks, respectively, go in the display name, and at the top of the message.
  * If the second part isn't empty, it'll also contain boldening markdown and a line break at the end, so that regardless of its value it
  * can be prepended to the message content as-is.
@@ -306,8 +304,9 @@ function getUserOrProxyOwnerID(mxid) {
  * @param {string} content
  * @param {{id: string, name: string}[]} attachments
  * @param {({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} pendingFiles
+ * @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
  */
-async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles) {
+async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, mxcDownloader) {
 	if (!content.includes("<::>")) return content // No unknown emojis, nothing to do
 	// Remove known and unknown emojis from the end of the message
 	const r = /<a?:[a-zA-Z0-9_]*:[0-9]*>\s*$/
@@ -315,7 +314,7 @@ async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
 		content = content.replace(r, "")
 	}
 	// Create a sprite sheet of known and unknown emojis from the end of the message
-	const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis)
+	const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis, mxcDownloader)
 	// Attach it
 	const name = "emojis.png"
 	attachments.push({id: String(attachments.length), name})
@@ -385,19 +384,35 @@ async function handleRoomOrMessageLinks(input, di) {
 
 /**
  * @param {string} content
+ * @param {string} senderMxid
+ * @param {string} roomID
  * @param {DiscordTypes.APIGuild} guild
  * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di
  */
-async function checkWrittenMentions(content, guild, di) {
+async function checkWrittenMentions(content, senderMxid, roomID, guild, di) {
 	let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+
 	if (writtenMentionMatch) {
-		const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]})
-		if (results[0]) {
-			assert(results[0].user)
-			return {
-				// @ts-ignore - typescript doesn't know about indices yet
-				content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]),
-				ensureJoined: results[0].user
+		if (writtenMentionMatch[1] === "room") { // convert @room to @everyone
+			const powerLevels = await di.api.getStateEvent(roomID, "m.room.power_levels", "")
+			const userPower = powerLevels.users?.[senderMxid] || 0
+			if (userPower >= powerLevels.notifications?.room) {
+				return {
+					// @ts-ignore - typescript doesn't know about indices yet
+					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `@everyone` + content.slice(writtenMentionMatch.indices[1][1]),
+					ensureJoined: [],
+					allowedMentionsParse: ["everyone"]
+				}
+			}
+		} else {
+			const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]})
+			if (results[0]) {
+				assert(results[0].user)
+				return {
+					// @ts-ignore - typescript doesn't know about indices yet
+					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]),
+					ensureJoined: [results[0].user],
+					allowedMentionsParse: []
+				}
 			}
 		}
 	}
@@ -423,14 +438,12 @@ const attachmentEmojis = new Map([
 /**
  * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event
  * @param {import("discord-api-types/v10").APIGuild} guild
- * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di simple-as-nails dependency injection for the matrix API
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"], mxcDownloader: (mxc: string) => Promise<Buffer | undefined>}} di simple-as-nails dependency injection for the matrix API
  */
 async function eventToMessage(event, guild, di) {
-	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
-	let messages = []
-
 	let displayName = event.sender
 	let avatarURL = undefined
+	const allowedMentionsParse = ["users", "roles"]
 	/** @type {string[]} */
 	let messageIDsToEdit = []
 	let replyLine = ""
@@ -660,10 +673,11 @@ async function eventToMessage(event, guild, di) {
 				for (; node; node = node.nextSibling) {
 					// Check written mentions
 					if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) {
-						const result = await checkWrittenMentions(node.nodeValue, guild, di)
+						const result = await checkWrittenMentions(node.nodeValue, event.sender, event.room_id, guild, di)
 						if (result) {
 							node.nodeValue = result.content
-							ensureJoined.push(result.ensureJoined)
+							ensureJoined.push(...result.ensureJoined)
+							allowedMentionsParse.push(...result.allowedMentionsParse)
 						}
 					}
 					// Check for incompatible backticks in code blocks
@@ -709,6 +723,9 @@ async function eventToMessage(event, guild, di) {
 			// @ts-ignore bad type from turndown
 			content = turndownService.turndown(root)
 
+			// Put < > around any surviving matrix.to links to hide the URL previews
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^ )]*/g, "<$&>")
+
 			// It's designed for commonmark, we need to replace the space-space-newline with just newline
 			content = content.replace(/  \n/g, "\n")
 
@@ -716,7 +733,7 @@ async function eventToMessage(event, guild, di) {
 			if (replyLine && content.startsWith("> ")) content = "\n" + content
 
 			// SPRITE SHEET EMOJIS FEATURE:
-			content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
+			content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader)
 		} else {
 			// Looks like we're using the plaintext body!
 			content = event.content.body
@@ -725,12 +742,14 @@ async function eventToMessage(event, guild, di) {
 				content = `* ${displayName} ${content}`
 			}
 
-			content = await handleRoomOrMessageLinks(content, di)
+			content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^ )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews
 
-			const result = await checkWrittenMentions(content, guild, di)
+			const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di)
 			if (result) {
 				content = result.content
-				ensureJoined.push(result.ensureJoined)
+				ensureJoined.push(...result.ensureJoined)
+				allowedMentionsParse.push(...result.allowedMentionsParse)
 			}
 
 			// Markdown needs to be escaped, though take care not to escape the middle of links
@@ -783,11 +802,15 @@ async function eventToMessage(event, guild, di) {
 
 	// Split into 2000 character chunks
 	const chunks = chunk(content, 2000)
-	messages = messages.concat(chunks.map(content => ({
+	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
+	const messages = chunks.map(content => ({
 		content,
+		allowed_mentions: {
+			parse: allowedMentionsParse
+		},
 		username: displayNameShortened,
 		avatar_url: avatarURL
-	})))
+	}))
 
 	if (attachments.length) {
 		// If content is empty (should be the case when uploading a file) then chunk-text will create 0 messages.

matrix/api.js

@@ -121,6 +121,19 @@ function getJoinedMembers(roomID) {
 	return mreq.mreq("GET", `/client/v3/rooms/${roomID}/joined_members`)
 }
 
+/**
+ * @param {string} roomID
+ * @param {{from?: string, limit?: any}} pagination
+ * @returns {Promise<Ty.HierarchyPagination<Ty.R.Hierarchy>>}
+ */
+function getHierarchy(roomID, pagination) {
+	let path = `/client/v1/rooms/${roomID}/hierarchy`
+	if (!pagination.from) delete pagination.from
+	if (!pagination.limit) pagination.limit = 50
+	path += `?${new URLSearchParams(pagination)}`
+	return mreq.mreq("GET", path)
+}
+
 /**
  * @param {string} roomID
  * @param {string} eventID
@@ -239,6 +252,7 @@ module.exports.getEventForTimestamp = getEventForTimestamp
 module.exports.getAllState = getAllState
 module.exports.getStateEvent = getStateEvent
 module.exports.getJoinedMembers = getJoinedMembers
+module.exports.getHierarchy = getHierarchy
 module.exports.getRelations = getRelations
 module.exports.sendState = sendState
 module.exports.sendEvent = sendEvent

matrix/kstate.js

@@ -2,7 +2,7 @@
 
 const assert = require("assert").strict
 const mixin = require("mixin-deep")
-const deepEqual = require("deep-equal")
+const {isDeepStrictEqual} = require("util")
 
 /** Mutates the input. */
 function kstateStripConditionals(kstate) {
@@ -51,14 +51,14 @@ function diffKState(actual, target) {
 			// Special handling for power levels, we want to deep merge the actual and target into the final state.
 			if (!(key in actual)) throw new Error(`want to apply a power levels diff, but original power level data is missing\nstarted with:  ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`)
 			const temp = mixin({}, actual[key], target[key])
-			if (!deepEqual(actual[key], temp, {strict: true})) {
+			if (!isDeepStrictEqual(actual[key], temp)) {
 				// they differ. use the newly prepared object as the diff.
 				diff[key] = temp
 			}
 
 		} else if (key in actual) {
 			// diff
-			if (!deepEqual(actual[key], target[key], {strict: true})) {
+			if (!isDeepStrictEqual(actual[key], target[key])) {
 				// they differ. use the target as the diff.
 				diff[key] = target[key]
 			}

matrix/kstate.test.js

@@ -1,3 +1,4 @@
+const assert = require("assert")
 const {kstateToState, stateToKState, diffKState, kstateStripConditionals} = require("./kstate")
 const {test} = require("supertape")
 
@@ -162,3 +163,29 @@ test("diffKState: power levels are mixed together", t => {
 	})
 	t.notDeepEqual(original, result)
 })
+
+test("diffKState: cannot merge power levels if original power levels are missing", t => {
+	const original = {}
+	assert.throws(() =>
+		diffKState(original, {
+			"m.room.power_levels/": {
+				"events": {
+					"m.room.avatar": 0
+				}
+			}
+		})
+	, /original power level data is missing/)
+	t.pass()
+})
+
+test("diffKState: kstate keys must contain a slash separator", t => {
+	assert.throws(() =>
+		diffKState({
+			"m.room.name/": {name: "test name"},
+		}, {
+			"m.room.name/": {name: "test name"},
+			"new": {a: 2}
+		})
+	, /does not contain a slash separator/)
+	t.pass()
+})

package.json

@@ -18,12 +18,11 @@
     "@chriscdn/promise-semaphore": "^2.0.1",
     "better-sqlite3": "^9.0.0",
     "chunk-text": "^2.0.1",
-    "cloudstorm": "^0.10.7",
-    "deep-equal": "^2.2.3",
+    "cloudstorm": "^0.10.8",
     "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#2881b447954fcea10510f212fa4c1dbbdc0a57a3",
     "entities": "^4.5.0",
     "get-stream": "^6.0.1",
-    "giframe": "github:cloudrac3r/giframe#v0.4.1",
+    "giframe": "github:cloudrac3r/giframe#v0.4.2",
     "heatsync": "^2.4.1",
     "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0",
     "js-yaml": "^4.1.0",
@@ -34,7 +33,7 @@
     "pngjs": "github:cloudrac3r/pngjs#v7.0.2",
     "prettier-bytes": "^1.0.4",
     "sharp": "^0.32.6",
-    "snowtransfer": "^0.10.4",
+    "snowtransfer": "^0.10.5",
     "stream-mime-type": "^1.0.2",
     "try-to-catch": "^3.0.1",
     "turndown": "^7.1.2",
@@ -44,15 +43,16 @@
     "@types/node": "^18.16.0",
     "@types/node-fetch": "^2.6.3",
     "c8": "^8.0.1",
+    "colorette": "^1.4.0",
     "cross-env": "^7.0.3",
     "discord-api-types": "^0.37.60",
-    "supertape": "^8.3.0",
+    "supertape": "^10.4.0",
     "tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d"
   },
   "scripts": {
     "addbot": "node addbot.js",
     "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap test/test.js | tap-dot",
-    "test-slow": "cross-env FORCE_COLOR=true SUPERTAPE_TIMEOUT=6000 supertape --no-check-assertions-count --format tap test/test.js -- --slow | tap-dot",
-    "cover": "c8 --skip-full -x db/migrations -x matrix/file.js -x matrix/api.js -x matrix/mreq.js -r html -r text supertape --no-check-assertions-count --format fail test/test.js -- --slow"
+    "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot",
+    "cover": "c8 --skip-full -x db/migrations -x matrix/file.js -x matrix/api.js -x matrix/mreq.js -x d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow"
   }
 }

readme.md

@@ -167,7 +167,6 @@ To get into the rooms on your Matrix account, either add yourself to `invite` in
 * (1) chunk-text: It does what I want.
 * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
 * (8) snowtransfer: Discord API library with bring-your-own-caching that I trust.
-* (0) deep-equal: It's already pulled in by supertape.
 * (1) discord-markdown: This is my fork!
 * (0) get-stream: Only needed if content_length_workaround is true.
 * (0) giframe: This is my fork!

scripts/capture-message-update-events.js

@@ -37,7 +37,7 @@ passthrough.discord = discord
 })()
 
 const events = new sqlite("scripts/events.db")
-const sql = "INSERT INTO \"update\" (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")"
+const sql = "INSERT INTO update_event (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")"
 console.log(sql)
 const prepared = events.prepare(sql)
 

scripts/seed.js

@@ -53,17 +53,19 @@ async function uploadAutoEmoji(guild, name, filename) {
 	const mxid = `@${reg.sender_localpart}:${reg.ooye.server_name}`
 
 	// ensure registration is correctly set...
-	assert(reg.sender_localpart.startsWith(reg.ooye.namespace_prefix)) // appservice's localpart must be in the namespace it controls
-	assert(utils.eventSenderIsFromDiscord(mxid)) // appservice's mxid must be in the namespace it controls
-	assert(reg.ooye.server_origin.match(/^https?:\/\//)) // must start with http or https
-	assert.notEqual(reg.ooye.server_origin.slice(-1), "/") // must not end in slash
+	assert(reg.sender_localpart.startsWith(reg.ooye.namespace_prefix), "appservice's localpart must be in the namespace it controls")
+	assert(utils.eventSenderIsFromDiscord(mxid), "appservice's mxid must be in the namespace it controls")
+	assert(reg.ooye.server_origin.match(/^https?:\/\//), "server origin must start with http or https")
+	assert.notEqual(reg.ooye.server_origin.slice(-1), "/", "server origin must not end in slash")
+	const botID = Buffer.from(config.discordToken.split(".")[0], "base64").toString()
+	assert(botID.match(/^[0-9]{10,}$/), "discord token must follow the correct format")
 	console.log("✅ Configuration looks good...")
 
 	// database ddl...
 	await migrate.migrate(db)
 
 	// add initial rows to database, like adding the bot to sim...
-	db.prepare("INSERT OR IGNORE INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run("0", reg.sender_localpart.slice(reg.ooye.namespace_prefix.length), reg.sender_localpart, mxid)
+	db.prepare("INSERT OR IGNORE INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(botID, reg.sender_localpart.slice(reg.ooye.namespace_prefix.length), reg.sender_localpart, mxid)
 
 	console.log("✅ Database is ready...")
 

test/data.js

@@ -47,6 +47,9 @@ module.exports = {
 				},
 				users: {
 					"@test_auto_invite:example.org": 100
+				},
+				notifications: {
+					room: 0
 				}
 			},
 			"chat.schildi.hide_ui/read_receipts": {hidden: true},
@@ -98,7 +101,6 @@ module.exports = {
 			icon: "a_f83622e09ead74f0c5c527fe241f8f8c",
 			emojis: [
 				{
-					version: 0,
 					roles: [],
 					require_colons: true,
 					name: "hippo",
@@ -108,7 +110,6 @@ module.exports = {
 					animated: false
 				},
 				{
-					version: 0,
 					roles: [],
 					require_colons: true,
 					name: "hipposcope",
@@ -121,7 +122,20 @@ module.exports = {
 			premium_subscription_count: 14,
 			roles: [
 				{
-					version: 1696964862461,
+					unicode_emoji: null,
+					tags: {},
+					position: 0,
+					permissions: '559623605575360',
+					name: '@everyone',
+					mentionable: false,
+					managed: false,
+					id: '112760669178241024',
+					icon: null,
+					hoist: false,
+					flags: 0,
+					color: 0
+				},
+				{
 					unicode_emoji: null,
 					tags: {},
 					position: 22,
@@ -135,7 +149,6 @@ module.exports = {
 					flags: 0,
 					color: 0
 				}, {
-					version: 1696964862776,
 					unicode_emoji: null,
 					tags: {},
 					position: 131,
@@ -149,7 +162,6 @@ module.exports = {
 					flags: 0,
 					color: 11076095
 				}, {
-					version: 1696964862698,
 					unicode_emoji: '🍂',
 					tags: {},
 					position: 102,
@@ -1925,6 +1937,163 @@ module.exports = {
 			webhook_id: "1195662438662680720"
 		}
 	},
+	message_mention_everyone: {
+		at_everyone: {
+			id: "1214510099058655252",
+			type: 0,
+			content: "@everyone",
+			channel_id: "1100319550446252084",
+			author: {
+				id: "772659086046658620",
+				username: "cadence.worm",
+				avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+				discriminator: "0",
+				public_flags: 0,
+				premium_type: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "cadence",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: true,
+			tts: false,
+			timestamp: "2024-03-05T09:49:32.122000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
+		at_here: {
+			id: "1214510192230797332",
+			type: 0,
+			content: "@here",
+			channel_id: "1100319550446252084",
+			author: {
+			  id: "772659086046658620",
+			  username: "cadence.worm",
+			  avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+			  discriminator: "0",
+			  public_flags: 0,
+			  premium_type: 0,
+			  flags: 0,
+			  banner: null,
+			  accent_color: null,
+			  global_name: "cadence",
+			  avatar_decoration_data: null,
+			  banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: true,
+			tts: false,
+			timestamp: "2024-03-05T09:49:54.336000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
+		at_everyone_without_permission: {
+			id: "1214510346623258654",
+			type: 0,
+			content: "@everyone <-- this is testing that it DOESN'T mention. if this mentions everyone then my apologies.",
+			channel_id: "112760669178241024",
+			author: {
+				id: "772659086046658620",
+				username: "cadence.worm",
+				avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+				discriminator: "0",
+				public_flags: 0,
+				premium_type: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "cadence",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2024-03-05T09:50:31.146000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
+		at_here_without_permission: {
+			id: "1214510346623258654",
+			type: 0,
+			content: "@here <-- this is testing that it DOESN'T mention. if this mentions people then my apologies.",
+			channel_id: "112760669178241024",
+			author: {
+				id: "772659086046658620",
+				username: "cadence.worm",
+				avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+				discriminator: "0",
+				public_flags: 0,
+				premium_type: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "cadence",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2024-03-05T09:50:31.146000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
+		at_everyone_within_link: {
+			id: "1214510225885888563",
+			type: 0,
+			content: "https://github.com/@everyone",
+			channel_id: "1100319550446252084",
+			author: {
+				id: "772659086046658620",
+				username: "cadence.worm",
+				avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+				discriminator: "0",
+				public_flags: 0,
+				premium_type: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "cadence",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2024-03-05T09:50:02.360000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		}
+	},
 	message_with_embeds: {
 		nothing_but_a_field: {
 			guild_id: "497159726455455754",
@@ -2123,6 +2292,185 @@ module.exports = {
 			attachments: [],
 			guild_id: "1150201337112449045"
 		},
+		vx_image: {
+			id: "1209926442981269544",
+			type: 0,
+			content: "https://vxtwitter.com/TomorrowCorp/status/1760330671074287875 we got a release date!!!",
+			channel_id: "288058913985789953",
+			author: {
+				id: "113340068197859328",
+				username: "kumaccino",
+				avatar: "b48302623a12bc7c59a71328f72ccb39",
+				discriminator: "0",
+				public_flags: 128,
+				premium_type: 0,
+				flags: 128,
+				banner: null,
+				accent_color: null,
+				global_name: "kumaccino",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "article",
+					url: "https://vxtwitter.com/TomorrowCorp/status/1760330671074287875",
+					title: "Tomorrow Corporation (@TomorrowCorp)",
+					description: "Mark your calendar with a wet black stain! World of Goo 2 releases on May 23, 2024 on Nintendo Switch, Epic Games Store (Win/Mac), and http://WorldOfGoo2.com (Win/Mac/Linux).\n" +
+						"\n" +
+						"https://tomorrowcorporation.com/posts/world-of-goo-2-now-with-100-more-release-dates-and-platforms\n" +
+						"\n" +
+						"💖 123 🔁 36",
+					color: 8388564,
+					author: {
+						name: "Twitter",
+						url: "https://twitter.com/tomorrowcorp/status/1760330671074287875"
+					},
+					provider: {
+						name: "vxTwitter / fixvx",
+						url: "https://github.com/dylanpdx/BetterTwitFix"
+					},
+					thumbnail: {
+						url: "https://pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg",
+						proxy_url: "https://images-ext-2.discordapp.net/external/eqA-NKoXzJ0Y_l-MlwN6shFDJibC0TbPxMNWSU5IpKY/https/pbs.twimg.com/media/GG3zUMGbIAAxs3h.jpg",
+						width: 1200,
+						height: 1200,
+						placeholder: "5SgKDwTIlqiPjIhzlspniIiNaN8It3AD",
+						placeholder_version: 1
+					}
+				}
+			],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2024-02-21T18:15:43.353000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
+		vx_video: {
+			id: "1209804622206599190",
+			type: 0,
+			content: "https://vxtwitter.com/McDonalds/status/1759971752254341417",
+			channel_id: "112760669178241024",
+			author: {
+				id: "113340068197859328",
+				username: "kumaccino",
+				avatar: "b48302623a12bc7c59a71328f72ccb39",
+				discriminator: "0",
+				public_flags: 128,
+				premium_type: 0,
+				flags: 128,
+				banner: null,
+				accent_color: null,
+				global_name: "kumaccino",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "video",
+					url: "https://vxtwitter.com/McDonalds/status/1759971752254341417",
+					title: "McDonald's (@McDonalds)",
+					description: "McDonald’s🤝@studiopierrot\n\n💖 89 🔁 21",
+					color: 8388564,
+					author: {
+						name: "McDonald’s🤝@studiopierrot\n\n💖 89 🔁 21",
+						url: "https://twitter.com/McDonalds/status/1759971752254341417"
+					},
+					provider: {
+						name: "vxTwitter / fixvx",
+						url: "https://github.com/dylanpdx/BetterTwitFix"
+					},
+					video: {
+						url: "https://video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4?tag=12",
+						proxy_url: "https://images-ext-1.discordapp.net/external/TInoGDskHFBRSQR0ErWEmvmzi75EO28aSyiEXs3SB8E/%3Ftag%3D12/https/video.twimg.com/ext_tw_video/1759967449548541952/pu/vid/avc1/1280x720/XN1LFIJqAFBdtaoh.mp4",
+						width: 1280,
+						height: 720,
+						placeholder: "AggGBIAIp4iGeYdxjHgAAAAAAA==",
+						placeholder_version: 1
+					}
+				}
+			],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2024-02-21T10:11:39.017000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
+		youtube_video: {
+			id: "1214383754479534100",
+			type: 0,
+			content: "https://youtu.be/kDMHHw8JqLE?si=NaqNjVTtXugHeG_E\n\n\nJutomi I'm gonna make these sounds in your walls tonight",
+			channel_id: "112760669178241024",
+			author: {
+				id: "1060361805152669766",
+				username: "occimyy",
+				avatar: "3bf268de3eab1c5441da9585534d8aa5",
+				discriminator: "0",
+				public_flags: 0,
+				premium_type: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "Occimyy",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "video",
+					url: "https://www.youtube.com/watch?v=kDMHHw8JqLE",
+					title: "Shoebill stork clattering sounds like machine guun~!! (Japan Matsue...",
+					description: "twitter\n" +
+						"https://twitter.com/matsuevogelpark\n" +
+						"\n" +
+						"The shoebill (Balaeniceps rex) also known as whalehead, whale-headed stork, or shoe-billed stork, is a very large stork-like bird. It derives its name from its enormous shoe-shaped bill\n" +
+						"some people also called them the living dinosaur~~\n" +
+						"\n" +
+						"#shoebill #livingdinosaur #happyofunny #weirdcreature #weirdsoun...",
+					color: 16711680,
+					author: {
+						name: "Happy O Funny",
+						url: "https://www.youtube.com/channel/UCEpQ9aEb1NafpvWp5Aoizrg"
+					},
+					provider: { name: "YouTube", url: "https://www.youtube.com" },
+					thumbnail: {
+						url: "https://i.ytimg.com/vi/kDMHHw8JqLE/maxresdefault.jpg",
+						proxy_url: "https://images-ext-1.discordapp.net/external/eEPOxZQXfTHqvPQJBWqsgG3wxTQN20b8LXqw3jSqyRM/https/i.ytimg.com/vi/kDMHHw8JqLE/maxresdefault.jpg",
+						width: 1280,
+						height: 720,
+						placeholder: "WAgSDIIIdIprl4h4h4dNoEoEaQ==",
+						placeholder_version: 1
+					},
+					video: {
+						url: "https://www.youtube.com/embed/kDMHHw8JqLE",
+						width: 1280,
+						height: 720,
+						placeholder: "WAgSDIIIdIprl4h4h4dNoEoEaQ==",
+						placeholder_version: 1
+					}
+				}
+			],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2024-03-05T01:27:29.227000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
 		image_embed_and_attachment: {
 			id: "1157854642810654821",
 			type: 0,
@@ -2282,6 +2630,286 @@ module.exports = {
 			edited_timestamp: null,
 			flags: 0,
 			components: []
+		},
+		title_without_url: {
+			guild_id: "497159726455455754",
+			mentions: [],
+			id: "1141934888862351440",
+			type: 20,
+			content: "",
+			channel_id: "497161350934560778",
+			author: {
+				id: "1109360903096369153",
+				username: "Amanda 🎵",
+				avatar: "d56cd1b26e043ae512edae2214962faa",
+				discriminator: "2192",
+				public_flags: 524288,
+				flags: 524288,
+				bot: true,
+				banner: null,
+				accent_color: null,
+				global_name: null,
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "rich",
+					color: 3092790,
+					title: "Hi, I'm Amanda!",
+					description: "I condone pirating music!"
+				}
+			],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-08-18T03:21:33.629000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: [],
+			application_id: "1109360903096369153",
+			interaction: {
+				id: "1141934887608254475",
+				type: 2,
+				name: "stats",
+				user: {
+					id: "320067006521147393",
+					username: "papiophidian",
+					avatar: "47a19b0445069b826e136da4df4259bb",
+					discriminator: "0",
+					public_flags: 4194880,
+					flags: 4194880,
+					banner: null,
+					accent_color: null,
+					global_name: "PapiOphidian",
+					avatar_decoration_data: null,
+					banner_color: null
+				}
+			},
+			webhook_id: "1109360903096369153"
+		},
+		url_without_title: {
+			guild_id: "497159726455455754",
+			mentions: [],
+			id: "1141934888862351440",
+			type: 20,
+			content: "",
+			channel_id: "497161350934560778",
+			author: {
+				id: "1109360903096369153",
+				username: "Amanda 🎵",
+				avatar: "d56cd1b26e043ae512edae2214962faa",
+				discriminator: "2192",
+				public_flags: 524288,
+				flags: 524288,
+				bot: true,
+				banner: null,
+				accent_color: null,
+				global_name: null,
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "rich",
+					color: 3092790,
+					url: "https://amanda.moe",
+					description: "I condone pirating music!"
+				}
+			],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-08-18T03:21:33.629000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: [],
+			application_id: "1109360903096369153",
+			interaction: {
+				id: "1141934887608254475",
+				type: 2,
+				name: "stats",
+				user: {
+					id: "320067006521147393",
+					username: "papiophidian",
+					avatar: "47a19b0445069b826e136da4df4259bb",
+					discriminator: "0",
+					public_flags: 4194880,
+					flags: 4194880,
+					banner: null,
+					accent_color: null,
+					global_name: "PapiOphidian",
+					avatar_decoration_data: null,
+					banner_color: null
+				}
+			},
+			webhook_id: "1109360903096369153"
+		},
+		author_without_url: {
+			guild_id: "497159726455455754",
+			mentions: [],
+			id: "1141934888862351440",
+			type: 20,
+			content: "",
+			channel_id: "497161350934560778",
+			author: {
+				id: "1109360903096369153",
+				username: "Amanda 🎵",
+				avatar: "d56cd1b26e043ae512edae2214962faa",
+				discriminator: "2192",
+				public_flags: 524288,
+				flags: 524288,
+				bot: true,
+				banner: null,
+				accent_color: null,
+				global_name: null,
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "rich",
+					color: 3092790,
+					author: {
+						name: "Amanda"
+					},
+					description: "I condone pirating music!"
+				}
+			],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-08-18T03:21:33.629000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: [],
+			application_id: "1109360903096369153",
+			interaction: {
+				id: "1141934887608254475",
+				type: 2,
+				name: "stats",
+				user: {
+					id: "320067006521147393",
+					username: "papiophidian",
+					avatar: "47a19b0445069b826e136da4df4259bb",
+					discriminator: "0",
+					public_flags: 4194880,
+					flags: 4194880,
+					banner: null,
+					accent_color: null,
+					global_name: "PapiOphidian",
+					avatar_decoration_data: null,
+					banner_color: null
+				}
+			},
+			webhook_id: "1109360903096369153"
+		},
+		author_url_without_name: {
+			guild_id: "497159726455455754",
+			mentions: [],
+			id: "1141934888862351440",
+			type: 20,
+			content: "",
+			channel_id: "497161350934560778",
+			author: {
+				id: "1109360903096369153",
+				username: "Amanda 🎵",
+				avatar: "d56cd1b26e043ae512edae2214962faa",
+				discriminator: "2192",
+				public_flags: 524288,
+				flags: 524288,
+				bot: true,
+				banner: null,
+				accent_color: null,
+				global_name: null,
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "rich",
+					color: 3092790,
+					author: {
+						url: "https://amanda.moe"
+					},
+					description: "I condone pirating music!"
+				}
+			],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-08-18T03:21:33.629000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: [],
+			application_id: "1109360903096369153",
+			interaction: {
+				id: "1141934887608254475",
+				type: 2,
+				name: "stats",
+				user: {
+					id: "320067006521147393",
+					username: "papiophidian",
+					avatar: "47a19b0445069b826e136da4df4259bb",
+					discriminator: "0",
+					public_flags: 4194880,
+					flags: 4194880,
+					banner: null,
+					accent_color: null,
+					global_name: "PapiOphidian",
+					avatar_decoration_data: null,
+					banner_color: null
+				}
+			},
+			webhook_id: "1109360903096369153"
+		},
+		discord_server_included_punctuation_bad_discord: {
+			id: "1221672425792606349",
+			type: 0,
+			content: "(test https://discord.com/channels/1160894080998461480/1160894080998461480/1202543413652881428)",
+			channel_id: "1160894080998461480",
+			author: {
+				id: "772659086046658620",
+				username: "cadence.worm",
+				avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+				discriminator: "0",
+				public_flags: 0,
+				premium_type: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "cadence",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "article",
+					url: "https://discord.com/channels/1160894080998461480/1160894080998461480/1202543413652881428)",
+					title: "Discord - A New Way to Chat with Friends & Communities",
+					description: "Discord is the easiest way to communicate over voice, video, and text.  Chat, hang out, and stay close with your friends and communities.",
+					provider: { name: "Discord" },
+					content_scan_version: 0
+				}
+			],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2024-03-25T04:10:03.885000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
 		}
 	},
 	message_update: {
@@ -2662,6 +3290,88 @@ module.exports = {
 				name: "pomu puff"
 			}]
 		},
+		edited_content_with_sticker_and_attachments_but_all_parts_equal_0: {
+			id: "1106366167788044451",
+			type: 0,
+			content: "only the content can be edited",
+			channel_id: "122155380120748034",
+			author: {
+				id: "113340068197859328",
+				username: "Cookie 🍪",
+				global_name: null,
+				display_name: null,
+				avatar: "b48302623a12bc7c59a71328f72ccb39",
+				discriminator: "7766",
+				public_flags: 128,
+				avatar_decoration: null
+			},
+			attachments: [{
+				id: "1106366167486038016",
+				filename: "image.png",
+				size: 127373,
+				url: "https://cdn.discordapp.com/attachments/122155380120748034/1106366167486038016/image.png",
+				proxy_url: "https://media.discordapp.net/attachments/122155380120748034/1106366167486038016/image.png",
+				width: 333,
+				height: 287,
+				content_type: "image/png"
+			}],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-05-11T23:44:09.690000+00:00",
+			edited_timestamp: "2023-05-11T23:44:19.690000+00:00",
+			flags: 0,
+			components: [],
+			sticker_items: [{
+				id: "1106323941183717586",
+				format_type: 1,
+				name: "pomu puff"
+			}]
+		},
+		edited_content_with_sticker_and_attachments_but_all_parts_equal_1: {
+			id: "1106366167788044452",
+			type: 0,
+			content: "only the content can be edited",
+			channel_id: "122155380120748034",
+			author: {
+				id: "113340068197859328",
+				username: "Cookie 🍪",
+				global_name: null,
+				display_name: null,
+				avatar: "b48302623a12bc7c59a71328f72ccb39",
+				discriminator: "7766",
+				public_flags: 128,
+				avatar_decoration: null
+			},
+			attachments: [{
+				id: "1106366167486038016",
+				filename: "image.png",
+				size: 127373,
+				url: "https://cdn.discordapp.com/attachments/122155380120748034/1106366167486038016/image.png",
+				proxy_url: "https://media.discordapp.net/attachments/122155380120748034/1106366167486038016/image.png",
+				width: 333,
+				height: 287,
+				content_type: "image/png"
+			}],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-05-11T23:44:09.690000+00:00",
+			edited_timestamp: "2023-05-11T23:44:19.690000+00:00",
+			flags: 0,
+			components: [],
+			sticker_items: [{
+				id: "1106323941183717586",
+				format_type: 1,
+				name: "pomu puff"
+			}]
+		},
 		edit_of_reply_to_skull_webp_attachment_with_content: {
 			type: 19,
 			tts: false,
@@ -2799,6 +3509,31 @@ module.exports = {
 				}
 			],
 			guild_id: "112760669178241024"
+		},
+		embed_generated_social_media_image: {
+			channel_id: "112760669178241024",
+			embeds: [
+				{
+					color: 8594767,
+					description: "1v1 physical card game. Each player gets one standard deck of cards with a different backing to differentiate. Every turn proceeds as follows:\n\n * Both players draw eight cards\n * Both players may choose up to eight cards to discard, then draw that number of cards to put back in their hand\n * Both players present their best five-or-less-card pok...",
+					provider: {
+						name: "hthrflwrs on cohost"
+					},
+					thumbnail: {
+						height: 1587,
+						placeholder: "GpoKP5BJZphshnhwmmmYlmh3l7+m+mwJ",
+						placeholder_version: 1,
+						proxy_url: "https://images-ext-2.discordapp.net/external/9vTXIzlXU4wyUZvWfmlmQkck8nGLUL-A090W4lWsZ48/https/staging.cohostcdn.org/avatar/292-6b64b03c-4ada-42f6-8452-109275bfe68d-profile.png",
+						url: "https://staging.cohostcdn.org/avatar/292-6b64b03c-4ada-42f6-8452-109275bfe68d-profile.png",
+						width: 1644
+					},
+					title: "This post nerdsniped me, so here's some RULES FOR REAL-LIFE BALATRO",
+					type: "link",
+					url: "https://cohost.org/jkap/post/4794219-empty"
+				}
+			],
+			guild_id: "112760669178241024",
+			id: "1210387798297682020"
 		}
 	},
 	special_message: {

test/ooye-test-data.sql

@@ -33,6 +33,8 @@ INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES
 
 INSERT INTO message_channel (message_id, channel_id) VALUES
 ('1106366167788044450', '122155380120748034'),
+('1106366167788044451', '122155380120748034'),
+('1106366167788044452', '122155380120748034'),
 ('1126786462646550579', '112760669178241024'),
 ('1128084748338741392', '112760669178241024'),
 ('1128084851279536279', '112760669178241024'),
@@ -51,7 +53,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES
 ('1158842413025071135', '176333891320283136'),
 ('1197612733600895076', '112760669178241024'),
 ('1202543413652881428', '1160894080998461480'),
-('1207486471489986620', '1160894080998461480');
+('1207486471489986620', '1160894080998461480'),
+('1210387798297682020', '112760669178241024');
 
 INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES
 ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1),
@@ -68,6 +71,12 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
 ('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1, 1),
 ('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 1, 0),
 ('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0, 0),
+('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999', 'm.room.message', 'm.text', '1106366167788044451', 0, 0, 1),
+('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZI999', 'm.room.message', 'm.image', '1106366167788044451', 0, 0, 1),
+('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd999', 'm.sticker', NULL, '1106366167788044451', 0, 0, 1),
+('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111', 'm.room.message', 'm.text', '1106366167788044452', 1, 1, 1),
+('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZI111', 'm.room.message', 'm.image', '1106366167788044452', 1, 1, 1),
+('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd111', 'm.sticker', NULL, '1106366167788044452', 1, 1, 1),
 ('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 0, 1),
 ('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0, 0),
 ('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0, 0),
@@ -79,7 +88,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
 ('$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo', 'm.room.message', 'm.text', '1158842413025071135', 0, 0, 1),
 ('$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw', 'm.room.message', 'm.text', '1197612733600895076', 0, 0, 1),
 ('$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU', 'm.room.message', 'm.text', '1202543413652881428', 0, 0, 0),
-('$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y', 'm.room.message', 'm.image', '1207486471489986620', 0, 0, 0);
+('$OEEK-Wam2FTh6J-6kVnnJ6KnLA_lLRnLTHatKKL62-Y', 'm.room.message', 'm.image', '1207486471489986620', 0, 0, 0),
+('$mPSzglkCu-6cZHbYro0RW2u5mHvbH9aXDjO5FCzosc0', 'm.room.message', 'm.text', '1210387798297682020', 0, 0, 1);
 
 INSERT INTO file (discord_url, mxc_url) VALUES
 ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),
@@ -104,6 +114,7 @@ INSERT INTO file (discord_url, mxc_url) VALUES
 INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES
 ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'),
 ('393635038903926784', 'hipposcope', 1, 'mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc'),
+('457898385297815911', 'emoji_from_unreachable_server', 0, 'mxc://cadence.moe/bZFuuUSEebJYXUMSxuuSuLTa'),
 ('362741439211503616', 'bn_re', 0, 'mxc://cadence.moe/OIpqpfxTnHKokcsYqDusxkBT'),
 ('551636841284108289', 'ae_botrac4r', 0, 'mxc://cadence.moe/skqfuItqxNmBYekzmVKyoLzs'),
 ('975572106295259148', 'brillillillilliant_move', 0, 'mxc://cadence.moe/scfRIDOGKWFDEBjVXocWYQHik'),

test/test.js

@@ -2,6 +2,7 @@
 
 const fs = require("fs")
 const {join} = require("path")
+const stp = require("stream").promises
 const sqlite = require("better-sqlite3")
 const migrate = require("../db/migrate")
 const HeatSync = require("heatsync")
@@ -10,6 +11,7 @@ const data = require("./data")
 /** @type {import("node-fetch").default} */
 // @ts-ignore
 const fetch = require("node-fetch")
+const {green} = require("colorette")
 
 const config = require("../config")
 const passthrough = require("../passthrough")
@@ -47,9 +49,52 @@ passthrough.from = orm.from
 passthrough.select = orm.select
 
 const file = sync.require("../matrix/file")
+/* c8 ignore next */
 file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) }
 
 ;(async () => {
+	/* c8 ignore start - maybe download some more test files in slow mode */
+	if (process.argv.includes("--slow")) {
+		test("test files: download", async t => {
+			/** @param {{url: string, to: string}[]} files */
+			async function allReporter(files) {
+				return new Promise(resolve => {
+					let resolved = 0
+					const report = files.map(file => file.to.split("/").slice(-1)[0][0])
+					files.map(download).forEach((p, i) => {
+						p.then(() => {
+							report[i] = green(".")
+							process.stderr.write("\r" + report.join(""))
+							if (++resolved === files.length) resolve(null)
+						})
+					})
+				})
+			}
+			async function download({url, to}) {
+				if (await fs.existsSync(to)) return
+				const res = await fetch(url)
+				await stp.pipeline(res.body, fs.createWriteStream(to, {encoding: "binary"}))
+			}
+			await allReporter([
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/RLMgJGfgTPjIQtvvWZsYjhjy", to: "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/bZFuuUSEebJYXUMSxuuSuLTa", to: "test/res/bZFuuUSEebJYXUMSxuuSuLTa.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/qWmbXeRspZRLPcjseyLmeyXC", to: "test/res/qWmbXeRspZRLPcjseyLmeyXC.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/wcouHVjbKJJYajkhJLsyeJAA", to: "test/res/wcouHVjbKJJYajkhJLsyeJAA.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/WbYqNlACRuicynBfdnPYtmvc", to: "test/res/WbYqNlACRuicynBfdnPYtmvc.gif"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/HYcztccFIPgevDvoaWNsEtGJ", to: "test/res/HYcztccFIPgevDvoaWNsEtGJ.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/lHfmJpzgoNyNtYHdAmBHxXix", to: "test/res/lHfmJpzgoNyNtYHdAmBHxXix.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/MtRdXixoKjKKOyHJGWLsWLNU", to: "test/res/MtRdXixoKjKKOyHJGWLsWLNU.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/HXfFuougamkURPPMflTJRxGc", to: "test/res/HXfFuougamkURPPMflTJRxGc.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ikYKbkhGhMERAuPPbsnQzZiX", to: "test/res/ikYKbkhGhMERAuPPbsnQzZiX.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/AYPpqXzVJvZdzMQJGjioIQBZ", to: "test/res/AYPpqXzVJvZdzMQJGjioIQBZ.png"},
+				{url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/UVuzvpVUhqjiueMxYXJiFEAj", to: "test/res/UVuzvpVUhqjiueMxYXJiFEAj.png"},
+				{url: "https://ezgif.com/images/format-demo/butterfly.gif", to: "test/res/butterfly.gif"},
+				{url: "https://ezgif.com/images/format-demo/butterfly.png", to: "test/res/butterfly.png"},
+			])
+		}, {timeout: 60000})
+	}
+	/* c8 ignore end */
+
 	const p = migrate.migrate(db)
 	test("migrate: migration works", async t => {
 		await p
@@ -64,26 +109,6 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
 
 	db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8"))
 
-	/* c8 ignore start - maybe download some more test files in slow mode */
-	if (process.argv.includes("--slow")) {
-		test("test files: download", async t => {
-			function download(url, to) {
-				return new Promise(async resolve => {
-					if (fs.existsSync(to)) return resolve(null)
-					const res = await fetch(url)
-					res.body.pipe(fs.createWriteStream(to, {encoding: "binary"}))
-					res.body.once("finish", resolve)
-				})
-			}
-			await Promise.all([
-				download("https://ezgif.com/images/format-demo/butterfly.png", "test/res/butterfly.png"),
-				download("https://ezgif.com/images/format-demo/butterfly.gif", "test/res/butterfly.gif")
-			])
-			t.pass("downloaded")
-		})
-	}
-	/* c8 ignore end */
-
 	require("../db/orm.test")
 	require("../discord/utils.test")
 	require("../matrix/kstate.test")

types.d.ts

@@ -257,6 +257,18 @@ export namespace R {
 	export type EventRedacted = {
 		event_id: string
 	}
+
+	export type Hierarchy = {
+		avatar_url?: string
+		canonical_alias?: string
+		children_state: {}
+		guest_can_join: boolean
+		join_rule?: string
+		name?: string
+		num_joined_members: number
+		room_id: string
+		room_type?: string
+	}
 }
 
 export type Pagination<T> = {
@@ -264,3 +276,8 @@ export type Pagination<T> = {
 	next_batch?: string
 	prev_match?: string
 }
+
+export type HierarchyPagination<T> = {
+	rooms: T[]
+	next_batch?: string
+}