Refactor kstate resource uploading

d2m/actions/create-room.js

@@ -51,8 +51,8 @@ async function roomToKState(roomID) {
  * @param {string} roomID
  * @param {any} kstate
  */
-function applyKStateDiffToRoom(roomID, kstate) {
-	const events = ks.kstateToState(kstate)
+async function applyKStateDiffToRoom(roomID, kstate) {
+	const events = await ks.kstateToState(kstate)
 	return Promise.all(events.map(({type, state_key, content}) =>
 		api.sendState(roomID, type, state_key, content)
 	))
@@ -220,7 +220,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 			preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
 			visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
 			invite: [],
-			initial_state: ks.kstateToState(kstate),
+			initial_state: await ks.kstateToState(kstate),
 			...spaceCreationContent
 		})
 

d2m/actions/create-space.js

@@ -45,7 +45,7 @@ async function createSpace(guild, kstate) {
 			creation_content: {
 				type: "m.space"
 			},
-			initial_state: ks.kstateToState(kstate)
+			initial_state: await ks.kstateToState(kstate)
 		})
 	})
 	db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, roomID)
@@ -57,15 +57,14 @@ async function createSpace(guild, kstate) {
  * @param {number} privacyLevel
  */
 async function guildToKState(guild, privacyLevel) {
-	const avatarEventContent = {}
-	if (guild.icon) {
-		avatarEventContent.discord_path = file.guildIcon(guild)
-		avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
-	}
-
+	assert.equal(typeof privacyLevel, "number")
 	const guildKState = {
 		"m.room.name/": {name: guild.name},
-		"m.room.avatar/": avatarEventContent,
+		"m.room.avatar/": {
+			$if: guild.icon,
+			discord_path: file.guildIcon(guild),
+			url: {$url: file.guildIcon(guild)}
+		},
 		"m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
 		"m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]},
 		"m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]},
@@ -123,7 +122,8 @@ async function _syncSpace(guild, shouldActuallySync) {
 		// don't try to update rooms with custom avatars though
 		const roomsWithCustomAvatars = select("channel_room", "room_id", {}, "WHERE custom_avatar IS NOT NULL").pluck().all()
 
-		const childRooms = ks.kstateToState(spaceKState).filter(({type, state_key, content}) => {
+		const state = await ks.kstateToState(spaceKState)
+		const childRooms = state.filter(({type, state_key, content}) => {
 			return type === "m.space.child" && "via" in content && !roomsWithCustomAvatars.includes(state_key)
 		}).map(({state_key}) => state_key)
 

d2m/actions/create-space.test.js

@@ -0,0 +1,39 @@
+// @ts-check
+
+const mixin = require("mixin-deep")
+const {guildToKState, ensureSpace} = require("./create-space")
+const {kstateStripConditionals, kstateUploadMxc} = require("../../matrix/kstate")
+const {test} = require("supertape")
+const testData = require("../../test/data")
+
+const passthrough = require("../../passthrough")
+const {db} = passthrough
+
+test("guild2space: can generate kstate for a guild, passing privacy level 0", async t => {
+	t.deepEqual(
+		await kstateUploadMxc(kstateStripConditionals(await guildToKState(testData.guild.general, 0))),
+		{
+			"m.room.avatar/": {
+				discord_path: "/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024",
+				url: "mxc://cadence.moe/zKXGZhmImMHuGQZWJEFKJbsF"
+			},
+			"m.room.guest_access/": {
+				guest_access: "can_join"
+			},
+			"m.room.history_visibility/": {
+				history_visibility: "invited"
+			},
+			"m.room.join_rules/": {
+				join_rule: "invite"
+			},
+			"m.room.name/": {
+				name: "Psychonauts 3"
+			},
+			"m.room.power_levels/": {
+				users: {
+					"@test_auto_invite:example.org": 100
+				},
+			},
+		}
+	)
+})

d2m/event-dispatcher.js

@@ -244,6 +244,8 @@ module.exports = {
 			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
 		}
 
+		if (dUtils.isEphemeralMessage(message)) return // Ephemeral messages are for the eyes of the receiver only!
+
 		const {affected, row} = await speedbump.maybeDoSpeedbump(message.channel_id, message.id)
 		if (affected) return
 
@@ -262,6 +264,8 @@ module.exports = {
 			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
 		}
 
+		if (dUtils.isEphemeralMessage(data)) return // Ephemeral messages are for the eyes of the receiver only!
+
 		// Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from.
 		const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id)
 		if (affected) return

discord/utils.js

@@ -101,6 +101,14 @@ function isWebhookMessage(message) {
 	return message.webhook_id && !isInteractionResponse
 }
 
+/**
+ * Ephemeral messages can be generated if a slash command is attached to the same bot that OOYE is running on
+ * @param {DiscordTypes.APIMessage} message
+ */
+function isEphemeralMessage(message) {
+	return message.flags & (1 << 6);
+}
+
 /** @param {string} snowflake */
 function snowflakeToTimestampExact(snowflake) {
 	return Number(BigInt(snowflake) >> 22n) + EPOCH
@@ -116,5 +124,6 @@ module.exports.hasPermission = hasPermission
 module.exports.hasSomePermissions = hasSomePermissions
 module.exports.hasAllPermissions = hasAllPermissions
 module.exports.isWebhookMessage = isWebhookMessage
+module.exports.isEphemeralMessage = isEphemeralMessage
 module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact
 module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact

m2d/actions/redact.js

@@ -16,7 +16,7 @@ async function deleteMessage(event) {
 	db.prepare("DELETE FROM event_message WHERE event_id = ?").run(event.event_id)
 	for (const row of rows) {
 		db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(row.message_id)
-		discord.snow.channel.deleteMessage(row.channel_id, row.message_id, event.content.reason)
+		await discord.snow.channel.deleteMessage(row.channel_id, row.message_id, event.content.reason)
 	}
 }
 

m2d/converters/event-to-message.js

@@ -54,16 +54,17 @@ const turndownService = new TurndownService({
  */
 // @ts-ignore bad type from turndown
 turndownService.escape = function (string) {
-	const escapedWords = string.split(" ").map(word => {
-		if (word.match(/^https?:\/\//)) {
-			return word
+	return string.replace(/\s+|\S+/g, part => { // match chunks of spaces or non-spaces
+		if (part.match(/\s/)) return part // don't process spaces
+
+		if (part.match(/^https?:\/\//)) {
+			return part
 		} else {
 			return markdownEscapes.reduce(function (accumulator, escape) {
 				return accumulator.replace(escape[0], escape[1])
-		 	}, word)
+			}, part)
 		}
 	})
-	return escapedWords.join(" ")
 }
 
 turndownService.remove("mx-reply")

m2d/converters/event-to-message.test.js

@@ -205,6 +205,34 @@ test("event2message: links in plaintext body are not broken", async t => {
 	)
 })
 
+test("event2message: links in plaintext body are not broken when preceded by a newline", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			type: "m.room.message",
+			sender: "@cadence:cadence.moe",
+			content: {
+				msgtype: "m.text",
+				body: "java redstoners will be like \"I hate bedrock edition redstone!!\" meanwhile java edition:\nhttps://youtu.be/g_ORb7bN3CM"
+			},
+			event_id: "$b1c5gJZfh1gq3zz6UkhI1whJ61JVvgvvzbdSPEYnTbY",
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe"
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "java redstoners will be like \"I hate bedrock edition redstone!!\" meanwhile java edition:\nhttps://youtu.be/g_ORb7bN3CM",
+				avatar_url: undefined,
+				allowed_mentions: {
+					parse: ["users", "roles"]
+				}
+			}]
+		}
+	)
+})
+
 test("event2message: links in formatted body where the text & href are the same, just post the link once", async t => {
 	t.deepEqual(
 		await eventToMessage({
@@ -556,7 +584,7 @@ test("event2message: code blocks work", async t => {
 				msgtype: "m.text",
 				body: "wrong body",
 				format: "org.matrix.custom.html",
-				formatted_body: "<p>preceding</p>\n<pre><code>code block\n</code></pre>\n<p>following <code>code</code> is inline</p>\n"
+				formatted_body: "<p>preceding</p>\n<pre><code>code block\n</code></pre>\n<p>following <code>code</code> is inline</p>"
 			},
 			event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
 			origin_server_ts: 1688301929913,
@@ -613,6 +641,66 @@ test("event2message: code block contents are formatted correctly and not escaped
 	)
 })
 
+test("event2message: code blocks use double backtick as delimiter when necessary", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			type: "m.room.message",
+			sender: "@cadence:cadence.moe",
+			content: {
+				msgtype: "m.text",
+				body: "wrong body",
+				format: "org.matrix.custom.html",
+				formatted_body: "<code>backtick in ` the middle</code>, <code>backtick at the edge`</code>"
+			},
+			event_id: "$pGkWQuGVmrPNByrFELxhzI6MCBgJecr5I2J3z88Gc2s",
+			room_id: "!BpMdOUkWWhFxmTrENV:cadence.moe"
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "``backtick in ` the middle``, `` backtick at the edge` ``",
+				avatar_url: undefined,
+				allowed_mentions: {
+						parse: ["users", "roles"]
+				}
+			}]
+		}
+	)
+})
+
+test("event2message: inline code is converted to code block if it contains both delimiters", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			type: "m.room.message",
+			sender: "@cadence:cadence.moe",
+			content: {
+				msgtype: "m.text",
+				body: "wrong body",
+				format: "org.matrix.custom.html",
+				formatted_body: "<code>` one two ``</code>"
+			},
+			event_id: "$pGkWQuGVmrPNByrFELxhzI6MCBgJecr5I2J3z88Gc2s",
+			room_id: "!BpMdOUkWWhFxmTrENV:cadence.moe"
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "``` ` one two `` ```",
+				avatar_url: undefined,
+				allowed_mentions: {
+						parse: ["users", "roles"]
+				}
+			}]
+		}
+	)
+})
+
 test("event2message: code blocks are uploaded as attachments instead if they contain incompatible backticks", async t => {
 	t.deepEqual(
 		await eventToMessage({

matrix/kstate.js

@@ -4,7 +4,12 @@ const assert = require("assert").strict
 const mixin = require("mixin-deep")
 const {isDeepStrictEqual} = require("util")
 
-/** Mutates the input. */
+const passthrough = require("../passthrough")
+const {sync} = passthrough
+/** @type {import("./file")} */
+const file = sync.require("./file")
+
+/** Mutates the input. Not recursive - can only include or exclude entire state events. */
 function kstateStripConditionals(kstate) {
 	for (const [k, content] of Object.entries(kstate)) {
 		// conditional for whether a key is even part of the kstate (doing this declaratively on json is hard, so represent it as a property instead.)
@@ -16,9 +21,33 @@ function kstateStripConditionals(kstate) {
 	return kstate
 }
 
-function kstateToState(kstate) {
+/** Mutates the input. Works recursively through object tree. */
+async function kstateUploadMxc(obj) {
+	const promises = []
+	function inner(obj) {
+		for (const [k, v] of Object.entries(obj)) {
+			if (v == null || typeof v !== "object") continue
+
+			if (v.$url) {
+				promises.push(
+					file.uploadDiscordFileToMxc(v.$url)
+					.then(mxc => obj[k] = mxc)
+				)
+			}
+
+			inner(v)
+		}
+	}
+	inner(obj)
+	await Promise.all(promises)
+	return obj
+}
+
+/** Automatically strips conditionals and uploads URLs to mxc. */
+async function kstateToState(kstate) {
 	const events = []
 	kstateStripConditionals(kstate)
+	await kstateUploadMxc(kstate)
 	for (const [k, content] of Object.entries(kstate)) {
 		const slashIndex = k.indexOf("/")
 		assert(slashIndex > 0)
@@ -74,6 +103,7 @@ function diffKState(actual, target) {
 }
 
 module.exports.kstateStripConditionals = kstateStripConditionals
+module.exports.kstateUploadMxc = kstateUploadMxc
 module.exports.kstateToState = kstateToState
 module.exports.stateToKState = stateToKState
 module.exports.diffKState = diffKState

matrix/kstate.test.js

@@ -1,5 +1,5 @@
 const assert = require("assert")
-const {kstateToState, stateToKState, diffKState, kstateStripConditionals} = require("./kstate")
+const {kstateToState, stateToKState, diffKState, kstateStripConditionals, kstateUploadMxc} = require("./kstate")
 const {test} = require("supertape")
 
 test("kstate strip: strips false conditions", t => {
@@ -21,8 +21,53 @@ test("kstate strip: keeps true conditions while removing $if", t => {
 	})
 })
 
-test("kstate2state: general", t => {
-	t.deepEqual(kstateToState({
+test("kstateUploadMxc: sets the mxc", async t => {
+	const input = {
+		"m.room.avatar/": {
+			url: {$url: "https://cdn.discordapp.com/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"},
+			test1: {
+				test2: {
+					test3: {$url: "https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg"}
+				}
+			}
+		}
+	}
+	await kstateUploadMxc(input)
+	t.deepEqual(input, {
+		"m.room.avatar/": {
+			url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl",
+			test1: {
+				test2: {
+					test3: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR"
+				}
+			}
+		}
+	})
+})
+
+test("kstateUploadMxc and strip: work together", async t => {
+	const input = {
+		"m.room.avatar/yes": {
+			$if: true,
+			url: {$url: "https://cdn.discordapp.com/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"}
+		},
+		"m.room.avatar/no": {
+			$if: false,
+			url: {$url: "https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024"}
+		},
+	}
+	kstateStripConditionals(input)
+	await kstateUploadMxc(input)
+	t.deepEqual(input, {
+		"m.room.avatar/yes": {
+			url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl"
+		}
+	})
+})
+
+
+test("kstate2state: general", async t => {
+	t.deepEqual(await kstateToState({
 		"m.room.name/": {name: "test name"},
 		"m.room.member/@cadence:cadence.moe": {membership: "join"},
 		"uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"}

test/test.js

@@ -21,6 +21,9 @@ const reg = require("../matrix/read-registration")
 reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded
 reg.ooye.server_name = "cadence.moe"
 reg.ooye.invite = ["@test_auto_invite:example.org"]
+reg.id = "baby" // don't actually take authenticated actions on the server
+reg.as_token = "baby"
+reg.hs_token = "baby"
 
 const sync = new HeatSync({watchFS: false})
 
@@ -117,6 +120,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
 	require("../matrix/read-registration.test")
 	require("../matrix/txnid.test")
 	require("../d2m/actions/create-room.test")
+	require("../d2m/actions/create-space.test")
 	require("../d2m/actions/register-user.test")
 	require("../d2m/converters/edit-to-changes.test")
 	require("../d2m/converters/emoji-to-key.test")