Use HTML to strip per-message profile fallback

.editorconfig

@@ -1,16 +0,0 @@
-root = true
-
-[*]
-indent_style = tab
-tab_width = 3
-end_of_line = lf
-charset = utf-8
-trim_trailing_whitespace = true
-insert_final_newline = true
-
-[*.json]
-indent_style = space
-indent_size = 2
-
-[*.md]
-trim_trailing_whitespace = false

.prettierignore

@@ -1 +0,0 @@
-*

.vscode/extensions.json

@@ -1,5 +0,0 @@
-{
-	"recommendations": [
-		"editorconfig.editorconfig"
-	]
-}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "out-of-your-element",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "out-of-your-element",
-      "version": "3.4.0",
+      "version": "3.5.0",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "@chriscdn/promise-semaphore": "^3.0.1",
@@ -30,7 +30,7 @@
         "enquirer": "^2.4.1",
         "entities": "^5.0.0",
         "get-relative-path": "^1.0.2",
-        "h3": "^1.15.1",
+        "h3": "^1.15.10",
         "heatsync": "^2.7.2",
         "htmx.org": "^2.0.4",
         "lru-cache": "^11.0.2",
@@ -276,9 +276,9 @@
       }
     },
     "node_modules/@emnapi/runtime": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz",
-      "integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==",
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.1.tgz",
+      "integrity": "sha512-VYi5+ZVLhpgK4hQ0TAjiQiZ6ol0oe4mBx7mVv7IflsiEp0OWoVsp/+f9Vc1hOhE0TtkORVrI1GvzyreqpgWtkA==",
       "license": "MIT",
       "optional": true,
       "dependencies": {
@@ -1163,9 +1163,9 @@
       }
     },
     "node_modules/brace-expansion": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz",
-      "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==",
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz",
+      "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -1488,9 +1488,9 @@
       "license": "MIT"
     },
     "node_modules/domino": {
-      "version": "2.1.6",
-      "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
-      "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==",
+      "version": "2.1.7",
+      "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.7.tgz",
+      "integrity": "sha512-3rcXhx0ixJV2nj8J0tljzejTF73A35LVVdnTQu79UAqTBFEgYPMgGtykMuu/BDqaOZphATku1ddRUn/RtqUHYQ==",
       "license": "BSD-2-Clause"
     },
     "node_modules/emoji-regex": {
@@ -1587,9 +1587,9 @@
       }
     },
     "node_modules/flatted": {
-      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.1.tgz",
-      "integrity": "sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==",
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz",
+      "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==",
       "dev": true,
       "license": "ISC"
     },
@@ -1617,9 +1617,9 @@
       "license": "MIT"
     },
     "node_modules/fullstore": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/fullstore/-/fullstore-4.0.0.tgz",
-      "integrity": "sha512-Y9hN79Q1CFU8akjGnTZoBnTzlA/o8wmtBijJOI8dKCmdC7GLX7OekpLxmbaeRetTOi4OdFGjfsg4c5dxP3jgPw==",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/fullstore/-/fullstore-4.0.2.tgz",
+      "integrity": "sha512-syOev4kA0lZy4VkfBJZ99ZL4cIiSgiKt0G8SpP0kla1tpM1c+V/jBOVY/OqqGtR2XLVcM83SjFPFC3R2YIwqjQ==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -1688,9 +1688,9 @@
       }
     },
     "node_modules/h3": {
-      "version": "1.15.6",
-      "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.6.tgz",
-      "integrity": "sha512-oi15ESLW5LRthZ+qPCi5GNasY/gvynSKUQxgiovrY63bPAtG59wtM+LSrlcwvOHAXzGrXVLnI97brbkdPF9WoQ==",
+      "version": "1.15.10",
+      "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.10.tgz",
+      "integrity": "sha512-YzJeWSkDZxAhvmp8dexjRK5hxziRO7I9m0N53WhvYL5NiWfkUkzssVzY9jvGu0HBoLFW6+duYmNSn6MaZBCCtg==",
       "license": "MIT",
       "dependencies": {
         "cookie-es": "^1.2.2",
@@ -1937,9 +1937,9 @@
       "license": "MIT"
     },
     "node_modules/json-with-bigint": {
-      "version": "3.5.7",
-      "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.7.tgz",
-      "integrity": "sha512-7ei3MdAI5+fJPVnKlW77TKNKwQ5ppSzWvhPuSuINT/GYW9ZOC1eRKOuhV9yHG5aEsUPj9BBx5JIekkmoLHxZOw==",
+      "version": "3.5.8",
+      "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.8.tgz",
+      "integrity": "sha512-eq/4KP6K34kwa7TcFdtvnftvHCD9KvHOGGICWwMFc4dOOKF5t4iYqnfLK8otCRCRv06FXOzGGyqE8h8ElMvvdw==",
       "dev": true,
       "license": "MIT"
     },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "out-of-your-element",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "description": "A bridge between Matrix and Discord",
   "main": "index.js",
   "repository": {
@@ -39,7 +39,7 @@
     "enquirer": "^2.4.1",
     "entities": "^5.0.0",
     "get-relative-path": "^1.0.2",
-    "h3": "^1.15.1",
+    "h3": "^1.15.10",
     "heatsync": "^2.7.2",
     "htmx.org": "^2.0.4",
     "lru-cache": "^11.0.2",

scripts/backfill.js

@@ -51,23 +51,9 @@ const preparedInsert = backfill.prepare("INSERT INTO backfill (channel_id, messa
 
 async function event(event) {
 	if (event.t !== "GUILD_CREATE") return
-	let channel = event.d.channels.find(c => c.id === channelID) || (event.d.threads || []).find(c => c.id === channelID)
-	const guild_id = event.d.id
-
-	if (!channel) {
-		// May be an archived thread not present in GUILD_CREATE data - try fetching via API
-		try {
-			const fetched = await discord.snow.channel.getChannel(channelID)
-			if (!fetched.guild_id || fetched.guild_id !== guild_id) return
-			fetched.guild_id = guild_id
-			discord.channels.set(fetched.id, fetched)
-			channel = fetched
-		} catch (e) {
-			return
-		}
-	}
-
+	const channel = event.d.channels.find(c => c.id === channelID)
 	if (!channel) return
+	const guild_id = event.d.id
 
 	try {
 		await createRoom.syncRoom(channelID)

src/d2m/actions/create-room.js

@@ -193,6 +193,16 @@ async function channelToKState(channel, guild, di) {
 	// Don't overwrite room topic if the topic has been customised
 	if (hasCustomTopic) delete channelKState["m.room.topic/"]
 
+	// Make voice channels be a Matrix voice room (MSC3417)
+	if (channel.type === DiscordTypes.ChannelType.GuildVoice) {
+		creationContent.type = "org.matrix.msc3417.call"
+		channelKState["org.matrix.msc3401.call/"] = {
+			"m.intent": "m.room",
+			"m.type": "m.voice",
+			"m.name": customName || channel.name
+		}
+	}
+
 	// Don't add a space parent if it's self service
 	// (The person setting up self-service has already put it in their preferred space to be able to get this far.)
 	const autocreate = select("guild_active", "autocreate", {guild_id: guild.id}).pluck().get()

src/d2m/actions/create-room.test.js

@@ -190,6 +190,17 @@ test("channel2room: read-only discord channel", async t => {
 	t.equal(api.getCalled(), 2)
 })
 
+test("channel2room: voice channel", async t => {
+	const api = mockAPI(t)
+	const state = kstateStripConditionals(await channelToKState(testData.channel.voice, testData.guild.general, {api}).then(x => x.channelKState))
+	t.equal(state["m.room.create/"].type, "org.matrix.msc3417.call")
+	t.deepEqual(state["org.matrix.msc3401.call/"], {
+		"m.intent": "m.room",
+		"m.name": "🍞丨[8user] Piece",
+		"m.type": "m.voice"
+	})
+})
+
 test("convertNameAndTopic: custom name and topic", t => {
 	t.deepEqual(
 		_convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, "hauntings"),

src/d2m/converters/pins-to-list.js

@@ -22,7 +22,7 @@ function pinsToList(pins, kstate) {
 	/** @type {string[]} */
 	const result = []
 	for (const pin of pins.items) {
-		const eventID = select("event_message", "event_id", {message_id: pin.message.id, part: 0}).pluck().get()
+		const eventID = select("event_message", "event_id", {message_id: pin.message.id}, "ORDER BY part ASC").pluck().get()
 		if (eventID && !alreadyPinned.includes(eventID)) result.push(eventID)
 	}
 	result.reverse()

src/m2d/actions/update-pins.js

@@ -13,7 +13,7 @@ async function updatePins(pins, prev) {
 	const diff = diffPins.diffPins(pins, prev)
 	for (const [event_id, added] of diff) {
 		const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
-			.select("reference_channel_id", "message_id").get()
+			.select("reference_channel_id", "message_id").where({event_id}).and("ORDER BY part ASC").get()
 		if (!row) continue
 		if (added) {
 			discord.snow.channel.addChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message pinned on Matrix")

src/m2d/converters/event-to-message.js

@@ -816,16 +816,6 @@ async function eventToMessage(event, guild, channel, di) {
 		if (shouldProcessTextEvent) {
 			if (event.content.format === "org.matrix.custom.html" && event.content.formatted_body) {
 				let input = event.content.formatted_body
-				if (perMessageProfile?.has_fallback) {
-					// Strip fallback elements added for clients that don't support per-message profiles.
-					// Deviates from recommended regexp in MSC to be less strict. Avoiding an HTML parser for performance reasons.
-					//                     ┌────A────┐                                               Opening HTML tag: capture tag name and stay within tag
-					//                     ┆         ┆┌─────────────B────────────┐                   This text in the tag somewhere, presumably an attribute name
-					//                     ┆         ┆┆                          ┆┌─C──┐             Rest of the opening tag
-					//                     ┆         ┆┆                          ┆┆    ┆┌─D─┐        Tag content (no more tags allowed within)
-					//                     ┆         ┆┆                          ┆┆    ┆┆   ┆┌─E──┐  Closing tag matching opening tag name
-					input = input.replace(/<(\w+)[^>]*\bdata-mx-profile-fallback\b[^>]*>[^<]*<\/\1>/g, "")
-				}
 				if (event.content.msgtype === "m.emote") {
 					input = `* ${displayName} ${input}`
 				}
@@ -886,8 +876,9 @@ async function eventToMessage(event, guild, channel, di) {
 				const doc = domino.createDocument(
 					// DOM parsers arrange elements in the <head> and <body>. Wrapping in a custom element ensures elements are reliably arranged in a single element.
 					'<x-turndown id="turndown-root">' + input + '</x-turndown>'
-				);
-				const root = doc.getElementById("turndown-root");
+				)
+				const root = doc.getElementById("turndown-root")
+				assert(root)
 				async function forEachNode(event, node) {
 					for (; node; node = node.nextSibling) {
 						// Check written mentions
@@ -940,6 +931,7 @@ async function eventToMessage(event, guild, channel, di) {
 					}
 				}
 				await forEachNode(event, root)
+				if (perMessageProfile?.has_fallback) root.querySelectorAll("[data-mx-profile-fallback]").forEach(x => x.remove())
 
 				// SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet.
 				// First we need to determine which emojis are at the end.

test/data.js

@@ -19,6 +19,26 @@ module.exports = {
 			default_thread_rate_limit_per_user: 0,
 			guild_id: "112760669178241024"
 		},
+		voice: {
+			voice_background_display: null,
+			version: 1774469910848,
+			user_limit: 0,
+			type: 2,
+			theme_color: null,
+			status: null,
+			rtc_region: null,
+			rate_limit_per_user: 0,
+			position: 0,
+			permission_overwrites: [],
+			parent_id: "805261291908104252",
+			nsfw: false,
+			name: "🍞丨[8user] Piece",
+			last_message_id: "1459912691098325137",
+			id: "1036840786093953084",
+			flags: 0,
+			bitrate: 256000,
+			guild_id: "112760669178241024"
+		},
 		updates: {
 			type: 0,
 			topic: "Updates and release announcements for Out Of Your Element.",