m->d: Code coverage for invalid events

m2d/converters/event-to-message.js

@@ -301,8 +301,12 @@ async function handleRoomOrMessageLinks(input, di) {
 				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, messageID)
 			} else {
 				// 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp
-				const originalEvent = await di.api.getEvent(roomID, eventID)
+				let originalEvent
-				if (!originalEvent) continue
+				try {
+					originalEvent = await di.api.getEvent(roomID, eventID)
+				} catch (e) {
+					continue // Our homeserver doesn't know about the event, so can't resolve it to a Discord link
+				}
 				const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts)
 				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, guessedMessageID)
 			}
@@ -318,7 +322,7 @@ async function handleRoomOrMessageLinks(input, di) {
 /**
  * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event
  * @param {import("discord-api-types/v10").APIGuild} guild
- * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer}} di simple-as-nails dependency injection for the matrix API
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: typeof fetch}} di simple-as-nails dependency injection for the matrix API
  */
 async function eventToMessage(event, guild, di) {
 	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
@@ -393,8 +397,43 @@ async function eventToMessage(event, guild, di) {
 		await (async () => {
 			const repliedToEventId = event.content["m.relates_to"]?.["m.in_reply_to"]?.event_id
 			if (!repliedToEventId) return
-			let repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId)
+			let repliedToEvent
-			if (!repliedToEvent) return
+			try {
+				repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId)
+			} catch (e) {
+				// Original event isn't on our homeserver, so we'll *partially* trust the client's reply fallback.
+				// We'll trust the fallback's quoted content and put it in the reply preview, but we won't trust the authorship info on it.
+
+				// (But if the fallback's quoted content doesn't exist, we give up. There's nothing for us to quote.)
+				if (event.content["format"] !== "org.matrix.custom.html" || typeof event.content["formatted_body"] !== "string") {
+					const lines = event.content.body.split("\n")
+					let stage = 0
+					for (let i = 0; i < lines.length; i++) {
+						if (stage >= 0 && lines[i][0] === ">") stage = 1
+						if (stage >= 1 && lines[i].trim() === "") stage = 2
+						if (stage === 2 && lines[i].trim() !== "") {
+							event.content.body = lines.slice(i).join("\n")
+							break
+						}
+					}
+					return
+				}
+				const mxReply = event.content["formatted_body"]
+				const quoted = mxReply.match(/^<mx-reply><blockquote>.*?In reply to.*?<br>(.*)<\/blockquote><\/mx-reply>/)?.[1]
+				if (!quoted) return
+				const contentPreviewChunks = chunk(
+					entities.decodeHTML5Strict( // Remove entities like &amp; &quot;
+						quoted.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
+							.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+							.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+							.replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting.
+					), 50)
+				replyLine = "> " + contentPreviewChunks[0]
+				if (contentPreviewChunks.length > 1) replyLine = replyLine.replace(/[,.']$/, "") + "..."
+				replyLine += "\n"
+				return
+			}
+
 			// @ts-ignore
 			const autoEmoji = new Map(select("auto_emoji", ["name", "emoji_id"], {}, "WHERE name = 'L1' OR name = 'L2'").raw().all())
 			replyLine = `<:L1:${autoEmoji.get("L1")}><:L2:${autoEmoji.get("L2")}>`
@@ -408,7 +447,11 @@ async function eventToMessage(event, guild, di) {
 				replyLine += `<@${authorID}>`
 			} else {
 				let senderName = select("member_cache", "displayname", {mxid: repliedToEvent.sender}).pluck().get()
-				if (!senderName) senderName = sender.match(/@([^:]*)/)?.[1] || sender
+				if (!senderName) {
+					const match = sender.match(/@([^:]*)/)
+					assert(match)
+					senderName = match[1]
+				}
 				replyLine += `Ⓜ️**${senderName}**`
 			}
 			// If the event has been edited, the homeserver will include the relation in `unsigned`.
@@ -507,7 +550,7 @@ async function eventToMessage(event, guild, di) {
 				if (!match[0].includes("data-mx-emoticon")) break
 				const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/)
 				if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1])
-				if (typeof match.index !== "number") break
+				assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec")
 				last = match.index
 			}
 
@@ -563,8 +606,11 @@ async function eventToMessage(event, guild, di) {
 			if (event.content.info?.mimetype?.includes("/")) {
 				mimetype = event.content.info.mimetype
 			} else {
-				const res = await fetch(url, {method: "HEAD"})
+				const res = await di.fetch(url, {method: "HEAD"})
-				mimetype = res.headers.get("content-type") || "image/webp"
+				if (res.status === 200) {
+					mimetype = res.headers.get("content-type")
+				}
+				if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`)
 			}
 			filename += "." + mimetype.split("/")[1]
 		}

m2d/converters/event-to-message.test.js

@@ -3,7 +3,7 @@ const {test} = require("supertape")
 const {eventToMessage} = require("./event-to-message")
 const data = require("../../test/data")
 const {MatrixServerError} = require("../../matrix/mreq")
-const {db, select} = require("../../passthrough")
+const {db, select, discord} = require("../../passthrough")
 
 /* c8 ignore next 7 */
 function slow() {
@@ -855,6 +855,151 @@ test("event2message: rich reply to an already-edited message will quote the new
 	)
 })
 
+test("event2message: rich reply to a missing event will quote from formatted_body without a link", async t => {
+	let called = 0
+	t.deepEqual(
+		await eventToMessage({
+			"type": "m.room.message",
+			"sender": "@cadence:cadence.moe",
+			"content": {
+				"msgtype": "m.text",
+				"body": "> <@_ooye_kyuugryphon:cadence.moe>\n> > She *sells* *sea*shells by the *sea*shore.\n> But who *sees* the *sea*shells she *sells* sitting sideways?\n\nWhat a tongue-bender...",
+				"format": "org.matrix.custom.html",
+				"formatted_body": "<mx-reply><blockquote><a href=\"https://matrix.to/#/!fGgIymcYWOqjbSRUdV:cadence.moe/$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup?via=cadence.moe&via=feather.onl\">In reply to</a> <a href=\"https://matrix.to/#/@_ooye_kyuugryphon:cadence.moe\">@_ooye_kyuugryphon:cadence.moe</a><br>"
+					+ "<blockquote>She <em>sells</em> <em>sea</em>shells by the <em>sea</em>shore.</blockquote>But who <em>sees</em> the <em>sea</em>shells she <em>sells</em> sitting sideways?"
+					+ "</blockquote></mx-reply>What a tongue-bender...",
+				"m.relates_to": {
+					"m.in_reply_to": {
+						"event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup"
+					}
+				}
+			},
+			"origin_server_ts": 1693029683016,
+			"unsigned": {
+				"age": 91,
+				"transaction_id": "m1693029682894.510"
+			},
+			"event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8",
+			"room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe"
+		}, data.guild.general, {
+			api: {
+				async getEvent(roomID, eventID) {
+					called++
+					t.equal(roomID, "!fGgIymcYWOqjbSRUdV:cadence.moe")
+					t.equal(eventID, "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup")
+					throw new Error("missing event or something")
+				}
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "> But who sees the seashells she sells sitting..."
+					+ "\nWhat a tongue-bender...",
+				avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU"
+			}]
+		}
+	)
+	t.equal(called, 1, "getEvent should be called once")
+})
+
+test("event2message: rich reply to a missing event without formatted_body will use plaintext body and strip reply fallback", async t => {
+	let called = 0
+	t.deepEqual(
+		await eventToMessage({
+			"type": "m.room.message",
+			"sender": "@cadence:cadence.moe",
+			"content": {
+				"msgtype": "m.text",
+				"body": "> <@_ooye_kyuugryphon:cadence.moe> Slow news day.\n\nTesting this reply, ignore",
+				"m.relates_to": {
+					"m.in_reply_to": {
+						"event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup"
+					}
+				}
+			},
+			"origin_server_ts": 1693029683016,
+			"unsigned": {
+				"age": 91,
+				"transaction_id": "m1693029682894.510"
+			},
+			"event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8",
+			"room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe"
+		}, data.guild.general, {
+			api: {
+				async getEvent(roomID, eventID) {
+					called++
+					t.equal(roomID, "!fGgIymcYWOqjbSRUdV:cadence.moe")
+					t.equal(eventID, "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup")
+					throw new Error("missing event or something")
+				}
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "Testing this reply, ignore",
+				avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU"
+			}]
+		}
+	)
+	t.equal(called, 1, "getEvent should be called once")
+})
+
+test("event2message: rich reply to a missing event and no reply fallback will not generate a reply", async t => {
+	let called = 0
+	t.deepEqual(
+		await eventToMessage({
+			"type": "m.room.message",
+			"sender": "@cadence:cadence.moe",
+			"content": {
+				"msgtype": "m.text",
+				"body": "Testing this reply, ignore.",
+				"format": "org.matrix.custom.html",
+				"formatted_body": "Testing this reply, ignore.",
+				"m.relates_to": {
+					"m.in_reply_to": {
+						"event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup"
+					}
+				}
+			},
+			"origin_server_ts": 1693029683016,
+			"unsigned": {
+				"age": 91,
+				"transaction_id": "m1693029682894.510"
+			},
+			"event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8",
+			"room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe"
+		}, data.guild.general, {
+			api: {
+				async getEvent(roomID, eventID) {
+					called++
+					t.equal(roomID, "!fGgIymcYWOqjbSRUdV:cadence.moe")
+					t.equal(eventID, "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup")
+					throw new Error("missing event or something")
+				}
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "Testing this reply, ignore.",
+				avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU"
+			}]
+		}
+	)
+	t.equal(called, 1, "getEvent should be called once")
+})
+
 test("event2message: should avoid using blockquote contents as reply preview in rich reply to a sim user", async t => {
 	t.deepEqual(
 		await eventToMessage({
@@ -1822,6 +1967,35 @@ test("event2message: mentioning bridged rooms works", async t => {
 	)
 })
 
+test("event2message: mentioning bridged rooms works (plaintext body)", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			content: {
+				msgtype: "m.text",
+				body: `I'm just https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe?via=cadence.moe testing channel mentions`
+			},
+			event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
+			origin_server_ts: 1688301929913,
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe",
+			sender: "@cadence:cadence.moe",
+			type: "m.room.message",
+			unsigned: {
+				age: 405299
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "I'm just <#1100319550446252084> testing channel mentions",
+				avatar_url: undefined
+			}]
+		}
+	)
+})
+
 test("event2message: mentioning known bridged events works (plaintext body)", async t => {
 	t.deepEqual(
 		await eventToMessage({
@@ -1913,7 +2087,7 @@ test("event2message: mentioning known bridged events works (formatted body)", as
 	)
 })
 
-test("event2message: mentioning unknown bridged events works", async t => {
+test("event2message: mentioning unknown bridged events can approximate with timestamps", async t => {
 	let called = 0
 	t.deepEqual(
 		await eventToMessage({
@@ -1957,6 +2131,88 @@ test("event2message: mentioning unknown bridged events works", async t => {
 	t.equal(called, 1, "getEvent should be called once")
 })
 
+test("event2message: mentioning events falls back to original link when server doesn't know about it", async t => {
+	let called = 0
+	t.deepEqual(
+		await eventToMessage({
+			content: {
+				msgtype: "m.text",
+				body: "wrong body",
+				format: "org.matrix.custom.html",
+				formatted_body: `it was uploaded years ago in <a href="https://matrix.to/#/!CzvdIdUQXgUjDVKxeU:cadence.moe/$zpzx6ABetMl8BrpsFbdZ7AefVU1Y_-t97bJRJM2JyW1?via=cadence.moe">amanda-spam</a>`
+			},
+			event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOV",
+			origin_server_ts: 1688301929913,
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe",
+			sender: "@cadence:cadence.moe",
+			type: "m.room.message",
+			unsigned: {
+				age: 405299
+			}
+		}, {}, {
+			api: {
+				async getEvent(roomID, eventID) {
+					called++
+					t.equal(roomID, "!CzvdIdUQXgUjDVKxeU:cadence.moe")
+					t.equal(eventID, "$zpzx6ABetMl8BrpsFbdZ7AefVU1Y_-t97bJRJM2JyW1")
+					throw new Error("missing event or something")
+				}
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "it was uploaded years ago in [amanda-spam](<https://matrix.to/#/!CzvdIdUQXgUjDVKxeU:cadence.moe/$zpzx6ABetMl8BrpsFbdZ7AefVU1Y_-t97bJRJM2JyW1?via=cadence.moe>)",
+				avatar_url: undefined
+			}]
+		}
+	)
+	t.equal(called, 1, "getEvent should be called once")
+})
+
+test("event2message: mentioning events falls back to original link when the channel-guild isn't in cache", async t => {
+	t.equal(select("channel_room", "channel_id", {room_id: "!tnedrGVYKFNUdnegvf:tchncs.de"}).pluck().get(), "489237891895768942", "consistency check: this channel-room needs to be in the database for the test to make sense")
+	t.equal(discord.channels.get("489237891895768942"), undefined, "consistency check: this channel needs to not be in client cache for the test to make sense")
+	t.deepEqual(
+		await eventToMessage({
+			content: {
+				msgtype: "m.text",
+				body: "wrong body",
+				format: "org.matrix.custom.html",
+				formatted_body: `it was uploaded years ago in <a href="https://matrix.to/#/!tnedrGVYKFNUdnegvf:tchncs.de/$zpzx6ABetMl8BrpsFbdZ7AefVU1Y_-t97bJRJM2JyW2?via=tchncs.de">ex-room-doesnt-exist-any-more</a>`
+			},
+			event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOX",
+			origin_server_ts: 1688301929913,
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe",
+			sender: "@cadence:cadence.moe",
+			type: "m.room.message",
+			unsigned: {
+				age: 405299
+			}
+		}, {}, {
+			api: {
+				/* c8 skip next 3 */
+				async getEvent() {
+					t.fail("getEvent should not be called because it should quit early due to no channel-guild")
+				}
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "it was uploaded years ago in [ex-room-doesnt-exist-any-more](<https://matrix.to/#/!tnedrGVYKFNUdnegvf:tchncs.de/$zpzx6ABetMl8BrpsFbdZ7AefVU1Y_-t97bJRJM2JyW2?via=tchncs.de>)",
+				avatar_url: undefined
+			}]
+		}
+	)
+})
+
 test("event2message: link to event in an unknown room", async t => {
 	t.deepEqual(
 		await eventToMessage({
@@ -2382,6 +2638,79 @@ test("event2message: stickers work", async t => {
 	)
 })
 
+test("event2message: stickers fetch mimetype from server when mimetype not provided", async t => {
+	let called = 0
+	t.deepEqual(
+		await eventToMessage({
+			type: "m.sticker",
+			sender: "@cadence:cadence.moe",
+			content: {
+				body: "YESYESYES",
+				url: "mxc://cadence.moe/ybOWQCaXysnyUGuUCaQlTGJf"
+			},
+			event_id: "$mL-eEVWCwOvFtoOiivDP7gepvf-fTYH6_ioK82bWDI0",
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe"
+		}, {}, {
+			async fetch(url, options) {
+				called++
+				t.equal(url, "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ybOWQCaXysnyUGuUCaQlTGJf")
+				t.equal(options.method, "HEAD")
+				return {
+					status: 200,
+					headers: new Map([
+						["content-type", "image/gif"]
+					])
+				}
+			}
+		}),
+		{
+			ensureJoined: [],
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "",
+				avatar_url: undefined,
+				attachments: [{id: "0", filename: "YESYESYES.gif"}],
+				pendingFiles: [{name: "YESYESYES.gif", url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ybOWQCaXysnyUGuUCaQlTGJf"}]
+			}]
+		}
+	)
+	t.equal(called, 1, "sticker headers should be fetched")
+})
+
+test("event2message: stickers with unknown mimetype are not allowed", async t => {
+	let called = 0
+	try {
+		await eventToMessage({
+			type: "m.sticker",
+			sender: "@cadence:cadence.moe",
+			content: {
+				body: "something",
+				url: "mxc://cadence.moe/ybOWQCaXysnyUGuUCaQlTGJe"
+			},
+			event_id: "$mL-eEVWCwOvFtoOiivDP7gepvf-fTYH6_ioK82bWDI0",
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe"
+		}, {}, {
+			async fetch(url, options) {
+				called++
+				t.equal(url, "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ybOWQCaXysnyUGuUCaQlTGJe")
+				t.equal(options.method, "HEAD")
+				return {
+					status: 404,
+					headers: new Map([
+						["content-type", "application/json"]
+					])
+				}
+			}
+		})
+		/* c8 ignore next */
+		t.fail("should throw an error")
+	} catch (e) {
+		t.match(e.toString(), "mimetype")
+	}
+})
+
 test("event2message: static emojis work", async t => {
 	t.deepEqual(
 		await eventToMessage({

test/ooye-test-data.sql

@@ -11,7 +11,8 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom
 ('1162005314908999790', '!FuDZhlOAtqswlyxzeR:cadence.moe', 'Hey.', NULL, '1100319550446252084', NULL),
 ('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL),
 ('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL),
-('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS');
+('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS'),
+('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL);
 
 INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES
 ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'),