d->m: Message links are now guessed when unknown

d2m/converters/message-to-event.js

@@ -16,6 +16,8 @@ const emojiToKey = sync.require("./emoji-to-key")
 const lottie = sync.require("./lottie")
 /** @type {import("../../m2d/converters/utils")} */
 const mxUtils = sync.require("../../m2d/converters/utils")
+/** @type {import("../../discord/utils")} */
+const dUtils = sync.require("../../discord/utils")
 const reg = require("../../matrix/read-registration")
 
 const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
@@ -53,7 +55,7 @@ function getDiscordParseCallbacks(message, guild, useHTML) {
 			if (useHTML) {
 				const mxc = select("emoji", "mxc_url", {emoji_id: node.id}).pluck().get()
 				assert(mxc) // All emojis should have been added ahead of time in the messageToEvent function.
-					return `<img data-mx-emoticon height="32" src="${mxc}" title=":${node.name}:" alt=":${node.name}:">`
+				return `<img data-mx-emoticon height="32" src="${mxc}" title=":${node.name}:" alt=":${node.name}:">`
 			} else {
 				return `:${node.name}:`
 			}
@@ -261,18 +263,32 @@ async function messageToEvent(message, guild, options = {}, di) {
 
 	/**
 	 * Translate Discord message links to Matrix event links.
+	 * If OOYE has handled this message in the past, this is an instant database lookup.
+	 * Otherwise, if OOYE knows the channel, this is a multi-second request to /timestamp_to_event to approximate.
 	 * @param {string} content Partial or complete Discord message content
 	 */
-	function transformContentMessageLinks(content) {
+	async function transformContentMessageLinks(content) {
-		return content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => {
+		for (const match of [...content.matchAll(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/g)]) {
-			const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get()
+			assert(typeof match.index === "number")
+			const channelID = match[2]
+			const messageID = match[3]
 			const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
-			if (eventID && roomID) {
+			let result
-				return `https://matrix.to/#/${roomID}/${eventID}`
+			if (roomID) {
+				const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get()
+				if (eventID && roomID) {
+					result = `https://matrix.to/#/${roomID}/${eventID}`
+				} else {
+					const ts = dUtils.snowflakeToTimestampExact(messageID)
+					const {event_id} = await di.api.getEventForTimestamp(roomID, ts)
+					result = `https://matrix.to/#/${roomID}/${event_id}`
+				}
 			} else {
-				return `${whole} [event not found]`
+				result = `${match[0]} [event is from another server]`
 			}
-		})
+			content = content.slice(0, match.index) + result + content.slice(match.index + match[0].length)
+		}
+		return content
 	}
 
 	/**
@@ -283,7 +299,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 	 * @param {any} customHtmlOutput
 	 */
 	async function transformContent(content, customOptions = {}, customParser = null, customHtmlOutput = null) {
-		content = transformContentMessageLinks(content)
+		content = await transformContentMessageLinks(content)
 
 		// Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter.
 		// So we scan the message ahead of time for all its emojis and ensure they are in the DB.
@@ -429,7 +445,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 		if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image
 		if (authorNameText || embed.author?.url) {
 			if (embed.author?.url) {
-				const authorURL = transformContentMessageLinks(embed.author.url)
+				const authorURL = await transformContentMessageLinks(embed.author.url)
 				rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`<strong><a href="${authorURL}">${authorNameText}</a></strong>`)
 			} else {
 				rep.addParagraph(`## ${authorNameText}`, tag`<strong>${authorNameText}</strong>`)

d2m/converters/message-to-event.test.js

@@ -109,6 +109,33 @@ test("message2event: simple message link", async t => {
 	}])
 })
 
+test("message2event: message link that OOYE doesn't know about", async t => {
+	let called = 0
+	const events = await messageToEvent(data.message.message_link_to_before_ooye, data.guild.general, {}, {
+		api: {
+			async getEventForTimestamp(roomID, ts) {
+				called++
+				t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe")
+				return {
+					event_id: "$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U",
+					origin_server_ts: 1613287812754
+				}
+			}
+		}
+	})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "Me: I'll scroll up to find a certain message I'll send\n_scrolls up and clicks message links for god knows how long_\n_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n_stumbles upon:_ "
+			+ "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U",
+		format: "org.matrix.custom.html",
+		formatted_body: "Me: I'll scroll up to find a certain message I'll send<br><em>scrolls up and clicks message links for god knows how long</em><br><em>completely forgets what they were looking for and simply begins scrolling up to find some fun moments</em><br><em>stumbles upon:</em> "
+			+ '<a href="https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U">https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U</a>'
+	}])
+	t.equal(called, 1, "getEventForTimestamp should be called once")
+})
+
 test("message2event: attachment with no content", async t => {
 	const events = await messageToEvent(data.message.attachment_no_content, data.guild.general, {})
 	t.deepEqual(events, [{

m2d/converters/event-to-message.js

@@ -505,7 +505,7 @@ async function eventToMessage(event, guild, di) {
 	content = displayNameRunoff + replyLine + content
 
 	// Handling written @mentions: we need to look for candidate Discord members to join to the room
-	let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // d flag requires Node 16+
+	let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/)
 	if (writtenMentionMatch) {
 		const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]})
 		if (results[0]) {

matrix/api.js

@@ -82,6 +82,16 @@ async function getEvent(roomID, eventID) {
 	return root
 }
 
+/**
+ * @param {string} roomID
+ * @param {number} ts unix silliseconds
+ */
+async function getEventForTimestamp(roomID, ts) {
+	/** @type {{event_id: string, origin_server_ts: number}} */
+	const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/timestamp_to_event`, null, {ts}))
+	return root
+}
+
 /**
  * @param {string} roomID
  * @returns {Promise<Ty.Event.BaseStateEvent[]>}
@@ -223,6 +233,7 @@ module.exports.joinRoom = joinRoom
 module.exports.inviteToRoom = inviteToRoom
 module.exports.leaveRoom = leaveRoom
 module.exports.getEvent = getEvent
+module.exports.getEventForTimestamp = getEventForTimestamp
 module.exports.getAllState = getAllState
 module.exports.getStateEvent = getStateEvent
 module.exports.getJoinedMembers = getJoinedMembers

test/data.js

@@ -508,6 +508,36 @@ module.exports = {
 			flags: 0,
 			components: []
 		},
+		unknown_role: {
+			id: "1162374402785153106",
+			type: 0,
+			content: "I'm just <@&4> testing a few role pings <@&B> don't mind me",
+			channel_id: "160197704226439168",
+			author: {
+				id: "772659086046658620",
+				username: "cadence.worm",
+				avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+				discriminator: "0",
+				public_flags: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "cadence",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [ "212762309364285440", "503685967463448616" ],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-10-13T13:00:53.496000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
 		simple_message_link: {
 			id: "1126788210308161626",
 			type: 0,
@@ -539,6 +569,40 @@ module.exports = {
 			flags: 0,
 			components: []
 		},
+		message_link_to_before_ooye: {
+			id: "1160824382755708948",
+			type: 0,
+			content: "Me: I'll scroll up to find a certain message I'll send\n" +
+				"_scrolls up and clicks message links for god knows how long_\n" +
+				"_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n" +
+				"_stumbles upon:_ https://discord.com/channels/112760669178241024/112760669178241024/810412561941921851",
+			channel_id: "112760669178241024",
+			author: {
+				id: "271237147401045000",
+				username: "jinx",
+				avatar: "a0ba563c16aff137289f67f38545807f",
+				discriminator: "0",
+				public_flags: 0,
+				premium_type: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "Jinx",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: '2023-10-09T06:21:39.923000+00:00',
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
 		simple_written_at_mention_for_matrix: {
 			id: "1159030564049915915",
 			type: 0,

test/test.js

@@ -45,6 +45,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
 	await p
 	db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8"))
 	require("../db/orm.test")
+	require("../discord/utils.test")
 	require("../matrix/kstate.test")
 	require("../matrix/api.test")
 	require("../matrix/file.test")