remove redactions from database in edit flow

d2m/actions/delete-message.js

@@ -0,0 +1,29 @@
+// @ts-check
+
+const passthrough = require("../../passthrough")
+const { sync, db } = passthrough
+/** @type {import("../converters/edit-to-changes")} */
+const editToChanges = sync.require("../converters/edit-to-changes")
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+
+/**
+ * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data
+ */
+async function deleteMessage(data) {
+	/** @type {string?} */
+	const roomID = db.prepare("SELECT channel_id FROM channel_room WHERE channel_id = ?").pluck().get(data.channel_id)
+	if (!roomID) return
+
+	/** @type {string[]} */
+	const eventsToRedact = db.prepare("SELECT event_id FROM event_message WHERE message_id = ?").pluck().all(data.id)
+
+	for (const eventID of eventsToRedact) {
+		// Unfortuately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs
+		await api.redactEvent(roomID, eventID)
+		db.prepare("DELETE from event_message WHERE event_id = ?").run(eventID)
+		// TODO: Consider whether this code could be reused between edited messages and deleted messages.
+	}
+}
+
+module.exports.deleteMessage = deleteMessage

d2m/actions/edit-message.js

@@ -14,7 +14,8 @@ const api = sync.require("../../matrix/api")
 async function editMessage(message, guild) {
 	console.log(`*** applying edit for message ${message.id} in channel ${message.channel_id}`)
 	const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid} = await editToChanges.editToChanges(message, guild, api)
-	console.log("making these changes:", {eventsToRedact, eventsToReplace, eventsToSend})
+	console.log("making these changes:")
+	console.dir({eventsToRedact, eventsToReplace, eventsToSend}, {depth: null})
 
 	// 1. Replace all the things.
 	for (const {oldID, newContent} of eventsToReplace) {
@@ -34,7 +35,9 @@ async function editMessage(message, guild) {
 	// Not redacting as the last action because the last action is likely to be shown in the room preview in clients, and we don't want it to look like somebody actually deleted a message.
 	for (const eventID of eventsToRedact) {
 		await api.redactEvent(roomID, eventID, senderMxid)
-		// TODO: I should almost certainly remove the redacted event from our database now, shouldn't I? I mean, it's literally not there any more... you can't do anything else with it...
+		// TODO: Reconsider whether it's the right thing to do to delete it from our database? I mean, it's literally not there any more... you can't do anything else with it...
+		// and you definitely want to mark it in *some* way to prevent duplicate redactions...
+		db.prepare("DELETE from event_message WHERE event_id = ?").run(eventID)
 		// TODO: If I just redacted part = 0, I should update one of the other events to make it the new part = 0, right?
 		// TODO: Consider whether this code could be reused between edited messages and deleted messages.
 	}

d2m/converters/edit-to-changes.js

@@ -22,7 +22,14 @@ async function editToChanges(message, guild, api) {
 	// Figure out what events we will be replacing
 
 	const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(message.channel_id)
-	const senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
+	/** @type {string?} */
+	let senderMxid = db.prepare("SELECT mxid FROM sim WHERE discord_id = ?").pluck().get(message.author.id) ?? null
+	if (senderMxid) {
+		const senderIsInRoom = db.prepare("SELECT * FROM sim_member WHERE room_id = ? and mxid = ?").get(roomID, senderMxid)
+		if (!senderIsInRoom) {
+			senderMxid = null // just send as ooye bot
+		}
+	}
 	/** @type {{event_id: string, event_type: string, event_subtype: string?, part: number}[]} */
    const oldEventRows = db.prepare("SELECT event_id, event_type, event_subtype, part FROM event_message WHERE message_id = ?").all(message.id)
 

d2m/converters/edit-to-changes.test.js

@@ -3,8 +3,34 @@ const {editToChanges} = require("./edit-to-changes")
 const data = require("../../test/data")
 const Ty = require("../../types")
 
+test("edit2changes: edit by webhook", async t => {
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edit_by_webhook, data.guild.general, {})
+	t.deepEqual(eventsToRedact, [])
+	t.deepEqual(eventsToSend, [])
+	t.deepEqual(eventsToReplace, [{
+		oldID: "$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10",
+		newContent: {
+			$type: "m.room.message",
+			msgtype: "m.text",
+			body: "* test 2",
+			"m.mentions": {},
+			"m.new_content": {
+				// *** Replaced With: ***
+				msgtype: "m.text",
+				body: "test 2",
+				"m.mentions": {}
+			},
+			"m.relates_to": {
+				rel_type: "m.replace",
+				event_id: "$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10"
+			}
+		}
+	}])
+	t.equal(senderMxid, null)
+})
+
 test("edit2changes: bot response", async t => {
-   const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.bot_response, data.guild.general, {
+	const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.bot_response, data.guild.general, {
 		async getJoinedMembers(roomID) {
 			t.equal(roomID, "!uCtjHhfGlYbVnPVlkG:cadence.moe")
 			return new Promise(resolve => {
@@ -55,6 +81,7 @@ test("edit2changes: bot response", async t => {
 			}
 		}
 	}])
+	t.equal(senderMxid, "@_ooye_bojack_horseman:cadence.moe")
 })
 
 test("edit2changes: remove caption from image", async t => {

d2m/converters/message-to-event.test.js

@@ -39,6 +39,16 @@ test("message2event: simple plaintext", async t => {
 	}])
 })
 
+test("message2event: simple plaintext with quotes", async t => {
+	const events = await messageToEvent(data.message.simple_plaintext_with_quotes, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: `then he said, "you and her aren't allowed in here!"`
+	}])
+})
+
 test("message2event: simple user mention", async t => {
 	const events = await messageToEvent(data.message.simple_user_mention, data.guild.general, {})
 	t.deepEqual(events, [{

d2m/discord-packets.js

@@ -16,6 +16,7 @@ const utils = {
 		/** @type {typeof import("./event-dispatcher")} */
 		const eventDispatcher = sync.require("./event-dispatcher")
 
+		// Client internals, keep track of the state we need
 		if (message.t === "READY") {
 			if (client.ready) return
 			client.ready = true
@@ -62,15 +63,26 @@ const utils = {
 					}
 				}
 			}
+		}
 
-
+		// Event dispatcher for OOYE bridge operations
-		} else if (message.t === "MESSAGE_CREATE") {
+		try {
+			if (message.t === "MESSAGE_CREATE") {
 				eventDispatcher.onMessageCreate(client, message.d)
 
+			} else if (message.t === "MESSAGE_UPDATE") {
+				eventDispatcher.onMessageUpdate(client, message.d)
+
+			} else if (message.t === "MESSAGE_DELETE") {
+				eventDispatcher.onMessageDelete(client, message.d)
 
 			} else if (message.t === "MESSAGE_REACTION_ADD") {
 				eventDispatcher.onReactionAdd(client, message.d)
 			}
+		} catch (e) {
+			// Let OOYE try to handle errors too
+			eventDispatcher.onError(client, e, message)
+		}
 	}
 }
 

d2m/event-dispatcher.js

@@ -1,17 +1,61 @@
 const assert = require("assert").strict
+const util = require("util")
 const {sync, db} = require("../passthrough")
 
 /** @type {import("./actions/send-message")}) */
 const sendMessage = sync.require("./actions/send-message")
 /** @type {import("./actions/edit-message")}) */
 const editMessage = sync.require("./actions/edit-message")
-
+/** @type {import("./actions/delete-message")}) */
+const deleteMessage = sync.require("./actions/delete-message")
 /** @type {import("./actions/add-reaction")}) */
 const addReaction = sync.require("./actions/add-reaction")
+/** @type {import("../matrix/api")}) */
+const api = sync.require("../matrix/api")
+
+let lastReportedEvent = 0
 
 // Grab Discord events we care about for the bridge, check them, and pass them on
 
 module.exports = {
+	/**
+	 * @param {import("./discord-client")} client
+	 * @param {Error} e
+	 * @param {import("cloudstorm").IGatewayMessage} gatewayMessage
+	 */
+	onError(client, e, gatewayMessage) {
+		console.error("hit event-dispatcher's error handler with this exception:")
+		console.error(e) // TODO: also log errors into a file or into the database, maybe use a library for this? or just wing it? definitely need to be able to store the formatted event body to load back in later
+		console.error(`while handling this ${gatewayMessage.t} gateway event:`)
+		console.dir(gatewayMessage.d)
+
+		if (Date.now() - lastReportedEvent > 5000) {
+			lastReportedEvent = Date.now()
+			const channelID = gatewayMessage.d.channel_id
+			if (channelID) {
+				const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(channelID)
+				let stackLines = e.stack.split("\n")
+				let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/"))
+				if (cloudstormLine !== -1) {
+					stackLines = stackLines.slice(0, cloudstormLine - 2)
+				}
+				api.sendEvent(roomID, "m.room.message", {
+					msgtype: "m.text",
+					body: "\u26a0 Bridged event from Discord not delivered. See formatted content for full details.",
+					format: "org.matrix.custom.html",
+					formatted_body: "\u26a0 <strong>Bridged event from Discord not delivered</strong>"
+						+ `<br>Gateway event: ${gatewayMessage.t}`
+						+ `<pre>${stackLines.join("\n")}</pre>`
+						+ `<details><summary>Original payload</summary>`
+						+ `<pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`,
+					"m.mentions": {
+						user_ids: ["@cadence:cadence.moe"]
+					}
+				})
+			}
+		}
+	},
+
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
@@ -37,6 +81,13 @@ module.exports = {
 	 * @param {import("discord-api-types/v10").GatewayMessageUpdateDispatchData} message
 	 */
 	onMessageUpdate(client, data) {
+		if (data.webhook_id) {
+			const row = db.prepare("SELECT webhook_id FROM webhook WHERE webhook_id = ?").pluck().get(data.webhook_id)
+			if (row) {
+				// The update was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
+				return
+			}
+		}
 		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
 		// If the message content is a string then it includes all interesting fields and is meaningful.
 		if (typeof data.content === "string") {
@@ -60,5 +111,14 @@ module.exports = {
 		if (data.emoji.id !== null) return // TODO: image emoji reactions
 		console.log(data)
 		addReaction.addReaction(data)
+	},
+
+	/**
+	 * @param {import("./discord-client")} client
+	 * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data
+	 */
+	onMessageDelete(client, data) {
+		console.log(data)
+		deleteMessage.deleteMessage(data)
 	}
 }

package-lock.json

@@ -11,7 +11,7 @@
       "dependencies": {
         "better-sqlite3": "^8.3.0",
         "cloudstorm": "^0.8.0",
-        "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#9799e4f79912d07f89a030e479e82fcc1e75bc81",
+        "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#440130ef343c8183a81c7c09809731484aa3a182",
         "heatsync": "^2.4.1",
         "js-yaml": "^4.1.0",
         "matrix-appservice": "^2.0.0",
@@ -1051,7 +1051,7 @@
     },
     "node_modules/discord-markdown": {
       "version": "2.4.1",
-      "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#9799e4f79912d07f89a030e479e82fcc1e75bc81",
+      "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#440130ef343c8183a81c7c09809731484aa3a182",
       "license": "MIT",
       "dependencies": {
         "simple-markdown": "^0.7.2"

package.json

@@ -17,7 +17,7 @@
   "dependencies": {
     "better-sqlite3": "^8.3.0",
     "cloudstorm": "^0.8.0",
-    "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#9799e4f79912d07f89a030e479e82fcc1e75bc81",
+    "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#440130ef343c8183a81c7c09809731484aa3a182",
     "heatsync": "^2.4.1",
     "js-yaml": "^4.1.0",
     "matrix-appservice": "^2.0.0",

test/data.js

@@ -170,6 +170,37 @@ module.exports = {
 			flags: 0,
 			components: []
 		},
+		simple_plaintext_with_quotes: {
+			id: "1126733830494093454",
+			type: 0,
+			content: `then he said, "you and her aren't allowed in here!"`,
+			channel_id: "112760669178241024",
+			author: {
+				id: "111604486476181504",
+				username: "kyuugryphon",
+				avatar: "e4ce31267ca524d19be80e684d4cafa1",
+				discriminator: "0",
+				public_flags: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "KyuuGryphon",
+				avatar_decoration: null,
+				display_name: "KyuuGryphon",
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-07-07T04:37:58.892000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
 		simple_user_mention: {
 			id: "1126739682080858234",
 			type: 0,
@@ -788,6 +819,33 @@ module.exports = {
 		}
 	},
 	message_update: {
+		edit_by_webhook: {
+			application_id: "684280192553844747",
+			attachments: [],
+			author: {
+				avatar: null,
+				bot: true,
+				discriminator: "0000",
+				id: "700285844094845050",
+				username: "cadence [they]"
+			},
+			channel_id: "497161350934560778",
+			components: [],
+			content: "test 2",
+			edited_timestamp: "2023-08-17T06:29:34.167314+00:00",
+			embeds: [],
+			flags: 0,
+			guild_id: "497159726455455754",
+			id: "1141619794500649020",
+			mention_everyone: false,
+			mention_roles: [],
+			mentions: [],
+			pinned: false,
+			timestamp: "2023-08-17T06:29:29.279000+00:00",
+			tts: false,
+			type: 0,
+			webhook_id: "700285844094845050"
+		},
 		bot_response: {
 			attachments: [],
 			author: {