From cae591e5fd63a7afe66ac93f617a7ff53e5c430c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 15 Aug 2023 17:20:31 +1200 Subject: [PATCH] script for capturing message update events --- d2m/actions/edit-message.js | 96 +---------------------- d2m/converters/edit-to-changes.js | 96 +++++++++++++++++++++++ d2m/discord-client.js | 6 +- scripts/capture-message-update-events.js | 51 ++++++++++++ scripts/events.db | Bin 0 -> 8192 bytes 5 files changed, 154 insertions(+), 95 deletions(-) create mode 100644 d2m/converters/edit-to-changes.js create mode 100644 scripts/capture-message-update-events.js create mode 100644 scripts/events.db diff --git a/d2m/actions/edit-message.js b/d2m/actions/edit-message.js index 662e124..933267c 100644 --- a/d2m/actions/edit-message.js +++ b/d2m/actions/edit-message.js @@ -1,94 +1,5 @@ -// @ts-check - -const assert = require("assert") - -const passthrough = require("../../passthrough") -const { discord, sync, db } = passthrough -/** @type {import("../converters/message-to-event")} */ -const messageToEvent = sync.require("../converters/message-to-event") -/** @type {import("../../matrix/api")} */ -const api = sync.require("../../matrix/api") -/** @type {import("./register-user")} */ -const registerUser = sync.require("./register-user") -/** @type {import("../actions/create-room")} */ -const createRoom = sync.require("../actions/create-room") - -/** - * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message - * @param {import("discord-api-types/v10").APIGuild} guild - */ -async function editMessage(message, guild) { - // Figure out what events we will be replacing - - const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").get(message.channel_id) - const senderMxid = await registerUser.ensureSimJoined(message.author, roomID) - /** @type {{event_id: string, event_type: string, event_subtype: string?, part: number}[]} */ - const oldEventRows = db.prepare("SELECT event_id, event_type, event_subtype, part FROM event_message WHERE message_id = ?").all(message.id) - - // Figure out what we will be replacing them with - - const newEvents = await messageToEvent.messageToEvent(message, guild, api) - - // Match the new events to the old events - - /* - Rules: - + The events must have the same type. - + The events must have the same subtype. - Events will therefore be divided into three categories: - */ - /** 1. Events that are matched, and should be edited by sending another m.replace event */ - let eventsToReplace = [] - /** 2. Events that are present in the old version only, and should be blanked or redacted */ - let eventsToRedact = [] - /** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */ - let eventsToSend = [] - - // For each old event... - outer: while (newEvents.length) { - const newe = newEvents[0] - // Find a new event to pair it with... - let handled = false - for (let i = 0; i < oldEventRows.length; i++) { - const olde = oldEventRows[i] - if (olde.event_type === newe.$type && olde.event_subtype === (newe.msgtype || null)) { - // Found one! - // Set up the pairing - eventsToReplace.push({ - old: olde, - new: newe - }) - // These events have been handled now, so remove them from the source arrays - newEvents.shift() - oldEventRows.splice(i, 1) - // Go all the way back to the start of the next iteration of the outer loop - continue outer - } - } - // If we got this far, we could not pair it to an existing event, so it'll have to be a new one - eventsToSend.push(newe) - newEvents.shift() - } - // Anything remaining in oldEventRows is present in the old version only and should be redacted. - eventsToRedact = oldEventRows - - // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! - // (Consider a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.) - // So we'll remove entries from eventsToReplace that *definitely* cannot have changed. Everything remaining *may* have changed. - eventsToReplace = eventsToReplace.filter(ev => { - // Discord does not allow files, images, attachments, or videos to be edited. - if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote") { - return false - } - // Discord does not allow stickers to be edited. - if (ev.old.event_type === "m.sticker") { - return false - } - // Anything else is fair game. - return true - }) - - // Action time! +async function editMessage() { + // Action time! // 1. Replace all the things. @@ -113,6 +24,5 @@ async function editMessage(message, guild) { } return eventIDs -} -module.exports.editMessage = editMessage +{eventsToReplace, eventsToRedact, eventsToSend} diff --git a/d2m/converters/edit-to-changes.js b/d2m/converters/edit-to-changes.js new file mode 100644 index 0000000..0dd084f --- /dev/null +++ b/d2m/converters/edit-to-changes.js @@ -0,0 +1,96 @@ +// @ts-check + +const assert = require("assert") + +const passthrough = require("../../passthrough") +const { discord, sync, db } = passthrough +/** @type {import("./message-to-event")} */ +const messageToEvent = sync.require("../converters/message-to-event") +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../actions/register-user")} */ +const registerUser = sync.require("./register-user") +/** @type {import("../actions/create-room")} */ +const createRoom = sync.require("../actions/create-room") + +/** + * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message + * IMPORTANT: This may not have all the normal fields! The API documentation doesn't provide possible types, just says it's all optional! + * Since I don't have a spec, I will have to capture some real traffic and add it as test cases... I hope they don't change anything later... + * @param {import("discord-api-types/v10").APIGuild} guild + */ +async function editToChanges(message, guild) { + // Figure out what events we will be replacing + + const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").get(message.channel_id) + const senderMxid = await registerUser.ensureSimJoined(message.author, roomID) + /** @type {{event_id: string, event_type: string, event_subtype: string?, part: number}[]} */ + const oldEventRows = db.prepare("SELECT event_id, event_type, event_subtype, part FROM event_message WHERE message_id = ?").all(message.id) + + // Figure out what we will be replacing them with + + const newEvents = await messageToEvent.messageToEvent(message, guild, api) + + // Match the new events to the old events + + /* + Rules: + + The events must have the same type. + + The events must have the same subtype. + Events will therefore be divided into three categories: + */ + /** 1. Events that are matched, and should be edited by sending another m.replace event */ + let eventsToReplace = [] + /** 2. Events that are present in the old version only, and should be blanked or redacted */ + let eventsToRedact = [] + /** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */ + let eventsToSend = [] + + // For each old event... + outer: while (newEvents.length) { + const newe = newEvents[0] + // Find a new event to pair it with... + let handled = false + for (let i = 0; i < oldEventRows.length; i++) { + const olde = oldEventRows[i] + if (olde.event_type === newe.$type && olde.event_subtype === (newe.msgtype || null)) { + // Found one! + // Set up the pairing + eventsToReplace.push({ + old: olde, + new: newe + }) + // These events have been handled now, so remove them from the source arrays + newEvents.shift() + oldEventRows.splice(i, 1) + // Go all the way back to the start of the next iteration of the outer loop + continue outer + } + } + // If we got this far, we could not pair it to an existing event, so it'll have to be a new one + eventsToSend.push(newe) + newEvents.shift() + } + // Anything remaining in oldEventRows is present in the old version only and should be redacted. + eventsToRedact = oldEventRows + + // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! + // (Consider a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.) + // So we'll remove entries from eventsToReplace that *definitely* cannot have changed. Everything remaining *may* have changed. + eventsToReplace = eventsToReplace.filter(ev => { + // Discord does not allow files, images, attachments, or videos to be edited. + if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote") { + return false + } + // Discord does not allow stickers to be edited. + if (ev.old.event_type === "m.sticker") { + return false + } + // Anything else is fair game. + return true + }) + + return {eventsToReplace, eventsToRedact, eventsToSend} +} + +module.exports.editMessage = editMessage diff --git a/d2m/discord-client.js b/d2m/discord-client.js index 91682bd..5e90d85 100644 --- a/d2m/discord-client.js +++ b/d2m/discord-client.js @@ -14,7 +14,7 @@ class DiscordClient { * @param {string} discordToken * @param {boolean} listen whether to set up the event listeners for OOYE to operate */ - constructor(discordToken, listen) { + constructor(discordToken, listen = true) { this.discordToken = discordToken this.snow = new SnowTransfer(discordToken) this.cloud = new CloudStorm(discordToken, { @@ -44,7 +44,9 @@ class DiscordClient { this.guilds = new Map() /** @type {Map>} */ this.guildChannelMap = new Map() - this.cloud.on("event", message => discordPackets.onPacket(this, message)) + if (listen) { + this.cloud.on("event", message => discordPackets.onPacket(this, message)) + } this.cloud.on("error", console.error) } } diff --git a/scripts/capture-message-update-events.js b/scripts/capture-message-update-events.js new file mode 100644 index 0000000..2ff9b49 --- /dev/null +++ b/scripts/capture-message-update-events.js @@ -0,0 +1,51 @@ +// @ts-check + +// **** +const interestingFields = ["author", "content", "edited_timestamp", "mentions", "attachments", "embeds", "type", "message_reference", "referenced_message", "sticker_items"] +// ***** + +function fieldToPresenceValue(field) { + if (field === undefined) return 0 + else if (field === null) return 1 + else if (Array.isArray(field) && field.length === 0) return 10 + else if (typeof field === "object" && Object.keys(field).length === 0) return 20 + else if (field === "") return 30 + else return 99 +} + +const sqlite = require("better-sqlite3") +const HeatSync = require("heatsync") + +const config = require("../config") +const passthrough = require("../passthrough") + +const sync = new HeatSync({watchFS: false}) + +Object.assign(passthrough, {config, sync}) + +const DiscordClient = require("../d2m/discord-client", false) + +const discord = new DiscordClient(config.discordToken, false) +passthrough.discord = discord + +;(async () => { + await discord.cloud.connect() + console.log("Discord gateway started") + + const f = event => onPacket(discord, event, () => discord.cloud.off("event", f)) + discord.cloud.on("event", f) +})() + +const events = new sqlite("scripts/events.db") +const sql = "INSERT INTO \"update\" (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")" +console.log(sql) +const prepared = events.prepare(sql) + +/** @param {DiscordClient} discord */ +function onPacket(discord, event, unsubscribe) { + if (event.t === "MESSAGE_UPDATE") { + const data = [JSON.stringify(event.d), ...interestingFields.map(f => fieldToPresenceValue(event.d[f]))] + console.log(data) + prepared.run(...data) + } +} diff --git a/scripts/events.db b/scripts/events.db new file mode 100644 index 0000000000000000000000000000000000000000..c8f5bad88858b0ddc7a3b54ec36577c94791c001 GIT binary patch literal 8192 zcmeH~-EJH;6vuZ16-iJk?ojXCxuJz5^0&vM3elE`R*G6^7m13XiD&HHOfoZG%}1lC z>J@O&TONa_;T?Dp&g^bBOW8y5$w>2Euo@l8K~zy9e7^aOeWJ%OG;PoO8z z6X*%_1bPBJf&T}AA8!sn`1sbX;m_xdNpjatt!WnD8xNly+#emF(ftR92Pl|#1?bN9 zR&ZMTGT0g&d^bW*p2Fwb!^2%@Hf?j_t6=N#lhMIf2haZ0m-@1CWwXB1S*Uq!++>BT z8&gczTNN-q^JTrh*EEeuPdb}x3|x^odre<+F?H*w1&?YoadG9wu5x91joGr>#)}c_ zr>mPRJ#$qI@fK@**}mJ=<{O*84|aZqum0%?^aOeWJ%OG;PoO98-yrbj%Z;61isAg( z+}wN>%&uP%-VGYKyhUS^*ACv7x4|f&n6f<_?h!G~i$-vKW2;!)~zn6<0oYu)=DSd+DkRU%Hi%a8ay+=v8oO9S}m5R7^_Bgch1=2@4Vu zYRW^xg;bOfj5RbeO$jH0hni8$LMg(TC1qS<&9R2@8XWJ&Gb*70(@ri26mb6=+V2%|NIHX`jqzb_Sh<{qSB5RAd z&bk|HRJM5zjjps=Iavl?!qqg0f~R@mQkbe}n=2@%J_EY3X}|zB1Md;MNA-xZNQj7$ zeX7*zW{h()0nV_a)|v5gh13^C3(J`99v|DR?rIJjw{Wk|FTFA^O=AFm=i}?$AcHN1 zSUHC1fuRZ+Kmg?6;MF3FtNm-?I$lC zI`<_qD6QJ8cDK=;7p}g0R34R&VB&jGQbOGlf$xeF-bs@P4Dtd08xK>1zVm$m3EKjT z%qpiD!t}(zS<2%%dyuDw-Q^VY$~2Lkq8F7OoRPM6RcQ)PO7Os+n)D1k@>T5s9+*wr zB+t_LYNKW|Dn&$hIJi`BRwNu0FrDN+F?l?nYOYVvM_Ofo#WYY=Mu7&MgJt4DgiX~# zikNF9wl4L+1~m1yhyqLam@A@rB3gAL4~Z@{c2<=y zGR0czZwslzxlTx+F=qrP%rb)0P`E@pYmBxDBytyGBE(v~nCTQ`K2K^hs50af%rzAP NWX=`uviRRq)gLj?i?aX# literal 0 HcmV?d00001