From 343675950467e660ae0d5e52dbd6d9dd4d082bfa Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 19 Aug 2023 22:54:23 +1200 Subject: [PATCH] catch up on missed d->m messages when logging in --- d2m/actions/create-room.js | 28 +++++ d2m/actions/register-user.test.js | 2 +- d2m/actions/send-message.js | 9 +- .../message-to-event.embeds.test.js | 40 +++++++ d2m/converters/message-to-event.js | 25 ++++- d2m/converters/message-to-event.test.js | 12 ++- d2m/converters/user-to-mxid.test.js | 6 +- d2m/discord-packets.js | 1 + d2m/event-dispatcher.js | 38 ++++++- db/data-for-test.sql | 9 +- m2d/actions/channel-webhook.js | 3 +- m2d/converters/event-to-message.js | 6 ++ m2d/converters/event-to-message.test.js | 2 +- stdin.js | 1 + test/data.js | 101 +++++++++++++++++- test/test.js | 1 + 16 files changed, 268 insertions(+), 16 deletions(-) create mode 100644 d2m/converters/message-to-event.embeds.test.js diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js index 6d64d58..0178347 100644 --- a/d2m/actions/create-room.js +++ b/d2m/actions/create-room.js @@ -190,6 +190,33 @@ async function _syncRoom(channelID, shouldActuallySync) { } } +async function _unbridgeRoom(channelID) { + /** @ts-ignore @type {DiscordTypes.APIGuildChannel} */ + const channel = discord.channels.get(channelID) + assert.ok(channel) + const roomID = db.prepare("SELECT room_id from channel_room WHERE channel_id = ?").pluck().get(channelID) + assert.ok(roomID) + const spaceID = db.prepare("SELECT space_id FROM guild_space WHERE guild_id = ?").pluck().get(channel.guild_id) + assert.ok(spaceID) + + // remove room from being a space member + await api.sendState(spaceID, "m.space.child", roomID, {}) + + // send a notification in the room + await api.sendEvent(roomID, "m.room.message", { + msgtype: "m.notice", + body: "⚠️ This room was removed from the bridge." + }) + + // leave room + await api.leaveRoom(roomID) + + // delete room from database + const {changes} = db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channelID) + assert.equal(changes, 1) +} + + /** * @param {DiscordTypes.APIGuildTextChannel} channel * @param {string} spaceID @@ -237,3 +264,4 @@ module.exports.syncRoom = syncRoom module.exports.createAllForGuild = createAllForGuild module.exports.channelToKState = channelToKState module.exports._convertNameAndTopic = convertNameAndTopic +module.exports._unbridgeRoom = _unbridgeRoom diff --git a/d2m/actions/register-user.test.js b/d2m/actions/register-user.test.js index 0afce50..34470ba 100644 --- a/d2m/actions/register-user.test.js +++ b/d2m/actions/register-user.test.js @@ -8,7 +8,7 @@ test("member2state: general", async t => { await _memberToStateContent(testData.member.sheep.user, testData.member.sheep, testData.guild.general.id), { avatar_url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl", - displayname: "The Expert's Submarine | aprilsong", + displayname: "The Expert's Submarine", membership: "join", "moe.cadence.ooye.member": { avatar: "/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024" diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js index 2132905..a5c8dac 100644 --- a/d2m/actions/send-message.js +++ b/d2m/actions/send-message.js @@ -22,8 +22,11 @@ async function sendMessage(message, guild) { let senderMxid = null if (!message.webhook_id) { - assert(message.member) - senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID) + if (message.member) { // available on a gateway message create event + senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID) + } else { // well, good enough... + senderMxid = await registerUser.ensureSimJoined(message.author, roomID) + } } const events = await messageToEvent.messageToEvent(message, guild, {}, {api}) @@ -35,7 +38,7 @@ async function sendMessage(message, guild) { const eventWithoutType = {...event} delete eventWithoutType.$type - const eventID = await api.sendEvent(roomID, eventType, event, senderMxid) + const eventID = await api.sendEvent(roomID, eventType, event, senderMxid, new Date(message.timestamp).getTime()) db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, message.channel_id, eventPart) // source 1 = discord eventPart = 1 // TODO: use more intelligent algorithm to determine whether primary or supporting diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js new file mode 100644 index 0000000..7972f13 --- /dev/null +++ b/d2m/converters/message-to-event.embeds.test.js @@ -0,0 +1,40 @@ +const {test} = require("supertape") +const {messageToEvent} = require("./message-to-event") +const data = require("../../test/data") +const Ty = require("../../types") + +/** + * @param {string} roomID + * @param {string} eventID + * @returns {(roomID: string, eventID: string) => Promise>} + */ +function mockGetEvent(t, roomID_in, eventID_in, outer) { + return async function(roomID, eventID) { + t.equal(roomID, roomID_in) + t.equal(eventID, eventID_in) + return new Promise(resolve => { + setTimeout(() => { + resolve({ + event_id: eventID_in, + room_id: roomID_in, + origin_server_ts: 1680000000000, + unsigned: { + age: 2245, + transaction_id: "$local.whatever" + }, + ...outer + }) + }) + }) + } +} + +test("message2event embeds: nothing but a field", async t => { + const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "Amanda" + }]) +}) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 29730d4..3808beb 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -108,6 +108,13 @@ async function messageToEvent(message, guild, options = {}, di) { addMention(repliedToEventSenderMxid) } + let msgtype = "m.text" + // Handle message type 4, channel name changed + if (message.type === DiscordTypes.MessageType.ChannelNameChange) { + msgtype = "m.emote" + message.content = "changed the channel name to **" + message.content + "**" + } + // Text content appears first if (message.content) { let content = message.content @@ -188,7 +195,7 @@ async function messageToEvent(message, guild, options = {}, di) { const newTextMessageEvent = { $type: "m.room.message", "m.mentions": mentions, - msgtype: "m.text", + msgtype, body: body } @@ -239,6 +246,22 @@ async function messageToEvent(message, guild, options = {}, di) { size: attachment.size } } + } else if (attachment.content_type?.startsWith("video/") && attachment.width && attachment.height) { + return { + $type: "m.room.message", + "m.mentions": mentions, + msgtype: "m.video", + url: await file.uploadDiscordFileToMxc(attachment.url), + external_url: attachment.url, + body: attachment.description || attachment.filename, + filename: attachment.filename, + info: { + mimetype: attachment.content_type, + w: attachment.width, + h: attachment.height, + size: attachment.size + } + } } else { return { $type: "m.room.message", diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index 86942a7..260ecda 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -330,4 +330,14 @@ test("message2event: very large attachment is linked instead of being uploaded", }]) }) -// TODO: read "edits of replies" in the spec +test("message2event: type 4 channel name change", async t => { + const events = await messageToEvent(data.special_message.thread_name_change, data.guild.general) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.emote", + body: "changed the channel name to **worming**", + format: "org.matrix.custom.html", + formatted_body: "changed the channel name to worming" + }]) +}) diff --git a/d2m/converters/user-to-mxid.test.js b/d2m/converters/user-to-mxid.test.js index 8c4c430..1b31260 100644 --- a/d2m/converters/user-to-mxid.test.js +++ b/d2m/converters/user-to-mxid.test.js @@ -16,6 +16,10 @@ test("user2name: works on emojis", t => { t.equal(userToSimName({username: "🍪 Cookie Monster 🍪", discriminator: "0001"}), "cookie_monster") }) +test("user2name: works on single emoji at the end", t => { + t.equal(userToSimName({username: "Amanda 🎵", discriminator: "2192"}), "amanda") +}) + test("user2name: works on crazy name", t => { t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7//") }) @@ -34,4 +38,4 @@ test("user2name: uses ID if name becomes too short", t => { test("user2name: uses ID when name has only disallowed characters", t => { t.equal(userToSimName({username: "!@#$%^&*", discriminator: "0001", id: "9"}), "9") -}) \ No newline at end of file +}) diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index 79138b2..970bab4 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -41,6 +41,7 @@ const utils = { arr.push(thread.id) client.channels.set(thread.id, thread) } + eventDispatcher.checkMissedMessages(client, message.d) } else if (message.t === "GUILD_DELETE") { diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 9387199..273e89b 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -63,6 +63,42 @@ module.exports = { }) }, + /** + * When logging back in, check if we missed any conversations in any channels. Bridge up to 49 missed messages per channel. + * If more messages were missed, only the latest missed message will be posted. TODO: Consider bridging more, or post a warning when skipping history? + * This can ONLY detect new messages, not any other kind of event. Any missed edits, deletes, reactions, etc will not be bridged. + * @param {import("./discord-client")} client + * @param {import("discord-api-types/v10").GatewayGuildCreateDispatchData} guild + */ + async checkMissedMessages(client, guild) { + if (guild.unavailable) return + const bridgedChannels = db.prepare("SELECT channel_id FROM channel_room").pluck().all() + const prepared = db.prepare("SELECT message_id FROM event_message WHERE channel_id = ? AND message_id = ?").pluck() + for (const channel of guild.channels.concat(guild.threads)) { + if (!bridgedChannels.includes(channel.id)) continue + if (!channel.last_message_id) continue + const latestWasBridged = prepared.get(channel.id, channel.last_message_id) + if (latestWasBridged) continue + + /** More recent messages come first. */ + console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) + const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50}) + let latestBridgedMessageIndex = messages.findIndex(m => { + return prepared.get(channel.id, m.id) + }) + console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`) + if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date. + for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) { + const simulatedGatewayDispatchData = { + guild_id: guild.id, + mentions: [], + ...messages[i] + } + await module.exports.onMessageCreate(client, simulatedGatewayDispatchData) + } + } + }, + /** * @param {import("./discord-client")} client * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message @@ -85,7 +121,7 @@ module.exports = { /** * @param {import("./discord-client")} client - * @param {import("discord-api-types/v10").GatewayMessageUpdateDispatchData} message + * @param {import("discord-api-types/v10").GatewayMessageUpdateDispatchData} data */ async onMessageUpdate(client, data) { if (data.webhook_id) { diff --git a/db/data-for-test.sql b/db/data-for-test.sql index aa82c91..ee31fe3 100644 --- a/db/data-for-test.sql +++ b/db/data-for-test.sql @@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS "channel_room" ( "room_id" TEXT NOT NULL UNIQUE, "name" TEXT, "nick" TEXT, + "thread_parent" TEXT, PRIMARY KEY("channel_id") ); CREATE TABLE IF NOT EXISTS "event_message" ( @@ -54,10 +55,10 @@ BEGIN TRANSACTION; INSERT INTO guild_space (guild_id, space_id) VALUES ('112760669178241024', '!jjWAGMeQdNrVZSSfvz:cadence.moe'); -INSERT INTO channel_room (channel_id, room_id, name, nick, is_thread) VALUES -('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL, 0), -('497161350934560778', '!edUxjVdzgUvXDUIQCK:cadence.moe', 'amanda-spam', NULL, 0), -('160197704226439168', '!uCtjHhfGlYbVnPVlkG:cadence.moe', 'the-stanley-parable-channel', 'bots', 0); +INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES +('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL), +('497161350934560778', '!edUxjVdzgUvXDUIQCK:cadence.moe', 'amanda-spam', NULL, NULL), +('160197704226439168', '!uCtjHhfGlYbVnPVlkG:cadence.moe', 'the-stanley-parable-channel', 'bots', NULL); INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'), diff --git a/m2d/actions/channel-webhook.js b/m2d/actions/channel-webhook.js index f5fd9a9..6d39da7 100644 --- a/m2d/actions/channel-webhook.js +++ b/m2d/actions/channel-webhook.js @@ -41,9 +41,8 @@ async function ensureWebhook(channelID, forceCreate = false) { async function withWebhook(channelID, callback) { const webhook = await ensureWebhook(channelID, false) return callback(webhook).catch(e => { - console.error(e) // TODO: check if the error was webhook-related and if webhook.created === false, then: const webhook = ensureWebhook(channelID, true); return callback(webhook) - throw new Error(e) + throw e }) } diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 74a45ce..b2c56a9 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -30,6 +30,12 @@ function eventToMessage(event) { username: displayName, avatar_url: avatarURL }) + } else if (event.content.msgtype === "m.emote") { + messages.push({ + content: `*${displayName} ${event.content.body}*`, + username: displayName, + avatar_url: avatarURL + }) } return messages diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index e687059..f0c4664 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -21,7 +21,7 @@ test("event2message: janky test", t => { } }), [{ - username: "cadence:cadence.moe", + username: "cadence", content: "test", avatar_url: undefined }] diff --git a/stdin.js b/stdin.js index 7e0db89..61a2a08 100644 --- a/stdin.js +++ b/stdin.js @@ -13,6 +13,7 @@ const registerUser = sync.require("./d2m/actions/register-user") const mreq = sync.require("./matrix/mreq") const api = sync.require("./matrix/api") const sendEvent = sync.require("./m2d/actions/send-event") +const eventDispatcher = sync.require("./d2m/event-dispatcher") const guildID = "112760669178241024" const extraContext = {} diff --git a/test/data.js b/test/data.js index b579a24..fc8cbbd 100644 --- a/test/data.js +++ b/test/data.js @@ -864,7 +864,7 @@ module.exports = { components: [], channel_id: "910283343378120754", author: { - username: "kumaccino", + username: "kumaccino", public_flags: 128, id: "113340068197859328", global_name: "kumaccino", @@ -876,6 +876,72 @@ module.exports = { guild_id: "112760669178241024" } }, + message_with_embeds: { + nothing_but_a_field: { + guild_id: "497159726455455754", + mentions: [], + id: "1141934888862351440", + type: 20, + content: "", + channel_id: "497161350934560778", + author: { + id: "1109360903096369153", + username: "Amanda 🎵", + avatar: "d56cd1b26e043ae512edae2214962faa", + discriminator: "2192", + public_flags: 524288, + flags: 524288, + bot: true, + banner: null, + accent_color: null, + global_name: null, + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [ + { + type: "rich", + color: 3092790, + fields: [ + { + name: "Amanda 🎵#2192 <:online:606664341298872324>\nwillow tree, branch 0", + value: "**❯ Uptime:**\n3m 55s\n**❯ Memory:**\n64.45MB", + inline: false + } + ] + } + ], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-08-18T03:21:33.629000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + application_id: "1109360903096369153", + interaction: { + id: "1141934887608254475", + type: 2, + name: "stats", + user: { + id: "320067006521147393", + username: "papiophidian", + avatar: "47a19b0445069b826e136da4df4259bb", + discriminator: "0", + public_flags: 4194880, + flags: 4194880, + banner: null, + accent_color: null, + global_name: "PapiOphidian", + avatar_decoration_data: null, + banner_color: null + } + }, + webhook_id: "1109360903096369153" + } + }, message_update: { edit_by_webhook: { application_id: "684280192553844747", @@ -1277,5 +1343,38 @@ module.exports = { ], guild_id: "112760669178241024" } + }, + special_message: { + thread_name_change: { + id: "1142391602799710298", + type: 4, + content: "worming", + channel_id: "1142271000067706880", + author: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "4b5c4b28051144e4c111f0113a0f1cf1", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-08-19T09:36:22.717000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + position: 12 + } } } diff --git a/test/test.js b/test/test.js index 31ab099..03394f0 100644 --- a/test/test.js +++ b/test/test.js @@ -21,6 +21,7 @@ require("../matrix/kstate.test") require("../matrix/api.test") require("../matrix/read-registration.test") require("../d2m/converters/message-to-event.test") +require("../d2m/converters/message-to-event.embeds.test") require("../d2m/converters/edit-to-changes.test") require("../d2m/actions/create-room.test") require("../d2m/converters/user-to-mxid.test")