From 8439512f1aaf619e5e007e11c02128e53b909c16 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 23 Nov 2023 15:51:25 +1300 Subject: [PATCH 1/3] Add snowflake timestamp converter functions --- .../message-to-event.embeds.test.js | 26 ------------------- discord/utils.js | 14 ++++++++++ discord/utils.test.js | 23 ++++++++++++++++ m2d/converters/event-to-message.test.js | 3 ++- 4 files changed, 39 insertions(+), 27 deletions(-) create mode 100644 discord/utils.test.js diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js index 173e016..b339687 100644 --- a/d2m/converters/message-to-event.embeds.test.js +++ b/d2m/converters/message-to-event.embeds.test.js @@ -3,32 +3,6 @@ const {messageToEvent} = require("./message-to-event") const data = require("../../test/data") const Ty = require("../../types") -/** - * @param {string} roomID - * @param {string} eventID - * @returns {(roomID: string, eventID: string) => Promise>} - */ -function mockGetEvent(t, roomID_in, eventID_in, outer) { - return async function(roomID, eventID) { - t.equal(roomID, roomID_in) - t.equal(eventID, eventID_in) - return new Promise(resolve => { - setTimeout(() => { - resolve({ - event_id: eventID_in, - room_id: roomID_in, - origin_server_ts: 1680000000000, - unsigned: { - age: 2245, - transaction_id: "$local.whatever" - }, - ...outer - }) - }) - }) - } -} - test("message2event embeds: nothing but a field", async t => { const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {}) t.deepEqual(events, [{ diff --git a/discord/utils.js b/discord/utils.js index 8856aa7..ceb4468 100644 --- a/discord/utils.js +++ b/discord/utils.js @@ -2,6 +2,8 @@ const DiscordTypes = require("discord-api-types/v10") +const EPOCH = 1420070400000 + /** * @param {string[]} userRoles * @param {DiscordTypes.APIGuild["roles"]} guildRoles @@ -56,5 +58,17 @@ function isWebhookMessage(message) { return message.webhook_id && !isInteractionResponse } +/** @param {string} snowflake */ +function snowflakeToTimestampExact(snowflake) { + return Number(BigInt(snowflake) >> 22n) + EPOCH +} + +/** @param {number} timestamp */ +function timestampToSnowflakeInexact(timestamp) { + return String((timestamp - EPOCH) * 2**22) +} + module.exports.getPermissions = getPermissions module.exports.isWebhookMessage = isWebhookMessage +module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact +module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact diff --git a/discord/utils.test.js b/discord/utils.test.js new file mode 100644 index 0000000..fd064ef --- /dev/null +++ b/discord/utils.test.js @@ -0,0 +1,23 @@ +const {test} = require("supertape") +const data = require("../test/data") +const utils = require("./utils") + +test("is webhook message: identifies bot interaction response as not a message", t => { + t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction), false) +}) + +test("is webhook message: identifies webhook interaction response as not a message", t => { + t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction_without_bot_user), false) +}) + +test("is webhook message: identifies webhook message as a message", t => { + t.equal(utils.isWebhookMessage(data.special_message.bridge_echo_webhook), true) +}) + +test("discord utils: converts snowflake to timestamp", t => { + t.equal(utils.snowflakeToTimestampExact("86913608335773696"), 1440792219004) +}) + +test("discerd utils: converts timestamp to snowflake", t => { + t.match(utils.timestampToSnowflakeInexact(1440792219004), /^869136083357.....$/) +}) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 8bd4223..1af5e42 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -2168,7 +2168,8 @@ test("event2message: guessed @mentions may join members to mention", async t => const subtext = { user: { id: "321876634777218072", - username: "subtext", + username: "subtextual", + global_name: "subtext", discriminator: "0" } } From 70292474619cfe14a66c4cf8e194bdd0b47b55d6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 23 Nov 2023 15:52:41 +1300 Subject: [PATCH 2/3] Coverage for role and emoji converter --- d2m/converters/message-to-event.js | 9 ++++----- d2m/converters/message-to-event.test.js | 12 ++++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 5994ff3..8a5ab17 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -52,11 +52,8 @@ function getDiscordParseCallbacks(message, guild, useHTML) { emoji: node => { if (useHTML) { const mxc = select("emoji", "mxc_url", {emoji_id: node.id}).pluck().get() - if (mxc) { + assert(mxc) // All emojis should have been added ahead of time in the messageToEvent function. return `:${node.name}:` - } else { // We shouldn't get here since all emojis should have been added ahead of time in the messageToEvent function. - return `:${node.name}:` - } } else { return `:${node.name}:` } @@ -64,7 +61,9 @@ function getDiscordParseCallbacks(message, guild, useHTML) { role: node => { const role = guild.roles.find(r => r.id === node.id) if (!role) { - return "@&" + node.id // fallback for if the cache breaks. if this happens, fix discord-packets.js to store the role info. + // This fallback should only trigger if somebody manually writes a silly message, or if the cache breaks (hasn't happened yet). + // If the cache breaks, fix discord-packets.js to store role info properly. + return "@&" + node.id } else if (useHTML && role.color) { return `@${role.name}` } else if (useHTML) { diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index 8f89a43..d40dd72 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -85,6 +85,18 @@ test("message2event: simple role mentions", async t => { }]) }) +test("message2event: manually constructed unknown roles should use fallback", async t => { + const events = await messageToEvent(data.message.unknown_role, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "I'm just @&4 testing a few role pings <@&B> don't mind me", + format: "org.matrix.custom.html", + formatted_body: "I'm just @&4 testing a few role pings <@&B> don't mind me" + }]) +}) + test("message2event: simple message link", async t => { const events = await messageToEvent(data.message.simple_message_link, data.guild.general, {}) t.deepEqual(events, [{ From 8d452102d5f7725479eb730e61636567ffd9db25 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 23 Nov 2023 16:11:46 +1300 Subject: [PATCH 3/3] d->m: Message links are now guessed when unknown --- d2m/converters/message-to-event.js | 36 ++++++++++---- d2m/converters/message-to-event.test.js | 27 +++++++++++ m2d/converters/event-to-message.js | 2 +- matrix/api.js | 11 +++++ test/data.js | 64 +++++++++++++++++++++++++ test/test.js | 1 + 6 files changed, 130 insertions(+), 11 deletions(-) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 8a5ab17..b9e13dd 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -16,6 +16,8 @@ const emojiToKey = sync.require("./emoji-to-key") const lottie = sync.require("./lottie") /** @type {import("../../m2d/converters/utils")} */ const mxUtils = sync.require("../../m2d/converters/utils") +/** @type {import("../../discord/utils")} */ +const dUtils = sync.require("../../discord/utils") const reg = require("../../matrix/read-registration") const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) @@ -53,7 +55,7 @@ function getDiscordParseCallbacks(message, guild, useHTML) { if (useHTML) { const mxc = select("emoji", "mxc_url", {emoji_id: node.id}).pluck().get() assert(mxc) // All emojis should have been added ahead of time in the messageToEvent function. - return `:${node.name}:` + return `:${node.name}:` } else { return `:${node.name}:` } @@ -261,18 +263,32 @@ async function messageToEvent(message, guild, options = {}, di) { /** * Translate Discord message links to Matrix event links. + * If OOYE has handled this message in the past, this is an instant database lookup. + * Otherwise, if OOYE knows the channel, this is a multi-second request to /timestamp_to_event to approximate. * @param {string} content Partial or complete Discord message content */ - function transformContentMessageLinks(content) { - return content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => { - const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get() + async function transformContentMessageLinks(content) { + for (const match of [...content.matchAll(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/g)]) { + assert(typeof match.index === "number") + const channelID = match[2] + const messageID = match[3] const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() - if (eventID && roomID) { - return `https://matrix.to/#/${roomID}/${eventID}` + let result + if (roomID) { + const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get() + if (eventID && roomID) { + result = `https://matrix.to/#/${roomID}/${eventID}` + } else { + const ts = dUtils.snowflakeToTimestampExact(messageID) + const {event_id} = await di.api.getEventForTimestamp(roomID, ts) + result = `https://matrix.to/#/${roomID}/${event_id}` + } } else { - return `${whole} [event not found]` + result = `${match[0]} [event is from another server]` } - }) + content = content.slice(0, match.index) + result + content.slice(match.index + match[0].length) + } + return content } /** @@ -283,7 +299,7 @@ async function messageToEvent(message, guild, options = {}, di) { * @param {any} customHtmlOutput */ async function transformContent(content, customOptions = {}, customParser = null, customHtmlOutput = null) { - content = transformContentMessageLinks(content) + content = await transformContentMessageLinks(content) // Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter. // So we scan the message ahead of time for all its emojis and ensure they are in the DB. @@ -429,7 +445,7 @@ async function messageToEvent(message, guild, options = {}, di) { if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image if (authorNameText || embed.author?.url) { if (embed.author?.url) { - const authorURL = transformContentMessageLinks(embed.author.url) + const authorURL = await transformContentMessageLinks(embed.author.url) rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`${authorNameText}`) } else { rep.addParagraph(`## ${authorNameText}`, tag`${authorNameText}`) diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index d40dd72..980b1f5 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -109,6 +109,33 @@ test("message2event: simple message link", async t => { }]) }) +test("message2event: message link that OOYE doesn't know about", async t => { + let called = 0 + const events = await messageToEvent(data.message.message_link_to_before_ooye, data.guild.general, {}, { + api: { + async getEventForTimestamp(roomID, ts) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + return { + event_id: "$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U", + origin_server_ts: 1613287812754 + } + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "Me: I'll scroll up to find a certain message I'll send\n_scrolls up and clicks message links for god knows how long_\n_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n_stumbles upon:_ " + + "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U", + format: "org.matrix.custom.html", + formatted_body: "Me: I'll scroll up to find a certain message I'll send
scrolls up and clicks message links for god knows how long
completely forgets what they were looking for and simply begins scrolling up to find some fun moments
stumbles upon: " + + 'https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U' + }]) + t.equal(called, 1, "getEventForTimestamp should be called once") +}) + test("message2event: attachment with no content", async t => { const events = await messageToEvent(data.message.attachment_no_content, data.guild.general, {}) t.deepEqual(events, [{ diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index ef913bb..ae40abf 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -505,7 +505,7 @@ async function eventToMessage(event, guild, di) { content = displayNameRunoff + replyLine + content // Handling written @mentions: we need to look for candidate Discord members to join to the room - let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // d flag requires Node 16+ + let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/) if (writtenMentionMatch) { const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]}) if (results[0]) { diff --git a/matrix/api.js b/matrix/api.js index 5509930..d6fd28b 100644 --- a/matrix/api.js +++ b/matrix/api.js @@ -82,6 +82,16 @@ async function getEvent(roomID, eventID) { return root } +/** + * @param {string} roomID + * @param {number} ts unix silliseconds + */ +async function getEventForTimestamp(roomID, ts) { + /** @type {{event_id: string, origin_server_ts: number}} */ + const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/timestamp_to_event`, null, {ts})) + return root +} + /** * @param {string} roomID * @returns {Promise} @@ -223,6 +233,7 @@ module.exports.joinRoom = joinRoom module.exports.inviteToRoom = inviteToRoom module.exports.leaveRoom = leaveRoom module.exports.getEvent = getEvent +module.exports.getEventForTimestamp = getEventForTimestamp module.exports.getAllState = getAllState module.exports.getStateEvent = getStateEvent module.exports.getJoinedMembers = getJoinedMembers diff --git a/test/data.js b/test/data.js index b75abd9..a7154b4 100644 --- a/test/data.js +++ b/test/data.js @@ -508,6 +508,36 @@ module.exports = { flags: 0, components: [] }, + unknown_role: { + id: "1162374402785153106", + type: 0, + content: "I'm just <@&4> testing a few role pings <@&B> don't mind me", + channel_id: "160197704226439168", + author: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "4b5c4b28051144e4c111f0113a0f1cf1", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [ "212762309364285440", "503685967463448616" ], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-10-13T13:00:53.496000+00:00", + edited_timestamp: null, + flags: 0, + components: [] + }, simple_message_link: { id: "1126788210308161626", type: 0, @@ -539,6 +569,40 @@ module.exports = { flags: 0, components: [] }, + message_link_to_before_ooye: { + id: "1160824382755708948", + type: 0, + content: "Me: I'll scroll up to find a certain message I'll send\n" + + "_scrolls up and clicks message links for god knows how long_\n" + + "_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n" + + "_stumbles upon:_ https://discord.com/channels/112760669178241024/112760669178241024/810412561941921851", + channel_id: "112760669178241024", + author: { + id: "271237147401045000", + username: "jinx", + avatar: "a0ba563c16aff137289f67f38545807f", + discriminator: "0", + public_flags: 0, + premium_type: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "Jinx", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: '2023-10-09T06:21:39.923000+00:00', + edited_timestamp: null, + flags: 0, + components: [] + }, simple_written_at_mention_for_matrix: { id: "1159030564049915915", type: 0, diff --git a/test/test.js b/test/test.js index 90e3f5a..553ec44 100644 --- a/test/test.js +++ b/test/test.js @@ -45,6 +45,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not await p db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8")) require("../db/orm.test") + require("../discord/utils.test") require("../matrix/kstate.test") require("../matrix/api.test") require("../matrix/file.test")