diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js index 173e016..b339687 100644 --- a/d2m/converters/message-to-event.embeds.test.js +++ b/d2m/converters/message-to-event.embeds.test.js @@ -3,32 +3,6 @@ const {messageToEvent} = require("./message-to-event") const data = require("../../test/data") const Ty = require("../../types") -/** - * @param {string} roomID - * @param {string} eventID - * @returns {(roomID: string, eventID: string) => Promise>} - */ -function mockGetEvent(t, roomID_in, eventID_in, outer) { - return async function(roomID, eventID) { - t.equal(roomID, roomID_in) - t.equal(eventID, eventID_in) - return new Promise(resolve => { - setTimeout(() => { - resolve({ - event_id: eventID_in, - room_id: roomID_in, - origin_server_ts: 1680000000000, - unsigned: { - age: 2245, - transaction_id: "$local.whatever" - }, - ...outer - }) - }) - }) - } -} - test("message2event embeds: nothing but a field", async t => { const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {}) t.deepEqual(events, [{ diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 5994ff3..b9e13dd 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -16,6 +16,8 @@ const emojiToKey = sync.require("./emoji-to-key") const lottie = sync.require("./lottie") /** @type {import("../../m2d/converters/utils")} */ const mxUtils = sync.require("../../m2d/converters/utils") +/** @type {import("../../discord/utils")} */ +const dUtils = sync.require("../../discord/utils") const reg = require("../../matrix/read-registration") const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) @@ -52,11 +54,8 @@ function getDiscordParseCallbacks(message, guild, useHTML) { emoji: node => { if (useHTML) { const mxc = select("emoji", "mxc_url", {emoji_id: node.id}).pluck().get() - if (mxc) { - return `:${node.name}:` - } else { // We shouldn't get here since all emojis should have been added ahead of time in the messageToEvent function. - return `:${node.name}:` - } + assert(mxc) // All emojis should have been added ahead of time in the messageToEvent function. + return `:${node.name}:` } else { return `:${node.name}:` } @@ -64,7 +63,9 @@ function getDiscordParseCallbacks(message, guild, useHTML) { role: node => { const role = guild.roles.find(r => r.id === node.id) if (!role) { - return "@&" + node.id // fallback for if the cache breaks. if this happens, fix discord-packets.js to store the role info. + // This fallback should only trigger if somebody manually writes a silly message, or if the cache breaks (hasn't happened yet). + // If the cache breaks, fix discord-packets.js to store role info properly. + return "@&" + node.id } else if (useHTML && role.color) { return `@${role.name}` } else if (useHTML) { @@ -262,18 +263,32 @@ async function messageToEvent(message, guild, options = {}, di) { /** * Translate Discord message links to Matrix event links. + * If OOYE has handled this message in the past, this is an instant database lookup. + * Otherwise, if OOYE knows the channel, this is a multi-second request to /timestamp_to_event to approximate. * @param {string} content Partial or complete Discord message content */ - function transformContentMessageLinks(content) { - return content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => { - const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get() + async function transformContentMessageLinks(content) { + for (const match of [...content.matchAll(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/g)]) { + assert(typeof match.index === "number") + const channelID = match[2] + const messageID = match[3] const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() - if (eventID && roomID) { - return `https://matrix.to/#/${roomID}/${eventID}` + let result + if (roomID) { + const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get() + if (eventID && roomID) { + result = `https://matrix.to/#/${roomID}/${eventID}` + } else { + const ts = dUtils.snowflakeToTimestampExact(messageID) + const {event_id} = await di.api.getEventForTimestamp(roomID, ts) + result = `https://matrix.to/#/${roomID}/${event_id}` + } } else { - return `${whole} [event not found]` + result = `${match[0]} [event is from another server]` } - }) + content = content.slice(0, match.index) + result + content.slice(match.index + match[0].length) + } + return content } /** @@ -284,7 +299,7 @@ async function messageToEvent(message, guild, options = {}, di) { * @param {any} customHtmlOutput */ async function transformContent(content, customOptions = {}, customParser = null, customHtmlOutput = null) { - content = transformContentMessageLinks(content) + content = await transformContentMessageLinks(content) // Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter. // So we scan the message ahead of time for all its emojis and ensure they are in the DB. @@ -430,7 +445,7 @@ async function messageToEvent(message, guild, options = {}, di) { if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image if (authorNameText || embed.author?.url) { if (embed.author?.url) { - const authorURL = transformContentMessageLinks(embed.author.url) + const authorURL = await transformContentMessageLinks(embed.author.url) rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`${authorNameText}`) } else { rep.addParagraph(`## ${authorNameText}`, tag`${authorNameText}`) diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index 8f89a43..980b1f5 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -85,6 +85,18 @@ test("message2event: simple role mentions", async t => { }]) }) +test("message2event: manually constructed unknown roles should use fallback", async t => { + const events = await messageToEvent(data.message.unknown_role, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "I'm just @&4 testing a few role pings <@&B> don't mind me", + format: "org.matrix.custom.html", + formatted_body: "I'm just @&4 testing a few role pings <@&B> don't mind me" + }]) +}) + test("message2event: simple message link", async t => { const events = await messageToEvent(data.message.simple_message_link, data.guild.general, {}) t.deepEqual(events, [{ @@ -97,6 +109,33 @@ test("message2event: simple message link", async t => { }]) }) +test("message2event: message link that OOYE doesn't know about", async t => { + let called = 0 + const events = await messageToEvent(data.message.message_link_to_before_ooye, data.guild.general, {}, { + api: { + async getEventForTimestamp(roomID, ts) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + return { + event_id: "$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U", + origin_server_ts: 1613287812754 + } + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "Me: I'll scroll up to find a certain message I'll send\n_scrolls up and clicks message links for god knows how long_\n_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n_stumbles upon:_ " + + "https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U", + format: "org.matrix.custom.html", + formatted_body: "Me: I'll scroll up to find a certain message I'll send
scrolls up and clicks message links for god knows how long
completely forgets what they were looking for and simply begins scrolling up to find some fun moments
stumbles upon: " + + 'https://matrix.to/#/!kLRqKKUQXcibIMtOpl:cadence.moe/$E8IQDGFqYzOU7BwY5Z74Bg-cwaU9OthXSroaWtgYc7U' + }]) + t.equal(called, 1, "getEventForTimestamp should be called once") +}) + test("message2event: attachment with no content", async t => { const events = await messageToEvent(data.message.attachment_no_content, data.guild.general, {}) t.deepEqual(events, [{ diff --git a/discord/utils.js b/discord/utils.js index 8856aa7..ceb4468 100644 --- a/discord/utils.js +++ b/discord/utils.js @@ -2,6 +2,8 @@ const DiscordTypes = require("discord-api-types/v10") +const EPOCH = 1420070400000 + /** * @param {string[]} userRoles * @param {DiscordTypes.APIGuild["roles"]} guildRoles @@ -56,5 +58,17 @@ function isWebhookMessage(message) { return message.webhook_id && !isInteractionResponse } +/** @param {string} snowflake */ +function snowflakeToTimestampExact(snowflake) { + return Number(BigInt(snowflake) >> 22n) + EPOCH +} + +/** @param {number} timestamp */ +function timestampToSnowflakeInexact(timestamp) { + return String((timestamp - EPOCH) * 2**22) +} + module.exports.getPermissions = getPermissions module.exports.isWebhookMessage = isWebhookMessage +module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact +module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact diff --git a/discord/utils.test.js b/discord/utils.test.js new file mode 100644 index 0000000..fd064ef --- /dev/null +++ b/discord/utils.test.js @@ -0,0 +1,23 @@ +const {test} = require("supertape") +const data = require("../test/data") +const utils = require("./utils") + +test("is webhook message: identifies bot interaction response as not a message", t => { + t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction), false) +}) + +test("is webhook message: identifies webhook interaction response as not a message", t => { + t.equal(utils.isWebhookMessage(data.interaction_message.thinking_interaction_without_bot_user), false) +}) + +test("is webhook message: identifies webhook message as a message", t => { + t.equal(utils.isWebhookMessage(data.special_message.bridge_echo_webhook), true) +}) + +test("discord utils: converts snowflake to timestamp", t => { + t.equal(utils.snowflakeToTimestampExact("86913608335773696"), 1440792219004) +}) + +test("discerd utils: converts timestamp to snowflake", t => { + t.match(utils.timestampToSnowflakeInexact(1440792219004), /^869136083357.....$/) +}) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index ef913bb..ae40abf 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -505,7 +505,7 @@ async function eventToMessage(event, guild, di) { content = displayNameRunoff + replyLine + content // Handling written @mentions: we need to look for candidate Discord members to join to the room - let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // d flag requires Node 16+ + let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/) if (writtenMentionMatch) { const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]}) if (results[0]) { diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 8bd4223..1af5e42 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -2168,7 +2168,8 @@ test("event2message: guessed @mentions may join members to mention", async t => const subtext = { user: { id: "321876634777218072", - username: "subtext", + username: "subtextual", + global_name: "subtext", discriminator: "0" } } diff --git a/matrix/api.js b/matrix/api.js index 5509930..d6fd28b 100644 --- a/matrix/api.js +++ b/matrix/api.js @@ -82,6 +82,16 @@ async function getEvent(roomID, eventID) { return root } +/** + * @param {string} roomID + * @param {number} ts unix silliseconds + */ +async function getEventForTimestamp(roomID, ts) { + /** @type {{event_id: string, origin_server_ts: number}} */ + const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/timestamp_to_event`, null, {ts})) + return root +} + /** * @param {string} roomID * @returns {Promise} @@ -223,6 +233,7 @@ module.exports.joinRoom = joinRoom module.exports.inviteToRoom = inviteToRoom module.exports.leaveRoom = leaveRoom module.exports.getEvent = getEvent +module.exports.getEventForTimestamp = getEventForTimestamp module.exports.getAllState = getAllState module.exports.getStateEvent = getStateEvent module.exports.getJoinedMembers = getJoinedMembers diff --git a/test/data.js b/test/data.js index b75abd9..a7154b4 100644 --- a/test/data.js +++ b/test/data.js @@ -508,6 +508,36 @@ module.exports = { flags: 0, components: [] }, + unknown_role: { + id: "1162374402785153106", + type: 0, + content: "I'm just <@&4> testing a few role pings <@&B> don't mind me", + channel_id: "160197704226439168", + author: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "4b5c4b28051144e4c111f0113a0f1cf1", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [ "212762309364285440", "503685967463448616" ], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-10-13T13:00:53.496000+00:00", + edited_timestamp: null, + flags: 0, + components: [] + }, simple_message_link: { id: "1126788210308161626", type: 0, @@ -539,6 +569,40 @@ module.exports = { flags: 0, components: [] }, + message_link_to_before_ooye: { + id: "1160824382755708948", + type: 0, + content: "Me: I'll scroll up to find a certain message I'll send\n" + + "_scrolls up and clicks message links for god knows how long_\n" + + "_completely forgets what they were looking for and simply begins scrolling up to find some fun moments_\n" + + "_stumbles upon:_ https://discord.com/channels/112760669178241024/112760669178241024/810412561941921851", + channel_id: "112760669178241024", + author: { + id: "271237147401045000", + username: "jinx", + avatar: "a0ba563c16aff137289f67f38545807f", + discriminator: "0", + public_flags: 0, + premium_type: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "Jinx", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: '2023-10-09T06:21:39.923000+00:00', + edited_timestamp: null, + flags: 0, + components: [] + }, simple_written_at_mention_for_matrix: { id: "1159030564049915915", type: 0, diff --git a/test/test.js b/test/test.js index 90e3f5a..553ec44 100644 --- a/test/test.js +++ b/test/test.js @@ -45,6 +45,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not await p db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8")) require("../db/orm.test") + require("../discord/utils.test") require("../matrix/kstate.test") require("../matrix/api.test") require("../matrix/file.test")