From 936c9820ec9f13383d1516782696fa75e94ca884 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 26 Aug 2023 18:58:41 +1200 Subject: [PATCH 1/6] store room name changes to nick in db --- m2d/event-dispatcher.js | 11 +++++++++++ types.d.ts | 11 ++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/m2d/event-dispatcher.js b/m2d/event-dispatcher.js index c62d805..3425fdb 100644 --- a/m2d/event-dispatcher.js +++ b/m2d/event-dispatcher.js @@ -80,3 +80,14 @@ async event => { const url = event.content.url || null db.prepare("UPDATE channel_room SET custom_avatar = ? WHERE room_id = ?").run(url, event.room_id) })) + +sync.addTemporaryListener(as, "type:m.room.name", guard("m.room.name", +/** + * @param {Ty.Event.StateOuter} event + */ +async event => { + if (event.state_key !== "") return + if (utils.eventSenderIsFromDiscord(event.sender)) return + const name = event.content.name || null + db.prepare("UPDATE channel_room SET nick = ? WHERE room_id = ?").run(name, event.room_id) +})) diff --git a/types.d.ts b/types.d.ts index badbbab..2bf0af0 100644 --- a/types.d.ts +++ b/types.d.ts @@ -70,7 +70,12 @@ export namespace Event { msgtype: "m.text" | "m.emote" body: string format?: "org.matrix.custom.html" - formatted_body?: string + formatted_body?: string, + "m.relates_to"?: { + "m.in_reply_to": { + event_id: string + } + } } export type M_Room_Member = { @@ -84,6 +89,10 @@ export namespace Event { url?: string } + export type M_Room_Name = { + name?: string + } + export type M_Reaction = { "m.relates_to": { rel_type: "m.annotation" From 300197c157faee54ac826abafd115d52119544f3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 26 Aug 2023 19:07:19 +1200 Subject: [PATCH 2/6] fix m->d formatting of quotes and code --- m2d/converters/event-to-message.js | 38 ++++++++++++++++++ m2d/converters/event-to-message.test.js | 52 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 7893799..f4382a3 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -33,6 +33,40 @@ turndownService.addRule("strikethrough", { } }) +turndownService.addRule("blockquote", { + filter: "blockquote", + replacement: function (content) { + content = content.replace(/^\n+|\n+$/g, "") + content = content.replace(/^/gm, "> ") + return content + } +}) + +turndownService.addRule("fencedCodeBlock", { + filter: function (node, options) { + return ( + options.codeBlockStyle === "fenced" && + node.nodeName === "PRE" && + node.firstChild && + node.firstChild.nodeName === "CODE" + ) + }, + replacement: function (content, node, options) { + const className = node.firstChild.getAttribute("class") || "" + const language = (className.match(/language-(\S+)/) || [null, ""])[1] + const code = node.firstChild + const visibleCode = code.childNodes.map(c => c.nodeName === "BR" ? "\n" : c.textContent).join("").replace(/\n*$/g, "") + + var fence = "```" + + return ( + fence + language + "\n" + + visibleCode + + "\n" + fence + ) + } +}) + /** * @param {Ty.Event.Outer} event */ @@ -61,9 +95,13 @@ function eventToMessage(event) { // input = input.replace(/ /g, " ") // There is also a corresponding test to uncomment, named "event2message: whitespace is retained" + // Element adds a bunch of
before but doesn't render them. I can't figure out how this works, so let's just delete those. + input = input.replace(/(?:\n|
\s*)*<\/blockquote>/g, "") + // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. // But I should not count it if it's between block elements. input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => { + // console.error(beforeContext, beforeTag, afterContext, afterTag) if (typeof beforeTag !== "string" && typeof afterTag !== "string") { return "
" } diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 7aefdb1..afa40de 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -271,6 +271,58 @@ test("event2message: code blocks work", t => { ) }) +test("event2message: code block contents are formatted correctly and not escaped", t => { + t.deepEqual( + eventToMessage({ + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "wrong body", + "format": "org.matrix.custom.html", + "formatted_body": "
input = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,\n_input_ = input = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,\n
\n

input = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,

\n" + }, + "origin_server_ts": 1693031482275, + "unsigned": { + "age": 99, + "transaction_id": "m1693031482146.511" + }, + "event_id": "$pGkWQuGVmrPNByrFELxhzI6MCBgJecr5I2J3z88Gc2s", + "room_id": "!BpMdOUkWWhFxmTrENV:cadence.moe" + }), + [{ + username: "cadence", + content: "```\ninput = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,\n_input_ = input = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,\n```\n\n`input = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,`", + avatar_url: undefined + }] + ) +}) + +test("event2message: quotes have an appropriate amount of whitespace", t => { + t.deepEqual( + eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: "
Chancellor of Germany Angela Merkel, on March 17, 2017: they did not shake hands



🤨" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + [{ + username: "cadence", + content: "> Chancellor of Germany Angela Merkel, on March 17, 2017: they did not shake hands\n🤨", + avatar_url: undefined + }] + ) +}) test("event2message: m.emote markdown syntax is escaped", t => { t.deepEqual( From 2eb9abef401662f43acb37bccdd68730611633d7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 26 Aug 2023 20:30:22 +1200 Subject: [PATCH 3/6] m->d rich replies --- db/data-for-test.sql | 6 +- m2d/actions/send-event.js | 8 +- m2d/converters/event-to-message.js | 37 +++- m2d/converters/event-to-message.test.js | 213 ++++++++++++++++++++---- 4 files changed, 231 insertions(+), 33 deletions(-) diff --git a/db/data-for-test.sql b/db/data-for-test.sql index 7148b19..b8da73c 100644 --- a/db/data-for-test.sql +++ b/db/data-for-test.sql @@ -67,7 +67,8 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES ('820865262526005258', 'crunch_god', '_ooye_crunch_god', '@_ooye_crunch_god:cadence.moe'), ('771520384671416320', 'bojack_horseman', '_ooye_bojack_horseman', '@_ooye_bojack_horseman:cadence.moe'), ('112890272819507200', '.wing.', '_ooye_.wing.', '@_ooye_.wing.:cadence.moe'), -('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'); +('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'), +('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');; INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES ('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL); @@ -86,7 +87,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, chan ('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', '1100319550446252084', 0, 1), ('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', '122155380120748034', 0, 1), ('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', '122155380120748034', 0, 0), -('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', '122155380120748034', 0, 0); +('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', '122155380120748034', 0, 0), +('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', '687028734322147344', 0, 1); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js index 39eed22..016768e 100644 --- a/m2d/actions/send-event.js +++ b/m2d/actions/send-event.js @@ -9,6 +9,8 @@ const {sync, discord, db} = passthrough const channelWebhook = sync.require("./channel-webhook") /** @type {import("../converters/event-to-message")} */ const eventToMessage = sync.require("../converters/event-to-message") +/** @type {import("../../matrix/api")}) */ +const api = sync.require("../../matrix/api") /** @param {import("../../types").Event.Outer} event */ async function sendEvent(event) { @@ -20,10 +22,14 @@ async function sendEvent(event) { threadID = channelID channelID = row.thread_parent // it's the thread's parent... get with the times... } + // @ts-ignore + const guildID = discord.channels.get(channelID).guild_id + const guild = discord.guilds.get(guildID) + assert(guild) // no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it - const messages = eventToMessage.eventToMessage(event) + const messages = await eventToMessage.eventToMessage(event, guild, {api}) assert(Array.isArray(messages)) // sanity /** @type {DiscordTypes.APIMessage[]} */ diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index f4382a3..cf34705 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -26,6 +26,8 @@ const turndownService = new TurndownService({ codeBlockStyle: "fenced" }) +turndownService.remove("mx-reply") + turndownService.addRule("strikethrough", { filter: ["del", "s", "strike"], replacement: function (content) { @@ -69,13 +71,16 @@ turndownService.addRule("fencedCodeBlock", { /** * @param {Ty.Event.Outer} event + * @param {import("discord-api-types/v10").APIGuild} guild + * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API */ -function eventToMessage(event) { +async function eventToMessage(event, guild, di) { /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]})[]} */ let messages = [] let displayName = event.sender let avatarURL = undefined + let replyLine = "" const match = event.sender.match(/^@(.*?):/) if (match) { displayName = match[1] @@ -95,7 +100,33 @@ function eventToMessage(event) { // input = input.replace(/ /g, " ") // There is also a corresponding test to uncomment, named "event2message: whitespace is retained" - // Element adds a bunch of
before but doesn't render them. I can't figure out how this works, so let's just delete those. + // Handling replies. We'll look up the data of the replied-to event from the Matrix homeserver. + await (async () => { + const repliedToEventId = event.content["m.relates_to"]?.["m.in_reply_to"].event_id + if (!repliedToEventId) return + const repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId) + if (!repliedToEvent) return + const row = db.prepare("SELECT channel_id, message_id FROM event_message WHERE event_id = ? ORDER BY part").get(repliedToEventId) + if (row) { + replyLine = `<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} ` + } else { + replyLine = `<:L1:1144820033948762203><:L2:1144820084079087647>` + } + const sender = repliedToEvent.sender + const senderName = sender.match(/@([^:]*)/)?.[1] || sender + const authorID = db.prepare("SELECT discord_id FROM sim WHERE mxid = ?").pluck().get(repliedToEvent.sender) + if (authorID) { + replyLine += `<@${authorID}>: ` + } else { + replyLine += `Ⓜ️**${senderName}**: ` + } + const repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body + const contentPreviewChunks = chunk(repliedToContent.replace(/.*<\/mx-reply>/, "").replace(/(?:\n|
)+/g, " ").replace(/<[^>]+>/g, ""), 24) + const contentPreview = contentPreviewChunks.length > 1 ? contentPreviewChunks[0] + "..." : contentPreviewChunks[0] + replyLine += contentPreview + "\n" + })() + + // Element adds a bunch of
before but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those. input = input.replace(/(?:\n|
\s*)*<\/blockquote>/g, "") // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. @@ -127,6 +158,8 @@ function eventToMessage(event) { content = content.replace(/([*_~`#])/g, `\\$1`) } + content = replyLine + content + // Split into 2000 character chunks const chunks = chunk(content, 2000) messages = messages.concat(chunks.map(content => ({ diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index afa40de..33c4d71 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1,18 +1,42 @@ -// @ts-check - const {test} = require("supertape") const {eventToMessage} = require("./event-to-message") const data = require("../../test/data") +/** + * @param {string} roomID + * @param {string} eventID + * @returns {(roomID: string, eventID: string) => Promise>} + */ +function mockGetEvent(t, roomID_in, eventID_in, outer) { + return async function(roomID, eventID) { + t.equal(roomID, roomID_in) + t.equal(eventID, eventID_in) + return new Promise(resolve => { + setTimeout(() => { + resolve({ + event_id: eventID_in, + room_id: roomID_in, + origin_server_ts: 1680000000000, + unsigned: { + age: 2245, + transaction_id: "$local.whatever" + }, + ...outer + }) + }) + }) + } +} + function sameFirstContentAndWhitespace(t, a, b) { const a2 = JSON.stringify(a[0].content) const b2 = JSON.stringify(b[0].content) t.equal(a2, b2) } -test("event2message: body is used when there is no formatted_body", t => { +test("event2message: body is used when there is no formatted_body", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { body: "testing plaintext", msgtype: "m.text" @@ -34,9 +58,9 @@ test("event2message: body is used when there is no formatted_body", t => { ) }) -test("event2message: any markdown in body is escaped", t => { +test("event2message: any markdown in body is escaped", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { body: "testing **special** ~~things~~ which _should_ *not* `trigger` @any ", msgtype: "m.text" @@ -58,9 +82,9 @@ test("event2message: any markdown in body is escaped", t => { ) }) -test("event2message: basic html is converted to markdown", t => { +test("event2message: basic html is converted to markdown", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.text", body: "wrong body", @@ -84,9 +108,9 @@ test("event2message: basic html is converted to markdown", t => { ) }) -test("event2message: markdown syntax is escaped", t => { +test("event2message: markdown syntax is escaped", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.text", body: "wrong body", @@ -110,9 +134,9 @@ test("event2message: markdown syntax is escaped", t => { ) }) -test("event2message: html lines are bridged correctly", t => { +test("event2message: html lines are bridged correctly", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.text", body: "wrong body", @@ -136,9 +160,9 @@ test("event2message: html lines are bridged correctly", t => { ) }) -/*test("event2message: whitespace is retained", t => { +/*test("event2message: whitespace is retained", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.text", body: "wrong body", @@ -162,10 +186,10 @@ test("event2message: html lines are bridged correctly", t => { ) })*/ -test("event2message: whitespace is collapsed", t => { +test("event2message: whitespace is collapsed", async t => { sameFirstContentAndWhitespace( t, - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.text", body: "wrong body", @@ -189,10 +213,10 @@ test("event2message: whitespace is collapsed", t => { ) }) -test("event2message: lists are bridged correctly", t => { +test("event2message: lists are bridged correctly", async t => { sameFirstContentAndWhitespace( t, - eventToMessage({ + await eventToMessage({ "type": "m.room.message", "sender": "@cadence:cadence.moe", "content": { @@ -217,9 +241,9 @@ test("event2message: lists are bridged correctly", t => { ) }) -test("event2message: long messages are split", t => { +test("event2message: long messages are split", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { body: ("a".repeat(130) + " ").repeat(19), msgtype: "m.text" @@ -245,9 +269,9 @@ test("event2message: long messages are split", t => { ) }) -test("event2message: code blocks work", t => { +test("event2message: code blocks work", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.text", body: "wrong body", @@ -271,9 +295,9 @@ test("event2message: code blocks work", t => { ) }) -test("event2message: code block contents are formatted correctly and not escaped", t => { +test("event2message: code block contents are formatted correctly and not escaped", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ "type": "m.room.message", "sender": "@cadence:cadence.moe", "content": { @@ -298,9 +322,9 @@ test("event2message: code block contents are formatted correctly and not escaped ) }) -test("event2message: quotes have an appropriate amount of whitespace", t => { +test("event2message: quotes have an appropriate amount of whitespace", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.text", body: "wrong body", @@ -324,9 +348,9 @@ test("event2message: quotes have an appropriate amount of whitespace", t => { ) }) -test("event2message: m.emote markdown syntax is escaped", t => { +test("event2message: m.emote markdown syntax is escaped", async t => { t.deepEqual( - eventToMessage({ + await eventToMessage({ content: { msgtype: "m.emote", body: "wrong body", @@ -349,3 +373,136 @@ test("event2message: m.emote markdown syntax is escaped", t => { }] ) }) + +test("event2message: rich reply to a sim user", async t => { + t.deepEqual( + await eventToMessage({ + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "> <@_ooye_kyuugryphon:cadence.moe> Slow news day.\n\nTesting this reply, ignore", + "format": "org.matrix.custom.html", + "formatted_body": "
In reply to @_ooye_kyuugryphon:cadence.moe
Slow news day.
Testing this reply, ignore", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04" + } + } + }, + "origin_server_ts": 1693029683016, + "unsigned": { + "age": 91, + "transaction_id": "m1693029682894.510" + }, + "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { + type: "m.room.message", + content: { + msgtype: "m.text", + body: "Slow news day." + }, + sender: "@_ooye_kyuugryphon:cadence.moe" + }) + } + }), + [{ + username: "cadence", + content: "<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 <@111604486476181504>: Slow news day.\nTesting this reply, ignore", + avatar_url: undefined + }] + ) +}) + +test("event2message: rich reply to a matrix user's long message with formatting", async t => { + t.deepEqual( + await eventToMessage({ + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "> <@cadence:cadence.moe> ```\n> i should have a little happy test\n> ```\n> * list **bold** _em_ ~~strike~~\n> # heading 1\n> ## heading 2\n> ### heading 3\n> https://cadence.moe\n> [legit website](https://cadence.moe)\n\nno you can't!!!", + "format": "org.matrix.custom.html", + "formatted_body": "
In reply to @cadence:cadence.moe
i should have a little happy test\n
\n
    \n
  • list bold em ~~strike~~
    • \n
\n

heading 1

\n

heading 2

\n

heading 3

\n

https://cadence.moe
legit website

\n
no you can't!!!", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04" + } + } + }, + "origin_server_ts": 1693037401693, + "unsigned": { + "age": 381, + "transaction_id": "m1693037401592.521" + }, + "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "```\ni should have a little happy test\n```\n* list **bold** _em_ ~~strike~~\n# heading 1\n## heading 2\n### heading 3\nhttps://cadence.moe\n[legit website](https://cadence.moe)", + "format": "org.matrix.custom.html", + "formatted_body": "
i should have a little happy test\n
\n
    \n
  • list bold em ~~strike~~
    • \n
\n

heading 1

\n

heading 2

\n

heading 3

\n

https://cadence.moe
legit website

\n" + } + }) + } + }), + [{ + username: "cadence", + content: "<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**: i should have a little...\n**no you can't!!!**", + avatar_url: undefined + }] + ) +}) + +test("event2message: with layered rich replies, the preview should only be the real text", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "> <@cadence:cadence.moe> two\n\nthree", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @cadence:cadence.moe
two
three", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04" + } + } + }, + event_id: "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "> <@cadence:cadence.moe> one\n\ntwo", + "format": "org.matrix.custom.html", + "formatted_body": "
In reply to @cadence:cadence.moe
one
two", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$5UtboIC30EFlAYD_Oh0pSYVW8JqOp6GsDIJZHtT0Wls" + } + } + } + }) + } + }), + [{ + username: "cadence", + content: "<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**: two\nthree", + avatar_url: undefined + }] + ) +}) From fd65a57a4c7cf1524e650ab8cdde496a0165c694 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 26 Aug 2023 22:22:54 +1200 Subject: [PATCH 4/6] show member details on discord from cache --- db/data-for-test.sql | 12 +++++++ m2d/converters/event-to-message.js | 28 ++++++++++++++--- m2d/converters/event-to-message.test.js | 42 ++++++++++++------------- m2d/converters/utils.js | 11 +++++++ 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/db/data-for-test.sql b/db/data-for-test.sql index b8da73c..8c564df 100644 --- a/db/data-for-test.sql +++ b/db/data-for-test.sql @@ -47,6 +47,13 @@ CREATE TABLE IF NOT EXISTS "event_message" ( "source" INTEGER NOT NULL, PRIMARY KEY("event_id","message_id") ); +CREATE TABLE IF NOT EXISTS "member_cache" ( + "room_id" TEXT NOT NULL, + "mxid" TEXT NOT NULL, + "displayname" TEXT, + "avatar_url" TEXT, + PRIMARY KEY("room_id", "mxid") +); COMMIT; @@ -101,4 +108,9 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024', 'mxc://cadence.moe/zKXGZhmImMHuGQZWJEFKJbsF'), ('https://cdn.discordapp.com/avatars/113340068197859328/b48302623a12bc7c59a71328f72ccb39.png?size=1024', 'mxc://cadence.moe/UpAeIqeclhKfeiZNdIWNcXXL'); +INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES +('!kLRqKKUQXcibIMtOpl:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', NULL), +('!BpMdOUkWWhFxmTrENV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', NULL), +('!fGgIymcYWOqjbSRUdV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'); + COMMIT; diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index cf34705..8363d8f 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -9,6 +9,8 @@ const passthrough = require("../../passthrough") const { sync, db, discord } = passthrough /** @type {import("../../matrix/file")} */ const file = sync.require("../../matrix/file") +/** @type {import("../converters/utils")} */ +const utils = sync.require("../converters/utils") const BLOCK_ELEMENTS = [ "ADDRESS", "ARTICLE", "ASIDE", "AUDIO", "BLOCKQUOTE", "BODY", "CANVAS", @@ -69,6 +71,22 @@ turndownService.addRule("fencedCodeBlock", { } }) +/** + * @param {string} roomID + * @param {string} mxid + * @returns {Promise<{displayname?: string?, avatar_url?: string?}>} + */ +async function getMemberFromCacheOrHomeserver(roomID, mxid, api) { + const row = db.prepare("SELECT displayname, avatar_url FROM member_cache WHERE room_id = ? AND mxid = ?").get(roomID, mxid) + if (row) return row + return api.getStateEvent(roomID, "m.room.member", mxid).then(event => { + db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(roomID, mxid, event?.displayname || null, event?.avatar_url || null) + return event + }).catch(() => { + return {displayname: null, avatar_url: null} + }) +} + /** * @param {Ty.Event.Outer} event * @param {import("discord-api-types/v10").APIGuild} guild @@ -81,11 +99,13 @@ async function eventToMessage(event, guild, di) { let displayName = event.sender let avatarURL = undefined let replyLine = "" + // Extract a basic display name from the sender const match = event.sender.match(/^@(.*?):/) - if (match) { - displayName = match[1] - // TODO: get the media repo domain and the avatar url from the matrix member event - } + if (match) displayName = match[1] + // Try to extract an accurate display name and avatar URL from the member event + const member = await getMemberFromCacheOrHomeserver(event.room_id, event.sender, di?.api) + if (member.displayname) displayName = member.displayname + if (member.avatar_url) avatarURL = utils.getPublicUrlForMxc(member.avatar_url) // Convert content depending on what the message is let content = event.content.body // ultimate fallback diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 33c4d71..b595be8 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -51,7 +51,7 @@ test("event2message: body is used when there is no formatted_body", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "testing plaintext", avatar_url: undefined }] @@ -75,7 +75,7 @@ test("event2message: any markdown in body is escaped", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "testing \\*\\*special\\*\\* \\~\\~things\\~\\~ which \\_should\\_ \\*not\\* \\`trigger\\` @any ", avatar_url: undefined }] @@ -101,7 +101,7 @@ test("event2message: basic html is converted to markdown", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "this **is** a **_test_** of ~~formatting~~", avatar_url: undefined }] @@ -127,7 +127,7 @@ test("event2message: markdown syntax is escaped", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "this \\*\\*is\\*\\* an **_extreme_** \\\\\\*test\\\\\\* of", avatar_url: undefined }] @@ -153,7 +153,7 @@ test("event2message: html lines are bridged correctly", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "paragraph one\nline _two_\nline three\n\nparagraph two\nline _two_\nline three\n\nparagraph three\n\nparagraph four\nline two\nline three\nline four\n\nparagraph five", avatar_url: undefined }] @@ -179,7 +179,7 @@ test("event2message: html lines are bridged correctly", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "line one: test test\nline two: **test** **test**\nline three: **test test**\nline four: test test\n line five", avatar_url: undefined }] @@ -206,7 +206,7 @@ test("event2message: whitespace is collapsed", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "line one: test test\nline two: **test** **test**\nline three: **test test**\nline four: test test\nline five", avatar_url: undefined }] @@ -234,7 +234,7 @@ test("event2message: lists are bridged correctly", async t => { "room_id": "!BpMdOUkWWhFxmTrENV:cadence.moe" }), [{ - username: "cadence", + username: "cadence [they]", content: "* line one\n* line two\n* line three\n * nested one\n * nested two\n* line four", avatar_url: undefined }] @@ -258,11 +258,11 @@ test("event2message: long messages are split", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: (("a".repeat(130) + " ").repeat(15)).slice(0, -1), avatar_url: undefined }, { - username: "cadence", + username: "cadence [they]", content: (("a".repeat(130) + " ").repeat(4)).slice(0, -1), avatar_url: undefined }] @@ -288,7 +288,7 @@ test("event2message: code blocks work", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "preceding\n\n```\ncode block\n```\n\nfollowing `code` is inline", avatar_url: undefined }] @@ -315,7 +315,7 @@ test("event2message: code block contents are formatted correctly and not escaped "room_id": "!BpMdOUkWWhFxmTrENV:cadence.moe" }), [{ - username: "cadence", + username: "cadence [they]", content: "```\ninput = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,\n_input_ = input = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,\n```\n\n`input = input.replace(/(<\\/?([^ >]+)[^>]*>)?\\n(<\\/?([^ >]+)[^>]*>)?/g,`", avatar_url: undefined }] @@ -341,7 +341,7 @@ test("event2message: quotes have an appropriate amount of whitespace", async t = } }), [{ - username: "cadence", + username: "cadence [they]", content: "> Chancellor of Germany Angela Merkel, on March 17, 2017: they did not shake hands\n🤨", avatar_url: undefined }] @@ -367,8 +367,8 @@ test("event2message: m.emote markdown syntax is escaped", async t => { } }), [{ - username: "cadence", - content: "\\* cadence shows you \\*\\*her\\*\\* **_extreme_** \\\\\\*test\\\\\\* of", + username: "cadence [they]", + content: "\\* cadence \\[they\\] shows you \\*\\*her\\*\\* **_extreme_** \\\\\\*test\\\\\\* of", avatar_url: undefined }] ) @@ -410,9 +410,9 @@ test("event2message: rich reply to a sim user", async t => { } }), [{ - username: "cadence", + username: "cadence [they]", content: "<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 <@111604486476181504>: Slow news day.\nTesting this reply, ignore", - avatar_url: undefined + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" }] ) }) @@ -455,9 +455,9 @@ test("event2message: rich reply to a matrix user's long message with formatting" } }), [{ - username: "cadence", + username: "cadence [they]", content: "<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**: i should have a little...\n**no you can't!!!**", - avatar_url: undefined + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" }] ) }) @@ -500,9 +500,9 @@ test("event2message: with layered rich replies, the preview should only be the r } }), [{ - username: "cadence", + username: "cadence [they]", content: "<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**: two\nthree", - avatar_url: undefined + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" }] ) }) diff --git a/m2d/converters/utils.js b/m2d/converters/utils.js index 7b9c504..02ec147 100644 --- a/m2d/converters/utils.js +++ b/m2d/converters/utils.js @@ -19,4 +19,15 @@ function eventSenderIsFromDiscord(sender) { return false } +/** + * @param {string} mxc + * @returns {string?} + */ +function getPublicUrlForMxc(mxc) { + const avatarURLParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) + if (avatarURLParts) return `https://matrix.cadence.moe/_matrix/media/r0/download/${avatarURLParts[1]}/${avatarURLParts[2]}` + else return null +} + module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord +module.exports.getPublicUrlForMxc = getPublicUrlForMxc From 6d9a58a31bad6d1941bee3aed52e2024028b06ca Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 26 Aug 2023 22:50:54 +1200 Subject: [PATCH 5/6] preemptively cache members as we find them --- m2d/converters/event-to-message.js | 2 +- m2d/event-dispatcher.js | 10 ++++++++++ types.d.ts | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 8363d8f..f5d3c90 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -80,7 +80,7 @@ async function getMemberFromCacheOrHomeserver(roomID, mxid, api) { const row = db.prepare("SELECT displayname, avatar_url FROM member_cache WHERE room_id = ? AND mxid = ?").get(roomID, mxid) if (row) return row return api.getStateEvent(roomID, "m.room.member", mxid).then(event => { - db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(roomID, mxid, event?.displayname || null, event?.avatar_url || null) + db.prepare("REPLACE INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(roomID, mxid, event?.displayname || null, event?.avatar_url || null) return event }).catch(() => { return {displayname: null, avatar_url: null} diff --git a/m2d/event-dispatcher.js b/m2d/event-dispatcher.js index 3425fdb..9a575fc 100644 --- a/m2d/event-dispatcher.js +++ b/m2d/event-dispatcher.js @@ -91,3 +91,13 @@ async event => { const name = event.content.name || null db.prepare("UPDATE channel_room SET nick = ? WHERE room_id = ?").run(name, event.room_id) })) + +sync.addTemporaryListener(as, "type:m.room.member", guard("m.room.member", +/** + * @param {Ty.Event.StateOuter} event + */ +async event => { + if (event.state_key[0] !== "@") return + if (utils.eventSenderIsFromDiscord(event.sender)) return + db.prepare("REPLACE INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(event.room_id, event.sender, event.content.displayname || null, event.content.avatar_url || null) +})) diff --git a/types.d.ts b/types.d.ts index 2bf0af0..5475904 100644 --- a/types.d.ts +++ b/types.d.ts @@ -80,7 +80,7 @@ export namespace Event { export type M_Room_Member = { membership: string - display_name?: string + displayname?: string avatar_url?: string } From fa1e01215c4fc0b569ca7007722afe2a388b8a77 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 26 Aug 2023 22:51:42 +1200 Subject: [PATCH 6/6] trying to make reaction emojis consistent --- m2d/actions/add-reaction.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/m2d/actions/add-reaction.js b/m2d/actions/add-reaction.js index 68828dd..49aa845 100644 --- a/m2d/actions/add-reaction.js +++ b/m2d/actions/add-reaction.js @@ -21,9 +21,20 @@ async function addReaction(event) { let encoded = encodeURIComponent(emoji) let encodedTrimmed = encoded.replace(/%EF%B8%8F/g, "") - console.log("add reaction from matrix:", emoji, encoded, encodedTrimmed) + // https://github.com/discord/discord-api-docs/issues/2723#issuecomment-807022205 ???????????? - return discord.snow.channel.createReaction(channelID, messageID, encoded) + const forceTrimmedList = [ + "%E2%AD%90" // ⭐ + ] + + let discordPreferredEncoding = + ( forceTrimmedList.includes(encodedTrimmed) ? encodedTrimmed + : encodedTrimmed !== encoded && [...emoji].length === 2 ? encoded + : encodedTrimmed) + + console.log("add reaction from matrix:", emoji, encoded, encodedTrimmed, "chosen:", discordPreferredEncoding) + + return discord.snow.channel.createReaction(channelID, messageID, discordPreferredEncoding) } module.exports.addReaction = addReaction