From f79833c4443ad937d6666da33fa24beee0e295d7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Feb 2024 20:47:10 +1300 Subject: [PATCH 1/5] m->d: Only care about data-mx-spoiler on span --- m2d/converters/event-to-message.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 2dcb2e8..1507ce3 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -87,7 +87,7 @@ turndownService.addRule("blockquote", { turndownService.addRule("spoiler", { filter: function (node, options) { - return node.hasAttribute("data-mx-spoiler") + return node.tagName === "SPAN" && node.hasAttribute("data-mx-spoiler") }, replacement: function (content, node) { From 7756a34a5fb70bb3cb3ff49ec62d3420ab6c1e91 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Feb 2024 22:27:55 +1300 Subject: [PATCH 2/5] m->d: Gracefully handle replies to redacted event --- m2d/converters/event-to-message.js | 2 + m2d/converters/event-to-message.test.js | 55 +++++++++++++++++++++++++ test/ooye-test-data.sql | 3 +- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 1507ce3..7cbf2dc 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -509,6 +509,8 @@ async function eventToMessage(event, guild, di) { const fileReplyContentAlternative = attachmentEmojis.get(repliedToEvent.content.msgtype) if (fileReplyContentAlternative) { contentPreview = " " + fileReplyContentAlternative + } else if (repliedToEvent.unsigned?.redacted_because) { + contentPreview = " (in reply to a deleted message)" } else { const repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body const contentPreviewChunks = chunk( diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index f116f8c..7d7aea9 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1778,6 +1778,61 @@ test("event2message: with layered rich replies, the preview should only be the r ) }) +test("event2message: rich reply to a deleted event", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@ampflower:matrix.org", + content: { + msgtype: "m.text", + body: "> <@ampflower:matrix.org> \n\nHuh it did the same thing here too", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @ampflower:matrix.org
Huh it did the same thing here too", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$f-noT-d-Eo_Xgpc05Ww89ErUXku4NwKWYGHLzWKo1kU" + } + } + }, + event_id: "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$f-noT-d-Eo_Xgpc05Ww89ErUXku4NwKWYGHLzWKo1kU", { + type: "m.room.message", + sender: "@ampflower:matrix.org", + content: {}, + origin_server_ts: 1707798292953, + unsigned: { + redacted_because: { + type: "m.room.redaction", + room_id: "!TqlyQmifxGUggEmdBN:cadence.moe", + sender: "@_ooye_bot:cadence.moe", + content: {}, + redacts: "$uyOzmYhqcgF5i0bZb4MrAIEKEvzDOLgXdlRr1zfvWo0", + origin_server_ts: 1707798294565, + event_id: "$enCV-40Sut8llwALAV0T3qjwK7MvO9jgY9C4DHbxKXA", + user_id: "@_ooye_bot:cadence.moe", + }, + }, + user_id: "@ampflower:matrix.org" + }) + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "Ampflower 🌺", + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>Ⓜ️**Ampflower 🌺** (in reply to a deleted message)" + + "\nHuh it did the same thing here too", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/PRfhXYBTOalvgQYtmCLeUXko" + }] + } + ) +}) + test("event2message: raw mentioning discord users in plaintext body works", async t => { t.deepEqual( await eventToMessage({ diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index f455d9e..644f732 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -115,7 +115,8 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES ('!BnKuBPCvyfOkhcUjEu:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), -('!cBxtVRxDlZvSVhJXVK:cadence.moe', '@Milan:tchncs.de', 'Milan', NULL); +('!cBxtVRxDlZvSVhJXVK:cadence.moe', '@Milan:tchncs.de', 'Milan', NULL), +('!TqlyQmifxGUggEmdBN:cadence.moe', '@ampflower:matrix.org', 'Ampflower 🌺', 'mxc://cadence.moe/PRfhXYBTOalvgQYtmCLeUXko'); INSERT INTO lottie (sticker_id, mxc_url) VALUES ('860171525772279849', 'mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR'); From 355ebfe2af594dae0961e59068c96dce0d4223f7 Mon Sep 17 00:00:00 2001 From: Wonder Collective <> Date: Tue, 13 Feb 2024 10:06:30 +0100 Subject: [PATCH 3/5] m->d: spoiler reasons & reply-quote separation a few m2d converter improvements --- m2d/converters/event-to-message.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 7cbf2dc..ce5cfcd 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -33,6 +33,12 @@ const markdownEscapes = [ [/^>/g, '\\>'], [/_/g, '\\_'], [/^(\d+)\. /g, '$1\\. '] + /* + Strikethrough is deliberately not escaped. Usually when Matrix users type ~~ it's not because they wanted to send ~~, + it's because they wanted strikethrough and it didn't work because their client doesn't support it. + As bridge developers, we can choose between "messages should look as similar as possible" vs "it was most likely intended to be strikethrough". + I went with the latter. Even though the appearance doesn't match, I'd rather it displayed as originally intended for 80% of the readers than for 0%. + */ ] const turndownService = new TurndownService({ @@ -91,7 +97,11 @@ turndownService.addRule("spoiler", { }, replacement: function (content, node) { - return "||" + content + "||" + if (node.getAttribute("data-mx-spoiler")) { + // escape parentheses so it can't become a link + return `\\(${node.getAttribute("data-mx-spoiler")}\\) ||${content}||` + } + return `||${content}||` } }) @@ -630,6 +640,9 @@ async function eventToMessage(event, guild, di) { // It's designed for commonmark, we need to replace the space-space-newline with just newline content = content.replace(/ \n/g, "\n") + // If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated + if (replyLine && content.startsWith("> ")) content = "\n" + content + // SPRITE SHEET EMOJIS FEATURE: content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles) } else { From e999fcf81987faa5dfcc405877c2b3df40caf479 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Feb 2024 22:53:31 +1300 Subject: [PATCH 4/5] m->d: test: Line break between reply and quote msg --- m2d/converters/event-to-message.test.js | 54 +++++++++++++++++++++++++ test/ooye-test-data.sql | 3 +- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 7d7aea9..d8dd3fb 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1778,6 +1778,60 @@ test("event2message: with layered rich replies, the preview should only be the r ) }) +test("event2message: if event is a reply and starts with a quote, they should be separated by a blank line, so that they don't visually merge together", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@aflower:syndicated.gay", + content: { + body: "> <@aflower:syndicated.gay> i have a feeling that clients are *meant to* strip these reply fallbacks too, just that none of them, in reality, do\n\n>To strip the fallback on the body, the client should iterate over each line of the string, removing any lines that start with the fallback prefix ("> “, including the space, without quotes) and stopping when a line is encountered without the prefix. This prefix is known as the “fallback prefix sequence”.", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @aflower:syndicated.gay
i have a feeling that clients are meant to strip these reply fallbacks too, just that none of them, in reality, do
\n

To strip the fallback on the body, the client should iterate over each line of the string, removing any lines that start with the fallback prefix ("> “, including the space, without quotes) and stopping when a line is encountered without the prefix. This prefix is known as the “fallback prefix sequence”.

\n
", + "im.nheko.relations.v1.relations": [ + { + event_id: "$tTYQcke93fwocsc1K6itwUq85EG0RZ0ksCuIglKioks", + rel_type: "im.nheko.relations.v1.in_reply_to" + } + ], + "m.relates_to": { + "m.in_reply_to": { + event_id: "$tTYQcke93fwocsc1K6itwUq85EG0RZ0ksCuIglKioks" + } + }, + msgtype: "m.text" + }, + room_id: "!TqlyQmifxGUggEmdBN:cadence.moe", + event_id: "$nCvtZeBFedYuEavt4OftloCHc0kaFW2ktHCfIOklhjU", + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$tTYQcke93fwocsc1K6itwUq85EG0RZ0ksCuIglKioks", { + sender: "@aflower:syndicated.gay", + type: "m.room.message", + content: { + body: "i have a feeling that clients are *meant to* strip these reply fallbacks too, just that none of them, in reality, do", + format: "org.matrix.custom.html", + formatted_body: "i have a feeling that clients are meant to strip these reply fallbacks too, just that none of them, in reality, do", + msgtype: "m.text" + } + }) + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "Rose", + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>Ⓜ️**Rose**:" + + "\n> i have a feeling that clients are meant to strip..." + + "\n" + + "\n> To strip the fallback on the `body`, the client should iterate over each line of the string, removing any lines that start with the fallback prefix (\"> “, including the space, without quotes) and stopping when a line is encountered without the prefix. This prefix is known as the “fallback prefix sequence”.", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/syndicated.gay/ZkBUPXCiXTjdJvONpLJmcbKP" + }] + } + ) +}) + test("event2message: rich reply to a deleted event", async t => { t.deepEqual( await eventToMessage({ diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 644f732..023d5bb 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -116,7 +116,8 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES ('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!cBxtVRxDlZvSVhJXVK:cadence.moe', '@Milan:tchncs.de', 'Milan', NULL), -('!TqlyQmifxGUggEmdBN:cadence.moe', '@ampflower:matrix.org', 'Ampflower 🌺', 'mxc://cadence.moe/PRfhXYBTOalvgQYtmCLeUXko'); +('!TqlyQmifxGUggEmdBN:cadence.moe', '@ampflower:matrix.org', 'Ampflower 🌺', 'mxc://cadence.moe/PRfhXYBTOalvgQYtmCLeUXko'), +('!TqlyQmifxGUggEmdBN:cadence.moe', '@aflower:syndicated.gay', 'Rose', 'mxc://syndicated.gay/ZkBUPXCiXTjdJvONpLJmcbKP'); INSERT INTO lottie (sticker_id, mxc_url) VALUES ('860171525772279849', 'mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR'); From 56f959e9f3d84f8482293afacf2764fdaacd0e09 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Feb 2024 23:02:55 +1300 Subject: [PATCH 5/5] m->d: test: spoiler reasons --- m2d/converters/event-to-message.test.js | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index d8dd3fb..fb21098 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -234,6 +234,37 @@ test("event2message: spoilers work", async t => { ) }) +test("event2message: spoiler reasons work", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `zoe kills a 5 letter noun at the end` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "\\(cw crossword spoilers you'll never believe. don't tell anybody\\) ||zoe kills a 5 letter noun at the end||", + avatar_url: undefined + }] + } + ) +}) + test("event2message: markdown syntax is escaped", async t => { t.deepEqual( await eventToMessage({