diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js index 62e0fb6..9b5527f 100644 --- a/d2m/actions/register-user.js +++ b/d2m/actions/register-user.js @@ -97,7 +97,7 @@ async function ensureSimJoined(user, roomID) { */ async function memberToStateContent(user, member, guildID) { let displayname = user.username - // if (member.nick && member.nick !== displayname) displayname = member.nick + " | " + displayname // prepend nick if present + if (user.global_name) displayname = user.global_name if (member.nick) displayname = member.nick const content = { diff --git a/d2m/actions/register-user.test.js b/d2m/actions/register-user.test.js index 5cee80b..96c73aa 100644 --- a/d2m/actions/register-user.test.js +++ b/d2m/actions/register-user.test.js @@ -22,6 +22,26 @@ test("member2state: without member nick or avatar", async t => { ) }) +test("member2state: with global name, without member nick or avatar", async t => { + t.deepEqual( + await _memberToStateContent(testData.member.papiophidian.user, testData.member.papiophidian, testData.guild.general.id), + { + avatar_url: "mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX", + displayname: "PapiOphidian", + membership: "join", + "moe.cadence.ooye.member": { + avatar: "/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024" + }, + "uk.half-shot.discord.member": { + bot: false, + displayColor: 1579292, + id: "320067006521147393", + username: "@papiophidian" + } + } + ) +}) + test("member2state: with member nick and avatar", async t => { t.deepEqual( await _memberToStateContent(testData.member.sheep.user, testData.member.sheep, testData.guild.general.id), diff --git a/d2m/converters/edit-to-changes.js b/d2m/converters/edit-to-changes.js index 9d1fc0b..dc42708 100644 --- a/d2m/converters/edit-to-changes.js +++ b/d2m/converters/edit-to-changes.js @@ -102,7 +102,7 @@ async function editToChanges(message, guild, api) { // So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed. eventsToReplace = eventsToReplace.filter(ev => { // Discord does not allow files, images, attachments, or videos to be edited. - if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote") { + if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") { return false } // Discord does not allow stickers to be edited. diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index fcf0bf8..e7bbda5 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -5,6 +5,7 @@ const DiscordTypes = require("discord-api-types/v10") const chunk = require("chunk-text") const TurndownService = require("turndown") const assert = require("assert").strict +const entities = require("entities") const passthrough = require("../../passthrough") const {sync, db, discord, select, from} = passthrough @@ -324,11 +325,12 @@ async function eventToMessage(event, guild, di) { replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} ` } const sender = repliedToEvent.sender - const senderName = sender.match(/@([^:]*)/)?.[1] || sender const authorID = select("sim", "user_id", {mxid: repliedToEvent.sender}).pluck().get() if (authorID) { replyLine += `<@${authorID}>` } else { + let senderName = select("member_cache", "displayname", {mxid: repliedToEvent.sender}).pluck().get() + if (!senderName) senderName = sender.match(/@([^:]*)/)?.[1] || sender replyLine += `Ⓜ️**${senderName}**` } // If the event has been edited, the homeserver will include the relation in `unsigned`. @@ -348,11 +350,13 @@ async function eventToMessage(event, guild, di) { } else { const repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body const contentPreviewChunks = chunk( - repliedToContent.replace(/.*<\/mx-reply>/, "") // Remove everything before replies, so just use the actual message body - .replace(/
.*?<\/blockquote>/, "") // If the message starts with a blockquote, don't count it and use the message body afterwards - .replace(/(?:\n|
)+/g, " ") // Should all be on one line - .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.) - .replace(/<[^>]+>/g, ""), 50) // Completely strip all other formatting. + entities.decodeHTML5Strict( // Remove entities like & " + repliedToContent.replace(/.*<\/mx-reply>/, "") // Remove everything before replies, so just use the actual message body + .replace(/
.*?<\/blockquote>/, "") // If the message starts with a blockquote, don't count it and use the message body afterwards + .replace(/(?:\n|
)+/g, " ") // Should all be on one line + .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.) + .replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting. + ), 50) contentPreview = ":\n> " contentPreview += contentPreviewChunks.length > 1 ? contentPreviewChunks[0] + "..." : contentPreviewChunks[0] } diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 87a4b12..1a1c3f0 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -813,6 +813,55 @@ test("event2message: should include a reply preview when message ends with a blo ) }) +test("event2message: entities are not escaped in main message or reply preview", async t => { + // Intended result: Testing? in italics, followed by the sequence "':.`[]&things + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "> <@cadence:cadence.moe> _Testing?_ \"':.`[]&things\n\n_Testing?_ \"':.`[]&things", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @cadence:cadence.moe
Testing? \"':.`[]&things
Testing? "':.`[]&things", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$yIWjZPi6Xk56fBxJwqV4ANs_hYLjnWI2cNKbZ2zwk60" + } + } + }, + event_id: "$2I7odT9okTdpwDcqOjkJb_A3utdO4V8Cp3LK6-Rvwcs", + room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$yIWjZPi6Xk56fBxJwqV4ANs_hYLjnWI2cNKbZ2zwk60", { + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + "msgtype": "m.text", + "body": "_Testing?_ \"':.`[]&things", + "format": "org.matrix.custom.html", + "formatted_body": "Testing? "':.`[]&things" + }, + event_id: "$yIWjZPi6Xk56fBxJwqV4ANs_hYLjnWI2cNKbZ2zwk60", + room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" + }) + } + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>Ⓜ️**cadence [they]**:" + + "\n> Testing? \"':.`[]&things" + + "\n_Testing?_ \"':.\\`\\[\\]&things", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" + }] + } + ) +}) + test("event2message: editing a rich reply to a sim user", async t => { const eventsFetched = [] t.deepEqual( @@ -1151,7 +1200,7 @@ test("event2message: rich reply to a matrix user's long message with formatting" messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**:" + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence [they]**:" + "\n> i should have a little happy test list bold em..." + "\n**no you can't!!!**", avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" @@ -1312,7 +1361,7 @@ test("event2message: with layered rich replies, the preview should only be the r messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**:" + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence [they]**:" + "\n> two" + "\nthree", avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" diff --git a/package-lock.json b/package-lock.json index d1ed09e..3847ee1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "chunk-text": "^2.0.1", "cloudstorm": "^0.8.0", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", "js-yaml": "^4.1.0", @@ -1096,6 +1097,17 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-get-iterator": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", diff --git a/package.json b/package.json index e8eec83..6a9deea 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "chunk-text": "^2.0.1", "cloudstorm": "^0.8.0", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", "js-yaml": "^4.1.0", diff --git a/readme.md b/readme.md index 77239f7..5c97745 100644 --- a/readme.md +++ b/readme.md @@ -164,6 +164,7 @@ Follow these steps: * (1) discord-markdown: This is my fork! I make sure it does what I want. * (0) giframe: This is my fork! It should do what I want. * (1) heatsync: Module hot-reloader that I trust. +* (0) entities: Looks fine. No dependencies. * (1) js-yaml: It seems to do what I want, and it's already pulled in by matrix-appservice. * (70) matrix-appservice: I wish it didn't pull in express :( * (0) minimist: It's already pulled in by better-sqlite3->prebuild-install diff --git a/test/data.js b/test/data.js index ea9f66a..1a526aa 100644 --- a/test/data.js +++ b/test/data.js @@ -222,6 +222,40 @@ module.exports = { }, mute: false, deaf: false + }, + papiophidian: { + avatar: null, + communication_disabled_until: null, + flags: 0, + joined_at: "2018-08-05T09:40:47.076000+00:00", + nick: null, + pending: false, + premium_since: "2021-09-30T18:58:44.996000+00:00", + roles: [ + "475599410068324352", + "475599471049310208", + "497586624390234112", + "613685290938138625", + "475603310955593729", + "1151970058730487898", + "1151970058730487901" + ], + unusual_dm_activity_until: null, + user: { + id: "320067006521147393", + username: "papiophidian", + avatar: "5fc4ad85c1ea876709e9a7d3374a78a1", + discriminator: "0", + public_flags: 4194880, + flags: 4194880, + banner: "a_6f311cf6a3851a98e2fa0335af85b1d1", + accent_color: 1579292, + global_name: "PapiOphidian", + avatar_decoration_data: null, + banner_color: "#18191c" + }, + mute: false, + deaf: false } }, pins: { diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index f5cfb5c..f8039cc 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -75,7 +75,8 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/emojis/230201364309868544.png', 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), ('https://cdn.discordapp.com/emojis/393635038903926784.gif', 'mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc'), ('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'), -('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'); +('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'), +('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX'); INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'),