From c802203391638d7024fb497b4c0e8b5e1b69ad65 Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:46:35 +0100 Subject: [PATCH 1/3] keep emote names when using end-of-message sprite sheets also fix the bug where using the same emote in the middle and the end of the message causes the one in the middle to get bridged as `<::>` --- src/m2d/converters/event-to-message.js | 40 +++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index ddad94d..3e097ef 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -185,6 +185,8 @@ turndownService.addRule("table", { /** @type {string[]} SPRITE SHEET EMOJIS FEATURE: mxc urls for the currently processing message */ let endOfMessageEmojis = [] +/** @type {number} SPRITE SHEET EMOJIS FEATURE: number of emojis in the entire currently processing message */ +let totalMessageEmojis = 0 turndownService.addRule("emoji", { filter: function (node, options) { if (node.nodeName !== "IMG" || !node.hasAttribute("data-mx-emoticon") || !node.getAttribute("src") || !node.getAttribute("title")) return false @@ -192,9 +194,15 @@ turndownService.addRule("emoji", { }, replacement: function (content, node) { + // the previous approach of matching end of message emojis by mxc was inaccurate: + // if someone put the same emoji both in the middle and at the end, the reference in the middle would break + // so instead, we abuse the fact that turndown appears to iterate over the elements from left to right + // every time this method is called, count down the total + // when EOME count > total, the current emoji is an end of message emoji + totalMessageEmojis -= 1 const mxcUrl = node.getAttribute("src") - const guessedName = node.getAttribute("title").replace(/^:|:$/g, "") - return convertEmoji(mxcUrl, guessedName, true, true) + const guessedName = node.getAttribute("title") + return convertEmoji(mxcUrl, guessedName, endOfMessageEmojis.length > totalMessageEmojis, true) } }) @@ -231,11 +239,12 @@ function getCodeContent(preCode) { /** * @param {string | null} mxcUrl * @param {string | null} nameForGuess without colons - * @param {boolean} allowSpriteSheetIndicator + * @param {boolean} isEndOfMessageEmoji * @param {boolean} allowLink * @returns {string} discord markdown that represents the custom emoji in some form */ -function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink) { +function convertEmoji(mxcUrl, nameForGuess, isEndOfMessageEmoji, allowLink) { + nameForGuess = nameForGuess?.replace(/^:|:$/g, "") ?? null // Get the known emoji from the database. if (mxcUrl) var row = select("emoji", ["emoji_id", "name", "animated"], {mxc_url: mxcUrl}).get() // Now we have to search all servers to see if we're able to send this emoji. @@ -266,9 +275,10 @@ function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink // We know an emoji, and we can use it const animatedChar = row.animated ? "a" : "" return `<${animatedChar}:${row.name}:${row.emoji_id}>` - } else if (allowSpriteSheetIndicator && mxcUrl && endOfMessageEmojis.includes(mxcUrl)) { + } else if (isEndOfMessageEmoji) { // We can't locate or use a suitable emoji. After control returns, it will rewind over this, delete this section, and upload the emojis as a sprite sheet. - return `<::>` + // but still include the name of the emote, so that we can display it + return `` } else if (allowLink && mxcUrl && nameForGuess) { // We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle. return `[:${nameForGuess}:](${mxUtils.getPublicUrlForMxc(mxcUrl)})` @@ -349,17 +359,20 @@ function getUserOrProxyOwnerMention(mxid) { * @returns {string} new content with emoji sheet link */ function linkEndOfMessageSpriteSheet(content) { - if (!content.includes("<::>")) return content // No unknown emojis, nothing to do + if (!//.test(content)) return content // No unknown emojis, nothing to do // Remove known and unknown emojis from the end of the message - const r = /\s*$/ + const r = /<[a!]?(:[a-zA-Z0-9_]*:)[0-9]*>\s*$/ + const emoteNames = [] - while (content.match(r)) { + let match + while (match = content.match(r)) { + emoteNames.unshift(match[1]) content = content.replace(r, "") } // Use a markdown link to hide the URL. If this is the only thing in the message, Discord will hide it entirely, same as lone URLs. Good for us. content = content.trimEnd() - content += " [\u2800](" // U+2800 Braille Pattern Blank is invisible on all known platforms but is digitally not a whitespace character + content += ` [${emoteNames.join(" ")}](` const afterLink = ")" // Make emojis URL params @@ -948,6 +961,9 @@ async function eventToMessage(event, guild, channel, di) { throw new Error(`Stuck iterating through nodes. Current node: ${node.innerHTML}`) } + // SPRITE SHEET EMOJIS FEATURE: count any emojis we encounter + if (node.nodeType === 1 && node.nodeName === "IMG" && node.hasAttribute("data-mx-emoticon") && node.getAttribute("src") && node.getAttribute("title")) totalMessageEmojis += 1 + // Don't process mx-reply further if (node.nodeType === 1 && node.tagName === "MX-REPLY") { const toRemove = node @@ -1044,12 +1060,14 @@ async function eventToMessage(event, guild, channel, di) { node = node.nextSibling } } + // SPRITE SHEET EMOJIS FEATURE: reset emoji counters, then forEachNode will update totalMessageEmojis + totalMessageEmojis = 0 + endOfMessageEmojis = [] await forEachNode(event, root) if (perMessageProfile?.has_fallback) root.querySelectorAll("[data-mx-profile-fallback]").forEach(x => x.remove()) // SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet. // First we need to determine which emojis are at the end. - endOfMessageEmojis = [] let match let last = input.length while ((match = input.slice(0, last).match(/]*>\s*$/))) { -- 2.34.1 From 519aedebe698e67c49b40ca55dda857f474d208d Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:00:16 +0100 Subject: [PATCH 2/3] implement roundtripping for m->d emotes --- src/d2m/converters/message-to-event.js | 54 +++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 9376154..35096d3 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -136,6 +136,58 @@ const embedTitleParser = markdown.markdownEngine.parserFor({ autolink: undefined, link: undefined }) +const contentHtmlOutput = markdown.markdownEngine.htmlFor(markdown.markdownEngine.ruleOutput({ + ...markdown.rules, + // special override for the link node that converts any m->d bridged emojis back to proper matrix emotes on the way back + link: Object.assign({}, markdown.markdownEngine.defaultRules.link, { + html: function(node, output, state) { + // first, check if this looks like a m->d bridged emoji + if ( + !node.target.startsWith(`${reg.ooye.bridge_origin}/download/`) + || node.content.length != 1 + || node.content[0].type !== "text" + || !/^:[a-zA-Z0-9_: ]*:$/.test(node.content[0].content) + ) { + // not a m->d bridged emote, run the standard link output code + return markdown.markdownEngine.defaultRules.link.html(node, output, state) + } + const singleEmoteUrlPrefix = `${reg.ooye.bridge_origin}/download/matrix/` + if (node.target.startsWith(`${reg.ooye.bridge_origin}/download/sheet?`)) { + let parsedUrl + try { + parsedUrl = new URL(node.target) + } catch { + // failed to parse url, fall back + return markdown.markdownEngine.defaultRules.link.html(node, output, state) + } + if (!parsedUrl.searchParams.has("e")) { + // no emotes, fall back + return markdown.markdownEngine.defaultRules.link.html(node, output, state) + } + const emojiNames = Array.from(node.content[0].content.matchAll(/:([^\s:]*):/g).map((m) => m[1])) + const htmlEmojis = [] + for (const mxc of parsedUrl.searchParams.getAll("e")) { + const name = emojiNames.shift() + htmlEmojis.push( + name == null || name.length === 0 + ? tag`` + : tag`${name}` + ) + } + return htmlEmojis.join(" ") + // stricter link label regex for single emotes - no spaces/colons allowed + } else if (node.target.startsWith(singleEmoteUrlPrefix) && /^:[a-zA-Z0-9_]*:$/.test(node.content[0].content)) { + const name = node.content[0].content.slice(1, -1) + return name.length === 0 + ? tag`` + : tag`${name}` + } else { + // maybe this is not an emote after all lol + return markdown.markdownEngine.defaultRules.link.html(node, output, state) + } + } + }) +}, 'html')) /** * @param {{room?: boolean, user_ids?: string[]}} mentions @@ -894,7 +946,7 @@ async function messageToEvent(message, guild, options = {}, di) { return `<:${name}:${id}>` }) - const {body, html} = await transformContent(content, {isTheMessageContent: true}) + const {body, html} = await transformContent(content, {isTheMessageContent: true}, markdown.parser, contentHtmlOutput) await addTextEvent(body, html, msgtype) } -- 2.34.1 From 7aa1f5870c5add034b64a2a689eb907db6b2bfac Mon Sep 17 00:00:00 2001 From: mini-bomba <55105495+mini-bomba@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:59:00 +0100 Subject: [PATCH 3/3] sprite sheet emote roundtripping without visible names --- src/d2m/converters/message-to-event.js | 8 +++----- src/m2d/converters/event-to-message.js | 16 +++++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 35096d3..ab26c77 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -143,16 +143,14 @@ const contentHtmlOutput = markdown.markdownEngine.htmlFor(markdown.markdownEngin html: function(node, output, state) { // first, check if this looks like a m->d bridged emoji if ( - !node.target.startsWith(`${reg.ooye.bridge_origin}/download/`) - || node.content.length != 1 + node.content.length != 1 || node.content[0].type !== "text" - || !/^:[a-zA-Z0-9_: ]*:$/.test(node.content[0].content) ) { // not a m->d bridged emote, run the standard link output code return markdown.markdownEngine.defaultRules.link.html(node, output, state) } const singleEmoteUrlPrefix = `${reg.ooye.bridge_origin}/download/matrix/` - if (node.target.startsWith(`${reg.ooye.bridge_origin}/download/sheet?`)) { + if (node.target.startsWith(`${reg.ooye.bridge_origin}/download/sheet?`) && node.content[0].content === "\u2800") { let parsedUrl try { parsedUrl = new URL(node.target) @@ -164,7 +162,7 @@ const contentHtmlOutput = markdown.markdownEngine.htmlFor(markdown.markdownEngin // no emotes, fall back return markdown.markdownEngine.defaultRules.link.html(node, output, state) } - const emojiNames = Array.from(node.content[0].content.matchAll(/:([^\s:]*):/g).map((m) => m[1])) + const emojiNames = parsedUrl.searchParams.getAll("n") const htmlEmojis = [] for (const mxc of parsedUrl.searchParams.getAll("e")) { const name = emojiNames.shift() diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 3e097ef..b8f7043 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -361,18 +361,19 @@ function getUserOrProxyOwnerMention(mxid) { function linkEndOfMessageSpriteSheet(content) { if (!//.test(content)) return content // No unknown emojis, nothing to do // Remove known and unknown emojis from the end of the message - const r = /<[a!]?(:[a-zA-Z0-9_]*:)[0-9]*>\s*$/ - const emoteNames = [] + const r = /<[a!]?:([a-zA-Z0-9_]*):[0-9]*>\s*$/ + const emojiNames = [] let match while (match = content.match(r)) { - emoteNames.unshift(match[1]) + emojiNames.unshift(match[1]) content = content.replace(r, "") } // Use a markdown link to hide the URL. If this is the only thing in the message, Discord will hide it entirely, same as lone URLs. Good for us. content = content.trimEnd() - content += ` [${emoteNames.join(" ")}](` + content += " [\u2800](" // U+2800 Braille Pattern Blank is invisible on all known platforms but is digitally not a whitespace character + content += `${reg.ooye.bridge_origin}/download/sheet?` const afterLink = ")" // Make emojis URL params @@ -381,15 +382,16 @@ function linkEndOfMessageSpriteSheet(content) { // We can do up to 2000 chars max. (In this maximal case it will get chunked to a separate message.) Ignore additional emojis. const withoutMxc = mxUtils.makeMxcPublic(mxc) assert(withoutMxc) - const emojisLength = params.toString().length + encodeURIComponent(withoutMxc).length + 2 + const emojiName = emojiNames.shift() ?? "" + const emojisLength = params.toString().length + encodeURIComponent(withoutMxc).length + encodeURIComponent(emojiName).length + 4 if (content.length + emojisLength + afterLink.length > 2000) { break } params.append("e", withoutMxc) + params.append("n", emojiName) } - const url = `${reg.ooye.bridge_origin}/download/sheet?${params.toString()}` - return content + url + afterLink + return content + params.toString() + afterLink } /** -- 2.34.1