m2d emoji roundtripping #110

Open
mschae wants to merge 3 commits from mschae/out-of-your-element:m2d-emote-roundtrip into main
Showing only changes of commit c802203391 - Show all commits

keep emote names when using end-of-message sprite sheets

also fix the bug where using the same emote in the middle and the end of
the message causes the one in the middle to get bridged as `<::>`
mini-bomba 2026-11-14 21:46:35 +01:00
No known key found for this signature in database
GPG key ID: 30F8B138B4794886

View file

@ -185,6 +185,8 @@ turndownService.addRule("table", {
/** @type {string[]} SPRITE SHEET EMOJIS FEATURE: mxc urls for the currently processing message */ /** @type {string[]} SPRITE SHEET EMOJIS FEATURE: mxc urls for the currently processing message */
let endOfMessageEmojis = [] let endOfMessageEmojis = []
/** @type {number} SPRITE SHEET EMOJIS FEATURE: number of emojis in the entire currently processing message */
let totalMessageEmojis = 0
turndownService.addRule("emoji", { turndownService.addRule("emoji", {
filter: function (node, options) { filter: function (node, options) {
if (node.nodeName !== "IMG" || !node.hasAttribute("data-mx-emoticon") || !node.getAttribute("src") || !node.getAttribute("title")) return false if (node.nodeName !== "IMG" || !node.hasAttribute("data-mx-emoticon") || !node.getAttribute("src") || !node.getAttribute("title")) return false
@ -192,9 +194,15 @@ turndownService.addRule("emoji", {
}, },
replacement: function (content, node) { replacement: function (content, node) {
// the previous approach of matching end of message emojis by mxc was inaccurate:
// if someone put the same emoji both in the middle and at the end, the reference in the middle would break
// so instead, we abuse the fact that turndown appears to iterate over the elements from left to right
// every time this method is called, count down the total
// when EOME count > total, the current emoji is an end of message emoji
totalMessageEmojis -= 1
const mxcUrl = node.getAttribute("src") const mxcUrl = node.getAttribute("src")
const guessedName = node.getAttribute("title").replace(/^:|:$/g, "") const guessedName = node.getAttribute("title")
return convertEmoji(mxcUrl, guessedName, true, true) return convertEmoji(mxcUrl, guessedName, endOfMessageEmojis.length > totalMessageEmojis, true)
} }
}) })
@ -231,11 +239,12 @@ function getCodeContent(preCode) {
/** /**
* @param {string | null} mxcUrl * @param {string | null} mxcUrl
* @param {string | null} nameForGuess without colons * @param {string | null} nameForGuess without colons
* @param {boolean} allowSpriteSheetIndicator * @param {boolean} isEndOfMessageEmoji
* @param {boolean} allowLink * @param {boolean} allowLink
* @returns {string} discord markdown that represents the custom emoji in some form * @returns {string} discord markdown that represents the custom emoji in some form
*/ */
function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink) { function convertEmoji(mxcUrl, nameForGuess, isEndOfMessageEmoji, allowLink) {
nameForGuess = nameForGuess?.replace(/^:|:$/g, "") ?? null
// Get the known emoji from the database. // Get the known emoji from the database.
if (mxcUrl) var row = select("emoji", ["emoji_id", "name", "animated"], {mxc_url: mxcUrl}).get() if (mxcUrl) var row = select("emoji", ["emoji_id", "name", "animated"], {mxc_url: mxcUrl}).get()
// Now we have to search all servers to see if we're able to send this emoji. // Now we have to search all servers to see if we're able to send this emoji.
@ -266,9 +275,10 @@ function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink
// We know an emoji, and we can use it // We know an emoji, and we can use it
const animatedChar = row.animated ? "a" : "" const animatedChar = row.animated ? "a" : ""
return `<${animatedChar}:${row.name}:${row.emoji_id}>` return `<${animatedChar}:${row.name}:${row.emoji_id}>`
} else if (allowSpriteSheetIndicator && mxcUrl && endOfMessageEmojis.includes(mxcUrl)) { } else if (isEndOfMessageEmoji) {
// We can't locate or use a suitable emoji. After control returns, it will rewind over this, delete this section, and upload the emojis as a sprite sheet. // We can't locate or use a suitable emoji. After control returns, it will rewind over this, delete this section, and upload the emojis as a sprite sheet.
return `<::>` // but still include the name of the emote, so that we can display it
return `<!:${nameForGuess}:>`
} else if (allowLink && mxcUrl && nameForGuess) { } else if (allowLink && mxcUrl && nameForGuess) {
// We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle. // We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle.
return `[:${nameForGuess}:](${mxUtils.getPublicUrlForMxc(mxcUrl)})` return `[:${nameForGuess}:](${mxUtils.getPublicUrlForMxc(mxcUrl)})`
@ -349,17 +359,20 @@ function getUserOrProxyOwnerMention(mxid) {
* @returns {string} new content with emoji sheet link * @returns {string} new content with emoji sheet link
*/ */
function linkEndOfMessageSpriteSheet(content) { function linkEndOfMessageSpriteSheet(content) {
if (!content.includes("<::>")) return content // No unknown emojis, nothing to do if (!/<!:[a-zA-Z0-9_]*:>/.test(content)) return content // No unknown emojis, nothing to do
// Remove known and unknown emojis from the end of the message // Remove known and unknown emojis from the end of the message
const r = /<a?:[a-zA-Z0-9_]*:[0-9]*>\s*$/ const r = /<[a!]?(:[a-zA-Z0-9_]*:)[0-9]*>\s*$/
const emoteNames = []
while (content.match(r)) { let match
while (match = content.match(r)) {
emoteNames.unshift(match[1])
content = content.replace(r, "") content = content.replace(r, "")
} }
// Use a markdown link to hide the URL. If this is the only thing in the message, Discord will hide it entirely, same as lone URLs. Good for us. // Use a markdown link to hide the URL. If this is the only thing in the message, Discord will hide it entirely, same as lone URLs. Good for us.
content = content.trimEnd() content = content.trimEnd()
content += " [\u2800](" // U+2800 Braille Pattern Blank is invisible on all known platforms but is digitally not a whitespace character content += ` [${emoteNames.join(" ")}](`
const afterLink = ")" const afterLink = ")"
// Make emojis URL params // Make emojis URL params
@ -948,6 +961,9 @@ async function eventToMessage(event, guild, channel, di) {
throw new Error(`Stuck iterating through nodes. Current node: ${node.innerHTML}`) throw new Error(`Stuck iterating through nodes. Current node: ${node.innerHTML}`)
} }
// SPRITE SHEET EMOJIS FEATURE: count any emojis we encounter
if (node.nodeType === 1 && node.nodeName === "IMG" && node.hasAttribute("data-mx-emoticon") && node.getAttribute("src") && node.getAttribute("title")) totalMessageEmojis += 1
// Don't process mx-reply further // Don't process mx-reply further
if (node.nodeType === 1 && node.tagName === "MX-REPLY") { if (node.nodeType === 1 && node.tagName === "MX-REPLY") {
const toRemove = node const toRemove = node
@ -1044,12 +1060,14 @@ async function eventToMessage(event, guild, channel, di) {
node = node.nextSibling node = node.nextSibling
} }
} }
// SPRITE SHEET EMOJIS FEATURE: reset emoji counters, then forEachNode will update totalMessageEmojis
totalMessageEmojis = 0
endOfMessageEmojis = []
await forEachNode(event, root) await forEachNode(event, root)
if (perMessageProfile?.has_fallback) root.querySelectorAll("[data-mx-profile-fallback]").forEach(x => x.remove()) if (perMessageProfile?.has_fallback) root.querySelectorAll("[data-mx-profile-fallback]").forEach(x => x.remove())
// SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet. // SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet.
// First we need to determine which emojis are at the end. // First we need to determine which emojis are at the end.
endOfMessageEmojis = []
let match let match
let last = input.length let last = input.length
while ((match = input.slice(0, last).match(/<img [^<>]*>\s*$/))) { while ((match = input.slice(0, last).match(/<img [^<>]*>\s*$/))) {