Fix more edge-case embed formatting

d2m/converters/message-to-event.embeds.test.js

@@ -35,14 +35,14 @@ test("message2event embeds: nothing but a field", async t => {
 		$type: "m.room.message",
 		"m.mentions": {},
 		msgtype: "m.notice",
-		body: "> **Amanda 🎵#2192 :online:"
+		body: "> ### Amanda 🎵#2192 :online:"
-			+ "\n> willow tree, branch 0**"
+			+ "\n> willow tree, branch 0"
 			+ "\n> **❯ Uptime:**\n> 3m 55s\n> **❯ Memory:**\n> 64.45MB",
 		format: "org.matrix.custom.html",
-		formatted_body: '<blockquote><strong>Amanda 🎵#2192 <img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ\" title=\":online:\" alt=\":online:\">'
+		formatted_body: '<blockquote><p><strong>Amanda 🎵#2192 <img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ\" title=\":online:\" alt=\":online:\">'
 			+ '<br>willow tree, branch 0</strong>'
 			+ '<br><strong>❯ Uptime:</strong><br>3m 55s'
-			+ '<br><strong>❯ Memory:</strong><br>64.45MB</blockquote>'
+			+ '<br><strong>❯ Memory:</strong><br>64.45MB</p></blockquote>'
 	}])
 })
 
@@ -52,19 +52,19 @@ test("message2event embeds: reply with just an embed", async t => {
 		$type: "m.room.message",
 		msgtype: "m.notice",
 		"m.mentions": {},
-		body: "> [**⏺️ dynastic (@dynastic)**](https://twitter.com/i/user/719631291747078145)"
+		body: "> ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
-			+ "\n> \n> **https://twitter.com/i/status/1707484191963648161**"
+			+ "\n> \n> ## https://twitter.com/i/status/1707484191963648161"
 			+ "\n> \n> does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
-			+ "\n> \n> **Retweets**"
+			+ "\n> \n> ### Retweets"
 			+ "\n> 119"
-			+ "\n> \n> **Likes**"
+			+ "\n> \n> ### Likes"
 			+ "\n> 5581"
-			+ "\n> \n> — Twitter",
+			+ "\n> — Twitter",
 		format: "org.matrix.custom.html",
-		formatted_body: '<blockquote><a href="https://twitter.com/i/user/719631291747078145"><strong>⏺️ dynastic (@dynastic)</strong></a>'
+		formatted_body: '<blockquote><p><strong><a href="https://twitter.com/i/user/719631291747078145">⏺️ dynastic (@dynastic)</a></strong></p>'
-			+ '<br><br><strong><a href="https://twitter.com/i/status/1707484191963648161">https://twitter.com/i/status/1707484191963648161</a></strong>'
+			+ '<p><strong><a href="https://twitter.com/i/status/1707484191963648161">https://twitter.com/i/status/1707484191963648161</a></strong>'
-			+ '<br><br>does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
+			+ '</p><p>does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
-			+ '<br><br><strong>Retweets</strong><br>119<br><br><strong>Likes</strong><br>5581<br><br>— Twitter</blockquote>'
+			+ '</p><p><strong>Retweets</strong><br>119</p><p><strong>Likes</strong><br>5581</p>— Twitter</blockquote>'
 	}])
 })
 
@@ -99,3 +99,45 @@ test("message2event embeds: image embed and attachment", async t => {
 		"m.mentions": {}
 	}])
 })
+
+test("message2event embeds: blockquote in embed", async t => {
+	const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.text",
+		body: ":emoji: **4 |** #wonderland",
+		format: "org.matrix.custom.html",
+		formatted_body: `<img data-mx-emoticon height=\"32\" src=\"mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO\" title=\":emoji:\" alt=\":emoji:\"> <strong>4 |</strong> <a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe\">#wonderland</a>`,
+		"m.mentions": {}
+	}, {
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo)",
+		format: "org.matrix.custom.html",
+		formatted_body: "<blockquote><p><strong><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\">⏺️ minimus</a></strong></p><p>reply draft<br><blockquote>The following is a message composed via consensus of the Stinker Council.<br><br>For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.<br><br>Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.<br><br>There will be no further communication.</blockquote></p><p><a href=\"https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\">Go to Message</a></p></blockquote>",
+		"m.mentions": {}
+	}])
+})
+
+test("message2event embeds: crazy html is all escaped", async t => {
+	const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		msgtype: "m.notice",
+		body: "> ## ⏺️ <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n> \n> ## <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;) https://a.co/&amp;<script>"
+			+ "\n> \n> <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n> \n> ### <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n> <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+			+ "\n> — <strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+		format: "org.matrix.custom.html",
+		formatted_body: `<blockquote>`
+			+ `<p><strong><a href="https://a.co/&amp;amp;&lt;script&gt;">⏺️ &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</a></strong></p>`
+			+ `<p><strong><a href=\"https://a.co/&amp;amp;&lt;script&gt;">&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href="https://a.co/&amp;amp">https://a.co/&amp;amp</a>;)</a></strong></p>`
+			+ `<p>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
+			+ `<p><strong>&lt;strong&gt;[&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;](<a href=\"https://a.co/&amp;amp\">https://a.co/&amp;amp</a>;)</strong>`
+			+ `<br>&lt;strong&gt;<a href="https://a.co/&amp;amp;">&lt;span data-mx-color='#123456'&gt;Hey&lt;script&gt;</a></p>`
+			+ `— &lt;strong&gt;[&lt;span data-mx-color=&#39;#123456&#39;&gt;Hey&lt;script&gt;](https://a.co/&amp;amp;)</blockquote>`,
+		"m.mentions": {}
+	}])
+})

d2m/converters/message-to-event.js

@@ -4,6 +4,7 @@ const assert = require("assert").strict
 const markdown = require("discord-markdown")
 const pb = require("prettier-bytes")
 const DiscordTypes = require("discord-api-types/v10")
+const {tag} = require("html-template-tag")
 
 const passthrough = require("../../passthrough")
 const {sync, db, discord, select, from} = passthrough
@@ -13,6 +14,8 @@ const file = sync.require("../../matrix/file")
 const emojiToKey = sync.require("./emoji-to-key")
 /** @type {import("./lottie")} */
 const lottie = sync.require("./lottie")
+/** @type {import("../../m2d/converters/utils")} */
+const mxUtils = sync.require("../../m2d/converters/utils")
 const reg = require("../../matrix/read-registration")
 
 const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
@@ -77,6 +80,12 @@ function getDiscordParseCallbacks(message, guild, useHTML) {
 	}
 }
 
+const embedTitleParser = markdown.markdownEngine.parserFor({
+	...markdown.rules,
+	autolink: undefined,
+	link: undefined
+})
+
 /**
  * @param {import("discord-api-types/v10").APIMessage} message
  * @param {import("discord-api-types/v10").APIGuild} guild
@@ -154,8 +163,12 @@ async function messageToEvent(message, guild, options = {}, di) {
 		addMention(repliedToEventSenderMxid)
 	}
 
-	async function addTextEvent(content, msgtype, {scanMentions}) {
+	/**
-		content = content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => {
+	 * Translate Discord message links to Matrix event links.
+	 * @param {string} content Partial or complete Discord message content
+	 */
+	function transformContentMessageLinks(content) {
+		return content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => {
 			const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get()
 			const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
 			if (eventID && roomID) {
@@ -164,6 +177,17 @@ async function messageToEvent(message, guild, options = {}, di) {
 				return `${whole} [event not found]`
 			}
 		})
+	}
+
+	/**
+	 * Translate links and emojis and mentions and stuff. Give back the text and HTML so they can be combined into bigger events.
+	 * @param {string} content Partial or complete Discord message content
+	 * @param {any} customOptions
+	 * @param {any} customParser
+	 * @param {any} customHtmlOutput
+	 */
+	async function transformContent(content, customOptions = {}, customParser = null, customHtmlOutput = null) {
+		content = transformContentMessageLinks(content)
 
 		// Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter.
 		// So we scan the message ahead of time for all its emojis and ensure they are in the DB.
@@ -171,39 +195,26 @@ async function messageToEvent(message, guild, options = {}, di) {
 		await Promise.all(emojiMatches.map(match => {
 			const id = match[3]
 			const name = match[2]
-			const animated = match[1]
+			const animated = !!match[1]
 			return emojiToKey.emojiToKey({id, name, animated}) // Register the custom emoji if needed
 		}))
 
 		let html = markdown.toHTML(content, {
-			discordCallback: getDiscordParseCallbacks(message, guild, true)
+			discordCallback: getDiscordParseCallbacks(message, guild, true),
-		}, null, null)
+			...customOptions
+		}, customParser, customHtmlOutput)
 
 		let body = markdown.toHTML(content, {
 			discordCallback: getDiscordParseCallbacks(message, guild, false),
 			discordOnly: true,
 			escapeHTML: false,
+			...customOptions
 		}, null, null)
 
-		// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
+		return {body, html}
-		if (scanMentions) {
-			const matches = [...content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
-			if (matches.length && matches.some(m => m[1].match(/[a-z]/i))) {
-				const writtenMentionsText = matches.map(m => m[1].toLowerCase())
-				const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
-				assert(roomID)
-				const {joined} = await di.api.getJoinedMembers(roomID)
-				for (const [mxid, member] of Object.entries(joined)) {
-					if (!userRegex.some(rx => mxid.match(rx))) {
-						const localpart = mxid.match(/@([^:]*)/)
-						assert(localpart)
-						const displayName = member.display_name || localpart[1]
-						if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
-					}
-				}
-			}
 	}
 
+	async function addTextEvent(body, html, msgtype, {scanMentions}) {
 		// Star * prefix for fallback edits
 		if (options.includeEditFallbackStar) {
 			body = "* " + body
@@ -281,9 +292,27 @@ async function messageToEvent(message, guild, options = {}, di) {
 		message.content = "changed the channel name to **" + message.content + "**"
 	}
 
+	// Mentions scenario 3: scan the message content for written @mentions of matrix users. Allows for up to one space between @ and mention.
+	const matches = [...message.content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
+	if (matches.length && matches.some(m => m[1].match(/[a-z]/i))) {
+		const writtenMentionsText = matches.map(m => m[1].toLowerCase())
+		const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get()
+		assert(roomID)
+		const {joined} = await di.api.getJoinedMembers(roomID)
+		for (const [mxid, member] of Object.entries(joined)) {
+			if (!userRegex.some(rx => mxid.match(rx))) {
+				const localpart = mxid.match(/@([^:]*)/)
+				assert(localpart)
+				const displayName = member.display_name || localpart[1]
+				if (writtenMentionsText.includes(localpart[1].toLowerCase()) || writtenMentionsText.includes(displayName.toLowerCase())) addMention(mxid)
+			}
+		}
+	}
+
 	// Text content appears first
 	if (message.content) {
-		await addTextEvent(message.content, msgtype, {scanMentions: true})
+		const {body, html} = await transformContent(message.content)
+		await addTextEvent(body, html, msgtype, {scanMentions: true})
 	}
 
 	// Then attachments
@@ -303,7 +332,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 				msgtype: "m.text",
 				body: `${emoji} Uploaded SPOILER file: ${attachment.url} (${pb(attachment.size)})`,
 				format: "org.matrix.custom.html",
-				formatted_body: `<blockquote>${emoji} Uploaded SPOILER file: <span data-mx-spoiler><a href="${attachment.url}">View</a></span> (${pb(attachment.size)})</blockquote>`
+				formatted_body: `<blockquote>${emoji} Uploaded SPOILER file: <a href="${attachment.url}"><span data-mx-spoiler>${attachment.url}</span></a> (${pb(attachment.size)})</blockquote>`
 			}
 		}
 		// for large files, always link them instead of uploading so I don't use up all the space in the content repo
@@ -384,38 +413,60 @@ async function messageToEvent(message, guild, options = {}, di) {
 	// Then embeds
 	for (const embed of message.embeds || []) {
 		if (embed.type === "image") {
-			continue // Matrix already does a fine enough job of providing image embeds.
+			continue // Matrix's own image embeds are fine.
 		}
 
 		// Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once
-		let repParagraphs = []
+		const rep = new mxUtils.MatrixStringBuilder()
-		const makeUrlTitle = (text, url) =>
-			( text && url ? `[**${text}**](${url})`
-			: text ? `**${text}**`
-			: url ? `**${url}**`
-			: "")
 
+		// Author and URL into a paragraph
 		let authorNameText = embed.author?.name || ""
-		if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // not using the real image
+		if (authorNameText && embed.author?.icon_url) authorNameText = `⏺️ ${authorNameText}` // using the emoji instead of an image
-		let authorTitle = makeUrlTitle(authorNameText, embed.author?.url)
+		if (authorNameText || embed.author?.url) {
-		if (authorTitle) repParagraphs.push(authorTitle)
+			if (embed.author?.url) {
-
+				const authorURL = transformContentMessageLinks(embed.author.url)
-		let title = makeUrlTitle(embed.title, embed.url)
+				rep.addParagraph(`## ${authorNameText} ${authorURL}`, tag`<strong><a href="${authorURL}">${authorNameText}</a></strong>`)
-		if (title) repParagraphs.push(title)
+			} else {
-
+				rep.addParagraph(`## ${authorNameText}`, tag`<strong>${authorNameText}</strong>`)
-		if (embed.image?.url) repParagraphs.push(`📸 ${embed.image.url}`)
-		if (embed.video?.url) repParagraphs.push(`🎞️ ${embed.video.url}`)
-
-		if (embed.description) repParagraphs.push(embed.description)
-		for (const field of embed.fields || []) {
-			repParagraphs.push(`**${field.name}**\n${field.value}`)
 			}
-		if (embed.footer?.text) repParagraphs.push(`— ${embed.footer.text}`)
+		}
-		const repContent = repParagraphs.join("\n\n")
+
-		const repContentQuoted = repContent.split("\n").map(l => "> " + l).join("\n")
+		// Title and URL into a paragraph
+		if (embed.title) {
+			const {body, html} = await transformContent(embed.title, {}, embedTitleParser, markdown.htmlOutput)
+			if (embed.url) {
+				rep.addParagraph(`## ${body} ${embed.url}`, tag`<strong><a href="${embed.url}">$${html}</a></strong>`)
+			} else {
+				rep.addParagraph(`## ${body}`, `<strong>${html}</strong>`)
+			}
+		} else if (embed.url) {
+			rep.addParagraph(`## ${embed.url}`, tag`<strong><a href="${embed.url}">${embed.url}</a></strong>`)
+		}
+
+		if (embed.description) {
+			const {body, html} = await transformContent(embed.description)
+			rep.addParagraph(body, html)
+		}
+
+		for (const field of embed.fields || []) {
+			const name = field.name.match(/^[\s​­]*$/) ? {body: "", html: ""} : await transformContent(field.name, {}, embedTitleParser, markdown.htmlOutput)
+			const value = await transformContent(field.value)
+			const fieldRep = new mxUtils.MatrixStringBuilder()
+				.addLine(`### ${name.body}`, `<strong>${name.html}</strong>`, name.body)
+				.addLine(value.body, value.html, !!value.body)
+			rep.addParagraph(fieldRep.get().body, fieldRep.get().formatted_body)
+		}
+
+		if (embed.image?.url) rep.addParagraph(`📸 ${embed.image.url}`)
+		if (embed.video?.url) rep.addParagraph(`🎞️ ${embed.video.url}`)
+
+		if (embed.footer?.text) rep.addLine(`— ${embed.footer.text}`, tag`— ${embed.footer.text}`)
+		let {body, formatted_body: html} = rep.get()
+		body = body.split("\n").map(l => "> " + l).join("\n")
+		html = `<blockquote>${html}</blockquote>`
 
 		// Send as m.notice to apply the usual automated/subtle appearance, showing this wasn't actually typed by the person
-		await addTextEvent(repContentQuoted, "m.notice", {scanMentions: false})
+		await addTextEvent(body, html, "m.notice", {scanMentions: false})
 	}
 
 	// Then stickers

m2d/converters/emoji.js

@@ -40,6 +40,7 @@ function encodeEmoji(input, shortcode) {
 			"%E2%AD%90", // ⭐
 			"%F0%9F%90%88", // 🐈
 			"%E2%9D%93", // ❓
+			"%F0%9F%8F%86", // 🏆️
 		]
 
 		discordPreferredEncoding =

m2d/converters/event-to-message.js

@@ -110,6 +110,24 @@ turndownService.addRule("inlineLink", {
 	}
 })
 
+turndownService.addRule("listItem", {
+	filter: "li",
+	replacement: function (content, node, options) {
+		content = content
+		.replace(/^\n+/, "") // remove leading newlines
+		.replace(/\n+$/, "\n") // replace trailing newlines with just a single one
+		.replace(/\n/gm, "\n  ") // indent
+		var prefix = options.bulletListMarker + " "
+		var parent = node.parentNode
+		if (parent.nodeName === "OL") {
+			var start = parent.getAttribute("start")
+			var index = Array.prototype.indexOf.call(parent.children, node)
+			prefix = (start ? Number(start) + index : index + 1) + ".  "
+		}
+		return prefix + content + (node.nextSibling && !/\n$/.test(content) ? "\n" : "")
+	}
+})
+
 /** @type {string[]} SPRITE SHEET EMOJIS FEATURE: mxc urls for the currently processing message */
 let endOfMessageEmojis = []
 turndownService.addRule("emoji", {

m2d/converters/event-to-message.test.js

@@ -498,6 +498,47 @@ test("event2message: quotes have an appropriate amount of whitespace", async t =
 	)
 })
 
+test("event2message: lists have appropriate line breaks", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			content: {
+				body: 'i am not certain what you mean by "already exists with as discord". my goals are\n' +
+					'* bridgeing specific channels with existing matrix rooms\n' +
+					'  * optionally maybe entire "servers"\n' +
+					'* offering the bridge as a public service ',
+				format: 'org.matrix.custom.html',
+				formatted_body: '<p>i am not certain what you mean by "already exists with as discord". my goals are</p>\n' +
+					'<ul>\n' +
+					'<li>bridgeing specific channels with existing matrix rooms\n' +
+					'<ul>\n' +
+					'<li>optionally maybe entire "servers"</li>\n' +
+					'</ul>\n' +
+					'</li>\n' +
+					'<li>offering the bridge as a public service</li>\n' +
+					'</ul>\n',
+				'm.mentions': {},
+				msgtype: 'm.text'
+			},
+			room_id: '!cBxtVRxDlZvSVhJXVK:cadence.moe',
+			sender: '@Milan:tchncs.de',
+			type: 'm.room.message',
+		}, {}, {
+			api: {
+				getStateEvent: async () => ({displayname: "Milan"})
+			}
+		}),
+		{
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "Milan",
+				content: `i am not certain what you mean by "already exists with as discord". my goals are\n\n* bridgeing specific channels with existing matrix rooms\n  * optionally maybe entire "servers"\n* offering the bridge as a public service`,
+				avatar_url: undefined
+			}]
+		}
+	)
+})
+
 test("event2message: m.emote plaintext works", async t => {
 	t.deepEqual(
 		await eventToMessage({

m2d/converters/utils.js

@@ -16,6 +16,7 @@ const BLOCK_ELEMENTS = [
 	"NOSCRIPT", "OL", "OUTPUT", "P", "PRE", "SECTION", "SUMMARY", "TABLE", "TBODY", "TD",
 	"TFOOT", "TH", "THEAD", "TR", "UL"
 ]
+const NEWLINE_ELEMENTS = BLOCK_ELEMENTS.concat(["BR"])
 
 /**
  * Determine whether an event is the bridged representation of a discord message.
@@ -63,7 +64,71 @@ function getEventIDHash(eventID) {
 	return signedHash
 }
 
+class MatrixStringBuilder {
+	constructor() {
+		this.body = ""
+		this.formattedBody = ""
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} formattedBody
+	 * @param {any} [condition]
+	 */
+	add(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			this.body += body
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	addLine(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n"
+			this.body += body
+			const match = this.formattedBody.match(/<\/?([a-zA-Z]+[a-zA-Z0-9]*)[^>]*>\s*$/)
+			if (this.formattedBody.length && (!match || !NEWLINE_ELEMENTS.includes(match[1].toUpperCase()))) this.formattedBody += "<br>"
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	addParagraph(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n\n"
+			this.body += body
+			formattedBody = `<p>${formattedBody}</p>`
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	get() {
+		return {
+			msgtype: "m.text",
+			body: this.body,
+			format: "org.matrix.custom.html",
+			formatted_body: this.formattedBody
+		}
+	}
+}
+
 module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS
 module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord
 module.exports.getPublicUrlForMxc = getPublicUrlForMxc
 module.exports.getEventIDHash = getEventIDHash
+module.exports.MatrixStringBuilder = MatrixStringBuilder

matrix/matrix-command-handler.js

@@ -96,55 +96,6 @@ function replyctx(execute) {
 	}
 }
 
-const NEWLINE_ELEMENTS = mxUtils.BLOCK_ELEMENTS.concat(["BR"])
-
-class MatrixStringBuilder {
-	constructor() {
-		this.body = ""
-		this.formattedBody = ""
-	}
-
-	/**
-	 * @param {string} body
-	 * @param {string} formattedBody
-	 * @param {any} [condition]
-	 */
-	add(body, formattedBody, condition = true) {
-		if (condition) {
-			if (formattedBody == undefined) formattedBody = body
-			this.body += body
-			this.formattedBody += formattedBody
-		}
-		return this
-	}
-
-	/**
-	 * @param {string} body
-	 * @param {string} [formattedBody]
-	 * @param {any} [condition]
-	 */
-	addLine(body, formattedBody, condition = true) {
-		if (condition) {
-			if (formattedBody == undefined) formattedBody = body
-			if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n"
-			this.body += body
-			const match = this.formattedBody.match(/<\/?([a-zA-Z]+[a-zA-Z0-9]*)[^>]*>\s*$/)
-			if (this.formattedBody.length && (!match || !NEWLINE_ELEMENTS.includes(match[1].toUpperCase()))) this.formattedBody += "<br>"
-			this.formattedBody += formattedBody
-		}
-		return this
-	}
-
-	get() {
-		return {
-			msgtype: "m.text",
-			body: this.body,
-			format: "org.matrix.custom.html",
-			formatted_body: this.formattedBody
-		}
-	}
-}
-
 /** @type {Command[]} */
 const commands = [{
 	aliases: ["emoji"],
@@ -219,7 +170,7 @@ const commands = [{
 				})
 			}
 
-			const b = new MatrixStringBuilder()
+			const b = new mxUtils.MatrixStringBuilder()
 				.addLine("## Emoji preview", "<h2>Emoji preview</h2>")
 				.addLine(`Ⓜ️ This room isn't bridged to Discord. ${matrixOnlyConclusion}`, `Ⓜ️ <em>This room isn't bridged to Discord. ${matrixOnlyConclusion}</em>`, matrixOnlyReason === "NOT_BRIDGED")
 				.addLine(`Ⓜ️ *Discord ran out of space for emojis. ${matrixOnlyConclusion}`, `Ⓜ️ <em>Discord ran out of space for emojis. ${matrixOnlyConclusion}</em>`, matrixOnlyReason === "CAPACITY")
@@ -250,7 +201,7 @@ const commands = [{
 						}
 					}
 					if (!("images" in pack)) pack.images = {}
-					const b = new MatrixStringBuilder()
+					const b = new mxUtils.MatrixStringBuilder()
 						.addLine(`Created ${toUpload.length} emojis`, "")
 					for (const e of toUpload) {
 						pack.images[e.name] = {

package-lock.json

@@ -12,11 +12,13 @@
         "@chriscdn/promise-semaphore": "^2.0.1",
         "better-sqlite3": "^9.0.0",
         "chunk-text": "^2.0.1",
-        "cloudstorm": "^0.9.0",
+        "cloudstorm": ">=0.9.0",
         "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8",
         "entities": "^4.5.0",
         "giframe": "github:cloudrac3r/giframe#v0.4.1",
         "heatsync": "^2.4.1",
+        "html-es6cape": "^2.0.2",
+        "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0",
         "js-yaml": "^4.1.0",
         "matrix-appservice": "^2.0.0",
         "minimist": "^1.2.8",
@@ -782,9 +784,9 @@
       }
     },
     "node_modules/cloudstorm": {
-      "version": "0.9.0",
+      "version": "0.9.2",
-      "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.9.0.tgz",
+      "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.9.2.tgz",
-      "integrity": "sha512-n5M5TVnvm/X5vdNKy85q8muMregnvPWxv7HGSDCChL/FReOh2PGOm0FZJVm4hcB+KIM07KmiJTiCSQTnrTrSnQ==",
+      "integrity": "sha512-dXyK8/SseyhAvblPDbDILCb6ghpoJnBAiBx1ig5/yQ54TvOXlZJ4MC+So7EJDdaHkTgnf38F8qNyBNN29sMMcQ==",
       "dependencies": {
         "snowtransfer": "^0.9.0"
       },
@@ -1556,12 +1558,25 @@
         "backtracker": "3.3.2"
       }
     },
+    "node_modules/html-es6cape": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/html-es6cape/-/html-es6cape-2.0.2.tgz",
+      "integrity": "sha512-utzhH8rq2VABdW1LsPdv5tmxeMNOtP83If0jKCa79xPBgLWfcMvdf9K+EZoxJ5P7KioCxTs6WBnSDWLQHJ2lWA=="
+    },
     "node_modules/html-escaper": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
       "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==",
       "dev": true
     },
+    "node_modules/html-template-tag": {
+      "version": "5.0.0",
+      "resolved": "git+ssh://git@github.com/cloudrac3r/html-template-tag.git#9b2ec9efd344119997495c7889c11527cc6a35ed",
+      "license": "MIT",
+      "dependencies": {
+        "html-es6cape": "^2.0.0"
+      }
+    },
     "node_modules/http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",

package.json

@@ -18,11 +18,13 @@
     "@chriscdn/promise-semaphore": "^2.0.1",
     "better-sqlite3": "^9.0.0",
     "chunk-text": "^2.0.1",
-    "cloudstorm": "^0.9.0",
+    "cloudstorm": ">=0.9.0",
     "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8",
     "entities": "^4.5.0",
     "giframe": "github:cloudrac3r/giframe#v0.4.1",
     "heatsync": "^2.4.1",
+    "html-es6cape": "^2.0.2",
+    "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0",
     "js-yaml": "^4.1.0",
     "matrix-appservice": "^2.0.0",
     "minimist": "^1.2.8",

readme.md

@@ -161,14 +161,16 @@ Follow these steps:
 * (1) chunk-text: It does what I want.
 * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
 * (8) snowtransfer: Discord API library with bring-your-own-caching that I trust.
-* (1) discord-markdown: This is my fork! I make sure it does what I want.
+* (1) discord-markdown: This is my fork!
-* (0) giframe: This is my fork! It should do what I want.
+* (0) giframe: This is my fork!
 * (1) heatsync: Module hot-reloader that I trust.
 * (0) entities: Looks fine. No dependencies.
+* (0) html-es6cape: Looks great, and it's already pulled in by html-template-tag
+* (0) html-template-tag: This is my fork!
 * (1) js-yaml: It seems to do what I want, and it's already pulled in by matrix-appservice.
 * (70) matrix-appservice: I wish it didn't pull in express :(
 * (0) minimist: It's already pulled in by better-sqlite3->prebuild-install
-* (0) mixin-deep: This is my fork! It fixes a bug in regular mixin-deep.
+* (0) mixin-deep: This is my fork! (It fixes a bug in regular mixin-deep.)
 * (3) node-fetch@2: I like it and it does what I want.
 * (0) pngjs: Lottie stickers are converted to bitmaps with the vendored Rlottie WASM build, then the bitmaps are converted to PNG with pngjs.
 * (0) prettier-bytes: It does what I want and has no dependencies.

test/data.js

@@ -1605,6 +1605,114 @@ module.exports = {
 			edited_timestamp: "2023-10-01T01:42:05.631000+00:00",
 			flags: 0,
 			components: []
+		},
+		blockquote_in_embed: {
+			id: "1158894131322552391",
+			type: 0,
+			content: "<:emoji:288858540888686602> **4 |** <#176333891320283136>",
+			channel_id: "331390333810376704",
+			author: {
+				id: "700796664276844612",
+				username: "Starboard",
+				avatar: "1db8745493a3701235275be62ce05fea",
+				discriminator: "9387",
+				public_flags: 65536,
+				flags: 65536,
+				bot: true,
+				banner: null,
+				accent_color: null,
+				global_name: null,
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "rich",
+					description: "reply draft\n" +
+					"> The following is a message composed via consensus of the Stinker Council.\n" +
+					"> \n" +
+					"> For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n" +
+					"> \n" +
+					"> Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n" +
+					"> \n" +
+					"> There will be no further communication.",
+					color: 16769436,
+					timestamp: "2023-10-03T19:06:01.516000+00:00",
+					fields: [
+						{
+							name: "​",
+							value: "[Go to Message](https://discord.com/channels/112760669178241024/176333891320283136/1158842413025071135)",
+							inline: false
+						}
+					],
+					author: {
+						name: "minimus",
+						url: "https://discord.com/channels/112760669178241024/176333891320283136/1158842413025071135",
+						icon_url: "https://cdn.discordapp.com/guilds/112760669178241024/users/112760500130975744/avatars/caf8f18d190e92c280f8bc7e13f3dfb7.png",
+						proxy_icon_url: "https://images-ext-2.discordapp.net/external/ufuM1hu_C6wpfbLS-RVb5iqa_X6Ht3aIj-xntAo8jjw/https/cdn.discordapp.com/guilds/112760669178241024/users/112760500130975744/avatars/caf8f18d190e92c280f8bc7e13f3dfb7.png"
+					}
+				}
+			],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-10-03T22:31:32.119000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
+		},
+		escaping_crazy_html_tags: {
+			id: "1158894131322552391",
+			type: 0,
+			content: "",
+			channel_id: "331390333810376704",
+			author: {
+				id: "700796664276844612",
+				username: "Starboard",
+				avatar: "1db8745493a3701235275be62ce05fea",
+				discriminator: "9387",
+				public_flags: 65536,
+				flags: 65536,
+				bot: true,
+				banner: null,
+				accent_color: null,
+				global_name: null,
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [{
+				type: "rich",
+				title: "<strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+				description: "<strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+				url: "https://a.co/&amp;<script>",
+				footer: {
+					text: "<strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+				},
+				author: {
+					name: "<strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+					url: "https://a.co/&amp;<script>",
+					icon_url: "https://a.co/&amp;<script>"
+				},
+				fields: [
+					{
+						name: "<strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)",
+						value: "<strong>[<span data-mx-color='#123456'>Hey<script>](https://a.co/&amp;)"
+					}
+				]
+			}],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-10-03T22:31:32.119000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: []
 		}
 	},
 	message_update: {

test/ooye-test-data.sql

@@ -10,7 +10,8 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom
 ('1100319550446252084', '!BnKuBPCvyfOkhcUjEu:cadence.moe', 'worm-farm', NULL, NULL, NULL),
 ('1162005314908999790', '!FuDZhlOAtqswlyxzeR:cadence.moe', 'Hey.', NULL, '1100319550446252084', NULL),
 ('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL),
-('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL);
+('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL),
+('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS');
 
 INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES
 ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'),
@@ -40,7 +41,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES
 ('1145688633186193480', '1100319550446252084'),
 ('1145688633186193481', '1100319550446252084'),
 ('1162005526675193909', '1162005314908999790'),
-('1162625810109317170', '497161350934560778');
+('1162625810109317170', '497161350934560778'),
+('1158842413025071135', '176333891320283136');
 
 INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES
 ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1),
@@ -64,7 +66,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
 ('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0, 0),
 ('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0, 0),
 ('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1, 1),
-('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 0, 1);
+('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 0, 1),
+('$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo', 'm.room.message', 'm.text', '1158842413025071135', 0, 0, 1);
 
 INSERT INTO file (discord_url, mxc_url) VALUES
 ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),
@@ -80,7 +83,8 @@ INSERT INTO file (discord_url, mxc_url) VALUES
 ('https://cdn.discordapp.com/emojis/393635038903926784.gif', 'mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc'),
 ('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'),
 ('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'),
-('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX');
+('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX'),
+('https://cdn.discordapp.com/emojis/288858540888686602.png', 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO');
 
 INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES
 ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'),
@@ -88,7 +92,8 @@ INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES
 ('362741439211503616', 'bn_re', 0, 'mxc://cadence.moe/OIpqpfxTnHKokcsYqDusxkBT'),
 ('551636841284108289', 'ae_botrac4r', 0, 'mxc://cadence.moe/skqfuItqxNmBYekzmVKyoLzs'),
 ('975572106295259148', 'brillillillilliant_move', 0, 'mxc://cadence.moe/scfRIDOGKWFDEBjVXocWYQHik'),
-('606664341298872324', 'online', 0, 'mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ');
+('606664341298872324', 'online', 0, 'mxc://cadence.moe/LCEqjStXCxvRQccEkuslXEyZ'),
+('288858540888686602', 'upstinky', 0, 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO');
 
 INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES
 ('!kLRqKKUQXcibIMtOpl:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', NULL),