Add support for Lottie stickers

d2m/converters/lottie.js

@@ -0,0 +1,74 @@
+// @ts-check
+
+const DiscordTypes = require("discord-api-types/v10")
+const Ty = require("../../types")
+const assert = require("assert").strict
+const {PNG} = require("pngjs")
+
+const passthrough = require("../../passthrough")
+const { sync, db, discord } = passthrough
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+//** @type {import("../../matrix/mreq")} */
+const mreq = sync.require("../../matrix/mreq")
+
+const SIZE = 160 // Discord's display size on 1x displays is 160
+
+const INFO = {
+	mimetype: "image/png",
+	w: SIZE,
+	h: SIZE
+}
+
+/**
+ * @typedef RlottieWasm
+ * @prop {(string) => boolean} load load lottie data from string of json
+ * @prop {() => number} frames get number of frames
+ * @prop {(frameCount: number, width: number, height: number) => Uint8Array} render render lottie data to bitmap
+ */
+
+const Rlottie = (async () => {
+	const Rlottie = require("./rlottie-wasm.js")
+	await new Promise(resolve => Rlottie.onRuntimeInitialized = resolve)
+	return Rlottie
+})()
+
+/**
+ * @param {DiscordTypes.APIStickerItem} stickerItem
+ * @returns {Promise<{mxc: string, info: typeof INFO}>}
+ */
+async function convert(stickerItem) {
+	const existingMxc = db.prepare("SELECT mxc FROM lottie WHERE id = ?").pluck().get(stickerItem.id)
+	if (existingMxc) return {mxc: existingMxc, info: INFO}
+	const r = await Rlottie
+	const res = await fetch(file.DISCORD_IMAGES_BASE + file.sticker(stickerItem))
+	if (res.status !== 200) throw new Error("Sticker data file not found.")
+	const text = await res.text()
+	/** @type RlottieWasm */
+	const rh = new r.RlottieWasm()
+	const status = rh.load(text)
+	if (!status) throw new Error(`Rlottie unable to load ${text.length} byte data file.`)
+	const rendered = rh.render(0, SIZE, SIZE)
+	let png = new PNG({
+		width: SIZE,
+		height: SIZE,
+		bitDepth: 8, // 8 red + 8 green + 8 blue + 8 alpha
+		colorType: 6, // RGBA
+		inputColorType: 6, // RGBA
+		inputHasAlpha: true,
+	})
+	png.data = Buffer.from(rendered)
+	// @ts-ignore wrong type from pngjs
+	const readablePng = png.pack()
+	/** @type {Ty.R.FileUploaded} */
+	const root = await mreq.mreq("POST", "/media/v3/upload", readablePng, {
+		headers: {
+			"Content-Type": INFO.mimetype
+		}
+	})
+	assert(root.content_uri)
+	db.prepare("INSERT INTO lottie (id, mxc) VALUES (?, ?)").run(stickerItem.id, root.content_uri)
+	return {mxc: root.content_uri, info: INFO}
+}
+
+module.exports.convert = convert

d2m/converters/message-to-event.js

@@ -9,6 +9,8 @@ const passthrough = require("../../passthrough")
 const { sync, db, discord } = passthrough
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
+/** @type {import("./lottie")} */
+const lottie = sync.require("./lottie")
 const reg = require("../../matrix/read-registration")
 
 const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
@@ -338,7 +340,25 @@ async function messageToEvent(message, guild, options = {}, di) {
 	if (message.sticker_items) {
 		const stickerEvents = await Promise.all(message.sticker_items.map(async stickerItem => {
 			const format = file.stickerFormat.get(stickerItem.format_type)
-			if (format?.mime) {
+			if (format?.mime === "lottie") {
+				try {
+					const {mxc, info} = await lottie.convert(stickerItem)
+					return {
+						$type: "m.sticker",
+						"m.mentions": mentions,
+						body: stickerItem.name,
+						info,
+						url: mxc
+					}
+				} catch (e) {
+					return {
+						$type: "m.room.message",
+						"m.mentions": mentions,
+						msgtype: "m.notice",
+						body: `Failed to convert Lottie sticker:\n${e.toString()}\n${e.stack}`
+					}
+				}
+			} else if (format?.mime) {
 				let body = stickerItem.name
 				const sticker = guild.stickers.find(sticker => sticker.id === stickerItem.id)
 				if (sticker && sticker.description) body += ` - ${sticker.description}`
@@ -351,13 +371,12 @@ async function messageToEvent(message, guild, options = {}, di) {
 					},
 					url: await file.uploadDiscordFileToMxc(file.sticker(stickerItem))
 				}
-			} else {
+			}
-				return {
+			return {
-					$type: "m.room.message",
+				$type: "m.room.message",
-					"m.mentions": mentions,
+				"m.mentions": mentions,
-					msgtype: "m.text",
+				msgtype: "m.notice",
-					body: "Unsupported sticker format. Name: " + stickerItem.name
+				body: `Unsupported sticker format ${format?.mime}. Name: ${stickerItem.name}`
-				}
 			}
 		}))
 		events.push(...stickerEvents)

matrix/file.js

@@ -87,7 +87,7 @@ function emoji(emojiID, animated) {
 const stickerFormat = new Map([
 	[1, {label: "PNG", ext: "png", mime: "image/png"}],
 	[2, {label: "APNG", ext: "png", mime: "image/apng"}],
-	[3, {label: "LOTTIE", ext: "json", mime: null}],
+	[3, {label: "LOTTIE", ext: "json", mime: "lottie"}],
 	[4, {label: "GIF", ext: "gif", mime: "image/gif"}]
 ])
 

package-lock.json

@@ -19,6 +19,7 @@
         "matrix-appservice": "^2.0.0",
         "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
         "node-fetch": "^2.6.7",
+        "pngjs": "^7.0.0",
         "prettier-bytes": "^1.0.4",
         "snowtransfer": "^0.8.0",
         "try-to-catch": "^3.0.1",
@@ -2281,6 +2282,14 @@
       "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
       "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
     },
+    "node_modules/pngjs": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz",
+      "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==",
+      "engines": {
+        "node": ">=14.19.0"
+      }
+    },
     "node_modules/prebuild-install": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",

package.json

@@ -25,6 +25,7 @@
     "matrix-appservice": "^2.0.0",
     "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
     "node-fetch": "^2.6.7",
+    "pngjs": "^7.0.0",
     "prettier-bytes": "^1.0.4",
     "snowtransfer": "^0.8.0",
     "try-to-catch": "^3.0.1",

readme.md

@@ -21,7 +21,7 @@ Most features you'd expect in both directions, plus a little extra spice:
 * Mentions
 * Replies
 * Threads
-* Stickers
+* Stickers (all formats: PNG, APNG, GIF, and Lottie)
 * Attachments
 * Spoiler attachments
 * Guild-Space details syncing
@@ -124,6 +124,7 @@ I recommend developing in Visual Studio Code so that the JSDoc x TypeScript anno
 * (70) matrix-appservice: I wish it didn't pull in express :(
 * (0) mixin-deep: This is my fork! It fixes a bug in regular mixin-deep.
 * (3) node-fetch@2: I like it and it does what I want.
+* (0) pngjs: Lottie stickers are converted to bitmaps with the vendored Rlottie WASM build, then the bitmaps are converted to PNG with pngjs.
 * (0) prettier-bytes: It does what I want and has no dependencies.
 * (0) try-to-catch: Not strictly necessary, but it does what I want and has no dependencies.
 * (1) turndown: I need an HTML-to-Markdown converter and this one looked suitable enough. It has some bugs that I've worked around, so I might switch away from it later.