Add support for Lottie stickers

This commit is contained in:
Cadence Ember 2023-09-10 21:35:51 +12:00
parent 5bf051c624
commit d759b5bd90
8 changed files with 115 additions and 10 deletions

74
d2m/converters/lottie.js Normal file
View file

@ -0,0 +1,74 @@
// @ts-check
const DiscordTypes = require("discord-api-types/v10")
const Ty = require("../../types")
const assert = require("assert").strict
const {PNG} = require("pngjs")
const passthrough = require("../../passthrough")
const { sync, db, discord } = passthrough
/** @type {import("../../matrix/file")} */
const file = sync.require("../../matrix/file")
//** @type {import("../../matrix/mreq")} */
const mreq = sync.require("../../matrix/mreq")
const SIZE = 160 // Discord's display size on 1x displays is 160
const INFO = {
mimetype: "image/png",
w: SIZE,
h: SIZE
}
/**
* @typedef RlottieWasm
* @prop {(string) => boolean} load load lottie data from string of json
* @prop {() => number} frames get number of frames
* @prop {(frameCount: number, width: number, height: number) => Uint8Array} render render lottie data to bitmap
*/
const Rlottie = (async () => {
const Rlottie = require("./rlottie-wasm.js")
await new Promise(resolve => Rlottie.onRuntimeInitialized = resolve)
return Rlottie
})()
/**
* @param {DiscordTypes.APIStickerItem} stickerItem
* @returns {Promise<{mxc: string, info: typeof INFO}>}
*/
async function convert(stickerItem) {
const existingMxc = db.prepare("SELECT mxc FROM lottie WHERE id = ?").pluck().get(stickerItem.id)
if (existingMxc) return {mxc: existingMxc, info: INFO}
const r = await Rlottie
const res = await fetch(file.DISCORD_IMAGES_BASE + file.sticker(stickerItem))
if (res.status !== 200) throw new Error("Sticker data file not found.")
const text = await res.text()
/** @type RlottieWasm */
const rh = new r.RlottieWasm()
const status = rh.load(text)
if (!status) throw new Error(`Rlottie unable to load ${text.length} byte data file.`)
const rendered = rh.render(0, SIZE, SIZE)
let png = new PNG({
width: SIZE,
height: SIZE,
bitDepth: 8, // 8 red + 8 green + 8 blue + 8 alpha
colorType: 6, // RGBA
inputColorType: 6, // RGBA
inputHasAlpha: true,
})
png.data = Buffer.from(rendered)
// @ts-ignore wrong type from pngjs
const readablePng = png.pack()
/** @type {Ty.R.FileUploaded} */
const root = await mreq.mreq("POST", "/media/v3/upload", readablePng, {
headers: {
"Content-Type": INFO.mimetype
}
})
assert(root.content_uri)
db.prepare("INSERT INTO lottie (id, mxc) VALUES (?, ?)").run(stickerItem.id, root.content_uri)
return {mxc: root.content_uri, info: INFO}
}
module.exports.convert = convert

View file

@ -9,6 +9,8 @@ const passthrough = require("../../passthrough")
const { sync, db, discord } = passthrough const { sync, db, discord } = passthrough
/** @type {import("../../matrix/file")} */ /** @type {import("../../matrix/file")} */
const file = sync.require("../../matrix/file") const file = sync.require("../../matrix/file")
/** @type {import("./lottie")} */
const lottie = sync.require("./lottie")
const reg = require("../../matrix/read-registration") const reg = require("../../matrix/read-registration")
const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex)) const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
@ -338,7 +340,25 @@ async function messageToEvent(message, guild, options = {}, di) {
if (message.sticker_items) { if (message.sticker_items) {
const stickerEvents = await Promise.all(message.sticker_items.map(async stickerItem => { const stickerEvents = await Promise.all(message.sticker_items.map(async stickerItem => {
const format = file.stickerFormat.get(stickerItem.format_type) const format = file.stickerFormat.get(stickerItem.format_type)
if (format?.mime) { if (format?.mime === "lottie") {
try {
const {mxc, info} = await lottie.convert(stickerItem)
return {
$type: "m.sticker",
"m.mentions": mentions,
body: stickerItem.name,
info,
url: mxc
}
} catch (e) {
return {
$type: "m.room.message",
"m.mentions": mentions,
msgtype: "m.notice",
body: `Failed to convert Lottie sticker:\n${e.toString()}\n${e.stack}`
}
}
} else if (format?.mime) {
let body = stickerItem.name let body = stickerItem.name
const sticker = guild.stickers.find(sticker => sticker.id === stickerItem.id) const sticker = guild.stickers.find(sticker => sticker.id === stickerItem.id)
if (sticker && sticker.description) body += ` - ${sticker.description}` if (sticker && sticker.description) body += ` - ${sticker.description}`
@ -351,13 +371,12 @@ async function messageToEvent(message, guild, options = {}, di) {
}, },
url: await file.uploadDiscordFileToMxc(file.sticker(stickerItem)) url: await file.uploadDiscordFileToMxc(file.sticker(stickerItem))
} }
} else { }
return { return {
$type: "m.room.message", $type: "m.room.message",
"m.mentions": mentions, "m.mentions": mentions,
msgtype: "m.text", msgtype: "m.notice",
body: "Unsupported sticker format. Name: " + stickerItem.name body: `Unsupported sticker format ${format?.mime}. Name: ${stickerItem.name}`
}
} }
})) }))
events.push(...stickerEvents) events.push(...stickerEvents)

File diff suppressed because one or more lines are too long

BIN
d2m/converters/rlottie-wasm.wasm Executable file

Binary file not shown.

View file

@ -87,7 +87,7 @@ function emoji(emojiID, animated) {
const stickerFormat = new Map([ const stickerFormat = new Map([
[1, {label: "PNG", ext: "png", mime: "image/png"}], [1, {label: "PNG", ext: "png", mime: "image/png"}],
[2, {label: "APNG", ext: "png", mime: "image/apng"}], [2, {label: "APNG", ext: "png", mime: "image/apng"}],
[3, {label: "LOTTIE", ext: "json", mime: null}], [3, {label: "LOTTIE", ext: "json", mime: "lottie"}],
[4, {label: "GIF", ext: "gif", mime: "image/gif"}] [4, {label: "GIF", ext: "gif", mime: "image/gif"}]
]) ])

9
package-lock.json generated
View file

@ -19,6 +19,7 @@
"matrix-appservice": "^2.0.0", "matrix-appservice": "^2.0.0",
"mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
"node-fetch": "^2.6.7", "node-fetch": "^2.6.7",
"pngjs": "^7.0.0",
"prettier-bytes": "^1.0.4", "prettier-bytes": "^1.0.4",
"snowtransfer": "^0.8.0", "snowtransfer": "^0.8.0",
"try-to-catch": "^3.0.1", "try-to-catch": "^3.0.1",
@ -2281,6 +2282,14 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
}, },
"node_modules/pngjs": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz",
"integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==",
"engines": {
"node": ">=14.19.0"
}
},
"node_modules/prebuild-install": { "node_modules/prebuild-install": {
"version": "7.1.1", "version": "7.1.1",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",

View file

@ -25,6 +25,7 @@
"matrix-appservice": "^2.0.0", "matrix-appservice": "^2.0.0",
"mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
"node-fetch": "^2.6.7", "node-fetch": "^2.6.7",
"pngjs": "^7.0.0",
"prettier-bytes": "^1.0.4", "prettier-bytes": "^1.0.4",
"snowtransfer": "^0.8.0", "snowtransfer": "^0.8.0",
"try-to-catch": "^3.0.1", "try-to-catch": "^3.0.1",

View file

@ -21,7 +21,7 @@ Most features you'd expect in both directions, plus a little extra spice:
* Mentions * Mentions
* Replies * Replies
* Threads * Threads
* Stickers * Stickers (all formats: PNG, APNG, GIF, and Lottie)
* Attachments * Attachments
* Spoiler attachments * Spoiler attachments
* Guild-Space details syncing * Guild-Space details syncing
@ -124,6 +124,7 @@ I recommend developing in Visual Studio Code so that the JSDoc x TypeScript anno
* (70) matrix-appservice: I wish it didn't pull in express :( * (70) matrix-appservice: I wish it didn't pull in express :(
* (0) mixin-deep: This is my fork! It fixes a bug in regular mixin-deep. * (0) mixin-deep: This is my fork! It fixes a bug in regular mixin-deep.
* (3) node-fetch@2: I like it and it does what I want. * (3) node-fetch@2: I like it and it does what I want.
* (0) pngjs: Lottie stickers are converted to bitmaps with the vendored Rlottie WASM build, then the bitmaps are converted to PNG with pngjs.
* (0) prettier-bytes: It does what I want and has no dependencies. * (0) prettier-bytes: It does what I want and has no dependencies.
* (0) try-to-catch: Not strictly necessary, but it does what I want and has no dependencies. * (0) try-to-catch: Not strictly necessary, but it does what I want and has no dependencies.
* (1) turndown: I need an HTML-to-Markdown converter and this one looked suitable enough. It has some bugs that I've worked around, so I might switch away from it later. * (1) turndown: I need an HTML-to-Markdown converter and this one looked suitable enough. It has some bugs that I've worked around, so I might switch away from it later.