cadence/out-of-your-element
cadence
/
out-of-your-element
2
6
Fork 1
Code Issues 13 Pull Requests Releases 10 Activity

m->d: Emoji sprite sheet supports APNG

This commit is contained in:
Cadence Ember 2024-02-01 16:38:17 +13:00
parent f48c1f3f31
commit 6c3164edd6
5 changed files with 138 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
m2d/converters/emoji-sheet.js
View File

@ -4,6 +4,7 @@ const assert = require("assert").strict
const {pipeline} = require("stream").promises
const sharp = require("sharp")
const {GIFrame} = require("giframe")
const {PNG} = require("pngjs")
const utils = require("./utils")
const fetch = require("node-fetch").default
const streamMimeType = require("stream-mime-type")
@ -18,57 +19,23 @@ const IMAGES_ACROSS = Math.floor(RESULT_WIDTH / SIZE)
* @returns {Promise<Buffer>} PNG image
*/
async function compositeMatrixEmojis(mxcs) {
let buffers = await Promise.all(mxcs.map(async mxc => {
const buffers = await Promise.all(mxcs.map(async mxc => {
const abortController = new AbortController()
try {
const url = utils.getPublicUrlForMxc(mxc)
assert(url)
const url = utils.getPublicUrlForMxc(mxc)
assert(url)
/** @type {import("node-fetch").Response} res */
// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
// If we were using connection pooling, we would be forced to download the entire GIF.
// So we set no agent to ensure we are not connection pooling.
// @ts-ignore the signal is slightly different from the type it wants (still works fine)
const res = await fetch(url, {agent: false, signal: abortController.signal})
const {stream, mime} = await streamMimeType.getMimeType(res.body)
assert(["image/png", "image/jpeg", "image/webp", "image/gif"].includes(mime), `Mime type ${mime} is impossible for emojis`)
if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") {
/** @type {{info: sharp.OutputInfo, buffer: Buffer}} */
const result = await new Promise((resolve, reject) => {
const transformer = sharp()
.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
.png({compressionLevel: 0})
.toBuffer((err, buffer, info) => {
/* c8 ignore next */
if (err) return reject(err)
resolve({info, buffer})
})
pipeline(
stream,
transformer
)
})
return result.buffer
} else if (mime === "image/gif") {
const giframe = new GIFrame(0)
stream.on("data", chunk => {
giframe.feed(chunk)
})
const frame = await giframe.getFrame()
const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}})
.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
.png({compressionLevel: 0})
.toBuffer({resolveWithObject: true})
return buffer.data
}
} finally {
/** @type {import("node-fetch").Response} */
// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
// If we were using connection pooling, we would be forced to download the entire GIF.
// So we set no agent to ensure we are not connection pooling.
// @ts-ignore the signal is slightly different from the type it wants (still works fine)
const res = await fetch(url, {agent: false, signal: abortController.signal})
return convertImageStream(res.body, () => {
abortController.abort()
}
res.body.pause()
res.body.emit("end")
})
}))
// Calculate the size of the final composited image
@ -98,4 +65,67 @@ async function compositeMatrixEmojis(mxcs) {
return output.data
}
/**
* @param {import("node-fetch").Response["body"]} streamIn
* @param {() => any} stopStream
* @returns {Promise<Buffer | undefined>} Uncompressed PNG image
*/
async function convertImageStream(streamIn, stopStream) {
const {stream, mime} = await streamMimeType.getMimeType(streamIn)
assert(["image/png", "image/jpeg", "image/webp", "image/gif", "image/apng"].includes(mime), `Mime type ${mime} is impossible for emojis`)
try {
if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") {
/** @type {{info: sharp.OutputInfo, buffer: Buffer}} */
const result = await new Promise((resolve, reject) => {
const transformer = sharp()
.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
.png({compressionLevel: 0})
.toBuffer((err, buffer, info) => {
/* c8 ignore next */
if (err) return reject(err)
resolve({info, buffer})
})
pipeline(
stream,
transformer
)
})
return result.buffer
} else if (mime === "image/gif") {
const giframe = new GIFrame(0)
stream.on("data", chunk => {
giframe.feed(chunk)
})
const frame = await giframe.getFrame()
stopStream()
const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}})
.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
.png({compressionLevel: 0})
.toBuffer({resolveWithObject: true})
return buffer.data
} else if (mime === "image/apng") {
const png = new PNG({maxFrames: 1})
// @ts-ignore
stream.pipe(png)
/** @type {Buffer} */ // @ts-ignore
const frame = await new Promise(resolve => png.on("parsed", resolve))
stopStream()
const buffer = await sharp(frame, {raw: {width: png.width, height: png.height, channels: 4}})
.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
.png({compressionLevel: 0})
.toBuffer({resolveWithObject: true})
return buffer.data
}
} finally {
stopStream()
}
}
module.exports.compositeMatrixEmojis = compositeMatrixEmojis
module.exports._convertImageStream = convertImageStream

55
m2d/converters/emoji-sheet.test.js Normal file
View File

@ -0,0 +1,55 @@
const assert = require("assert").strict
const {test} = require("supertape")
const {_convertImageStream} = require("./emoji-sheet")
const fetch = require("node-fetch")
const {Transform} = require("stream").Transform
/* c8 ignore next 7 */
function slow() {
if (process.argv.includes("--slow")) {
return test
} else {
return test.skip
}
}
class Meter extends Transform {
bytes = 0
_transform(chunk, encoding, cb) {
this.bytes += chunk.length
this.push(chunk)
cb()
}
}
/**
* @param {import("supertape").Test} t
* @param {string} url
* @param {number} totalSize
*/
async function runSingleTest(t, url, totalSize) {
const abortController = new AbortController()
const res = await fetch("https://ezgif.com/images/format-demo/butterfly.png", {agent: false, signal: abortController.signal})
const meter = new Meter()
const p = res.body.pipe(meter)
const result = await _convertImageStream(p, () => {
abortController.abort()
res.body.pause()
res.body.emit("end")
})
t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `result was not a PNG file: ${result.toString("base64")}`)
if (meter.bytes < totalSize / 4) { // should download less than 25% of each file
t.pass("intentionally read partial file")
} else {
t.fail(`read more than 25% of file, read: ${meter.bytes}, total: ${totalSize}`)
}
}
slow()("emoji-sheet: only partial file is read for APNG", async t => {
await runSingleTest(t, "https://ezgif.com/images/format-demo/butterfly.png", 2438998)
})
slow()("emoji-sheet: only partial file is read for GIF", async t => {
await runSingleTest(t, "https://ezgif.com/images/format-demo/butterfly.gif", 781223)
})

8
package-lock.json generated
View File

@ -24,7 +24,7 @@
"minimist": "^1.2.8",
"mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
"node-fetch": "^2.6.7",
"pngjs": "^7.0.0",
"pngjs": "github:cloudrac3r/pngjs#v7.0.1",
"prettier-bytes": "^1.0.4",
"sharp": "^0.32.6",
"snowtransfer": "^0.9.0",
@ -2403,9 +2403,9 @@
}
},
"node_modules/pngjs": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz",
"integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==",
"version": "7.0.1",
"resolved": "git+ssh://git@github.com/cloudrac3r/pngjs.git#55ad02b641a4a4de5cbe00329e8bca600b5bfa3b",
"license": "MIT",
"engines": {
"node": ">=14.19.0"
}

2
package.json
View File

@ -30,7 +30,7 @@
"minimist": "^1.2.8",
"mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
"node-fetch": "^2.6.7",
"pngjs": "^7.0.0",
"pngjs": "github:cloudrac3r/pngjs#v7.0.1",
"prettier-bytes": "^1.0.4",
"sharp": "^0.32.6",
"snowtransfer": "^0.9.0",

1
test/test.js
View File

@ -69,4 +69,5 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
require("../d2m/converters/user-to-mxid.test")
require("../m2d/converters/event-to-message.test")
require("../m2d/converters/utils.test")
require("../m2d/converters/emoji-sheet.test")
})()