From 6c3164edd687d22577df8a259ce45da32ec144b9 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 1 Feb 2024 16:38:17 +1300 Subject: [PATCH] m->d: Emoji sprite sheet supports APNG --- m2d/converters/emoji-sheet.js | 124 ++++++++++++++++++----------- m2d/converters/emoji-sheet.test.js | 55 +++++++++++++ package-lock.json | 8 +- package.json | 2 +- test/test.js | 1 + 5 files changed, 138 insertions(+), 52 deletions(-) create mode 100644 m2d/converters/emoji-sheet.test.js diff --git a/m2d/converters/emoji-sheet.js b/m2d/converters/emoji-sheet.js index c85176b..9e8703d 100644 --- a/m2d/converters/emoji-sheet.js +++ b/m2d/converters/emoji-sheet.js @@ -4,6 +4,7 @@ const assert = require("assert").strict const {pipeline} = require("stream").promises const sharp = require("sharp") const {GIFrame} = require("giframe") +const {PNG} = require("pngjs") const utils = require("./utils") const fetch = require("node-fetch").default const streamMimeType = require("stream-mime-type") @@ -18,57 +19,23 @@ const IMAGES_ACROSS = Math.floor(RESULT_WIDTH / SIZE) * @returns {Promise} PNG image */ async function compositeMatrixEmojis(mxcs) { - let buffers = await Promise.all(mxcs.map(async mxc => { + const buffers = await Promise.all(mxcs.map(async mxc => { const abortController = new AbortController() - try { - const url = utils.getPublicUrlForMxc(mxc) - assert(url) + const url = utils.getPublicUrlForMxc(mxc) + assert(url) - /** @type {import("node-fetch").Response} res */ - // If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing. - // If we were using connection pooling, we would be forced to download the entire GIF. - // So we set no agent to ensure we are not connection pooling. - // @ts-ignore the signal is slightly different from the type it wants (still works fine) - const res = await fetch(url, {agent: false, signal: abortController.signal}) - const {stream, mime} = await streamMimeType.getMimeType(res.body) - assert(["image/png", "image/jpeg", "image/webp", "image/gif"].includes(mime), `Mime type ${mime} is impossible for emojis`) - - if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") { - /** @type {{info: sharp.OutputInfo, buffer: Buffer}} */ - const result = await new Promise((resolve, reject) => { - const transformer = sharp() - .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) - .png({compressionLevel: 0}) - .toBuffer((err, buffer, info) => { - /* c8 ignore next */ - if (err) return reject(err) - resolve({info, buffer}) - }) - pipeline( - stream, - transformer - ) - }) - return result.buffer - - } else if (mime === "image/gif") { - const giframe = new GIFrame(0) - stream.on("data", chunk => { - giframe.feed(chunk) - }) - const frame = await giframe.getFrame() - - const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}}) - .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) - .png({compressionLevel: 0}) - .toBuffer({resolveWithObject: true}) - return buffer.data - - } - } finally { + /** @type {import("node-fetch").Response} */ + // If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing. + // If we were using connection pooling, we would be forced to download the entire GIF. + // So we set no agent to ensure we are not connection pooling. + // @ts-ignore the signal is slightly different from the type it wants (still works fine) + const res = await fetch(url, {agent: false, signal: abortController.signal}) + return convertImageStream(res.body, () => { abortController.abort() - } + res.body.pause() + res.body.emit("end") + }) })) // Calculate the size of the final composited image @@ -98,4 +65,67 @@ async function compositeMatrixEmojis(mxcs) { return output.data } +/** + * @param {import("node-fetch").Response["body"]} streamIn + * @param {() => any} stopStream + * @returns {Promise} Uncompressed PNG image + */ +async function convertImageStream(streamIn, stopStream) { + const {stream, mime} = await streamMimeType.getMimeType(streamIn) + assert(["image/png", "image/jpeg", "image/webp", "image/gif", "image/apng"].includes(mime), `Mime type ${mime} is impossible for emojis`) + + try { + if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") { + /** @type {{info: sharp.OutputInfo, buffer: Buffer}} */ + const result = await new Promise((resolve, reject) => { + const transformer = sharp() + .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .png({compressionLevel: 0}) + .toBuffer((err, buffer, info) => { + /* c8 ignore next */ + if (err) return reject(err) + resolve({info, buffer}) + }) + pipeline( + stream, + transformer + ) + }) + return result.buffer + + } else if (mime === "image/gif") { + const giframe = new GIFrame(0) + stream.on("data", chunk => { + giframe.feed(chunk) + }) + const frame = await giframe.getFrame() + stopStream() + + const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}}) + .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .png({compressionLevel: 0}) + .toBuffer({resolveWithObject: true}) + return buffer.data + + } else if (mime === "image/apng") { + const png = new PNG({maxFrames: 1}) + // @ts-ignore + stream.pipe(png) + /** @type {Buffer} */ // @ts-ignore + const frame = await new Promise(resolve => png.on("parsed", resolve)) + stopStream() + + const buffer = await sharp(frame, {raw: {width: png.width, height: png.height, channels: 4}}) + .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .png({compressionLevel: 0}) + .toBuffer({resolveWithObject: true}) + return buffer.data + + } + } finally { + stopStream() + } +} + module.exports.compositeMatrixEmojis = compositeMatrixEmojis +module.exports._convertImageStream = convertImageStream diff --git a/m2d/converters/emoji-sheet.test.js b/m2d/converters/emoji-sheet.test.js new file mode 100644 index 0000000..f75fafc --- /dev/null +++ b/m2d/converters/emoji-sheet.test.js @@ -0,0 +1,55 @@ +const assert = require("assert").strict +const {test} = require("supertape") +const {_convertImageStream} = require("./emoji-sheet") +const fetch = require("node-fetch") +const {Transform} = require("stream").Transform + +/* c8 ignore next 7 */ +function slow() { + if (process.argv.includes("--slow")) { + return test + } else { + return test.skip + } +} + +class Meter extends Transform { + bytes = 0 + + _transform(chunk, encoding, cb) { + this.bytes += chunk.length + this.push(chunk) + cb() + } +} + +/** + * @param {import("supertape").Test} t + * @param {string} url + * @param {number} totalSize + */ +async function runSingleTest(t, url, totalSize) { + const abortController = new AbortController() + const res = await fetch("https://ezgif.com/images/format-demo/butterfly.png", {agent: false, signal: abortController.signal}) + const meter = new Meter() + const p = res.body.pipe(meter) + const result = await _convertImageStream(p, () => { + abortController.abort() + res.body.pause() + res.body.emit("end") + }) + t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `result was not a PNG file: ${result.toString("base64")}`) + if (meter.bytes < totalSize / 4) { // should download less than 25% of each file + t.pass("intentionally read partial file") + } else { + t.fail(`read more than 25% of file, read: ${meter.bytes}, total: ${totalSize}`) + } +} + +slow()("emoji-sheet: only partial file is read for APNG", async t => { + await runSingleTest(t, "https://ezgif.com/images/format-demo/butterfly.png", 2438998) +}) + +slow()("emoji-sheet: only partial file is read for GIF", async t => { + await runSingleTest(t, "https://ezgif.com/images/format-demo/butterfly.gif", 781223) +}) diff --git a/package-lock.json b/package-lock.json index 054e701..5bc10cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "minimist": "^1.2.8", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "node-fetch": "^2.6.7", - "pngjs": "^7.0.0", + "pngjs": "github:cloudrac3r/pngjs#v7.0.1", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", "snowtransfer": "^0.9.0", @@ -2403,9 +2403,9 @@ } }, "node_modules/pngjs": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", - "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "version": "7.0.1", + "resolved": "git+ssh://git@github.com/cloudrac3r/pngjs.git#55ad02b641a4a4de5cbe00329e8bca600b5bfa3b", + "license": "MIT", "engines": { "node": ">=14.19.0" } diff --git a/package.json b/package.json index 58a8674..c9d3910 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "minimist": "^1.2.8", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "node-fetch": "^2.6.7", - "pngjs": "^7.0.0", + "pngjs": "github:cloudrac3r/pngjs#v7.0.1", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", "snowtransfer": "^0.9.0", diff --git a/test/test.js b/test/test.js index 6c912c8..8e7f193 100644 --- a/test/test.js +++ b/test/test.js @@ -69,4 +69,5 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../d2m/converters/user-to-mxid.test") require("../m2d/converters/event-to-message.test") require("../m2d/converters/utils.test") + require("../m2d/converters/emoji-sheet.test") })()