Elliu/out-of-your-element
1
0
Fork 0
forked from cadence/out-of-your-element
Code Pull requests Activity

Rearrange testing emoji sheet images

Browse source
This commit is contained in:
Cadence Ember 2024-03-01 17:28:14 +13:00
parent 18ef337aef
commit c5d6c5e4c7
9 changed files with 515 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

4
m2d/actions/send-event.js
View file

@ -17,6 +17,8 @@ const eventToMessage = sync.require("../converters/event-to-message")
const api = sync.require("../../matrix/api")
/** @type {import("../../d2m/actions/register-user")} */
const registerUser = sync.require("../../d2m/actions/register-user")
/** @type {import("../converters/emoji-sheet")} */
const emojiSheet = sync.require("../converters/emoji-sheet")
/**
* @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message
@ -75,7 +77,7 @@ async function sendEvent(event) {
// no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it
let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch})
let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch, mxcDownloader: emojiSheet.getAndConvertEmoji})
messagesToEdit = await Promise.all(messagesToEdit.map(async e => {
e.message = await resolvePendingFiles(e.message)

68
m2d/converters/emoji-sheet.js
View file

@ -1,6 +1,7 @@
// @ts-check
const assert = require("assert").strict
const fs = require("fs")
const {pipeline} = require("stream").promises
const sharp = require("sharp")
const {GIFrame} = require("giframe")
@ -16,27 +17,11 @@ const IMAGES_ACROSS = Math.floor(RESULT_WIDTH / SIZE)
/**
* Composite a bunch of Matrix emojis into a kind of spritesheet image to upload to Discord.
* @param {string[]} mxcs mxc URLs, in order
* @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
* @returns {Promise<Buffer>} PNG image
*/
async function compositeMatrixEmojis(mxcs) {
const buffers = await Promise.all(mxcs.map(async mxc => {
const abortController = new AbortController()
const url = utils.getPublicUrlForMxc(mxc)
assert(url)
/** @type {import("node-fetch").Response} */
// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
// If we were using connection pooling, we would be forced to download the entire GIF.
// So we set no agent to ensure we are not connection pooling.
// @ts-ignore the signal is slightly different from the type it wants (still works fine)
const res = await fetch(url, {agent: false, signal: abortController.signal})
return convertImageStream(res.body, () => {
abortController.abort()
res.body.pause()
res.body.emit("end")
})
}))
async function compositeMatrixEmojis(mxcs, mxcDownloader) {
const buffers = await Promise.all(mxcs.map(mxcDownloader))
// Calculate the size of the final composited image
const totalWidth = Math.min(buffers.length, IMAGES_ACROSS) * SIZE
@ -65,6 +50,49 @@ async function compositeMatrixEmojis(mxcs) {
return output.data
}
/**
* Downloads the emoji from the web and converts to uncompressed PNG data.
* @param {string} mxc a single mxc:// URL
* @returns {Promise<Buffer | undefined>} uncompressed PNG data, or undefined if the downloaded emoji is not valid
*/
async function getAndConvertEmoji(mxc) {
const abortController = new AbortController()
const url = utils.getPublicUrlForMxc(mxc)
assert(url)
/** @type {import("node-fetch").Response} */
// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
// If we were using connection pooling, we would be forced to download the entire GIF.
// So we set no agent to ensure we are not connection pooling.
// @ts-ignore the signal is slightly different from the type it wants (still works fine)
const res = await fetch(url, {agent: false, signal: abortController.signal})
return convertImageStream(res.body, () => {
abortController.abort()
res.body.pause()
res.body.emit("end")
})
}
/**
* MOCK: Gets the emoji from the filesystem and converts to uncompressed PNG data.
* @param {string} mxc a single mxc:// URL
* @returns {Promise<Buffer | undefined>} uncompressed PNG data, or undefined if the downloaded emoji is not valid
*/
async function _mockGetAndConvertEmoji(mxc) {
const id = mxc.match(/\/([^./]*)$/)?.[1]
let s
if (fs.existsSync(`test/res/${id}.png`)) {
s = fs.createReadStream(`test/res/${id}.png`)
} else {
s = fs.createReadStream(`test/res/${id}.gif`)
}
return convertImageStream(s, () => {
s.pause()
s.emit("end")
})
}
/**
* @param {import("node-fetch").Response["body"]} streamIn
* @param {() => any} stopStream
@ -128,4 +156,6 @@ async function convertImageStream(streamIn, stopStream) {
}
module.exports.compositeMatrixEmojis = compositeMatrixEmojis
module.exports.getAndConvertEmoji = getAndConvertEmoji
module.exports._mockGetAndConvertEmoji = _mockGetAndConvertEmoji
module.exports._convertImageStream = convertImageStream

19
m2d/converters/emoji-sheet.test.js
View file

@ -27,8 +27,9 @@ class Meter extends Transform {
* @param {import("supertape").Test} t
* @param {string} path
* @param {number} totalSize
* @param {number => boolean} sizeCheck
*/
async function runSingleTest(t, path, totalSize) {
async function runSingleTest(t, path, totalSize, sizeCheck) {
const file = fs.createReadStream(path)
const meter = new Meter()
const p = file.pipe(meter)
@ -36,19 +37,23 @@ async function runSingleTest(t, path, totalSize) {
file.pause()
file.emit("end")
})
t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `result was not a PNG file: ${result.toString("base64")}`)
t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `test that this is a PNG file: ${result.toString("base64").slice(0, 100)}`)
/* c8 ignore next 5 */
if (meter.bytes < totalSize / 4) { // should download less than 25% of each file
t.pass("intentionally read partial file")
if (sizeCheck(meter.bytes)) {
t.pass("read the correct amount of the file")
} else {
t.fail(`read more than 25% of file, read: ${meter.bytes}, total: ${totalSize}`)
t.fail(`read too much or too little of the file, read: ${meter.bytes}, total: ${totalSize}`)
}
}
slow()("emoji-sheet: only partial file is read for APNG", async t => {
await runSingleTest(t, "test/res/butterfly.png", 2438998)
await runSingleTest(t, "test/res/butterfly.png", 2438998, n => n < 2438998 / 4) // should download less than 25% of the file
})
slow()("emoji-sheet: only partial file is read for GIF", async t => {
await runSingleTest(t, "test/res/butterfly.gif", 781223)
await runSingleTest(t, "test/res/butterfly.gif", 781223, n => n < 781223 / 4) // should download less than 25% of the file
})
slow()("emoji-sheet: entire file is read for static PNG", async t => {
await runSingleTest(t, "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png", 11301, n => n === 11301) // should download entire file
})

9
m2d/converters/event-to-message.js
View file

@ -304,8 +304,9 @@ function getUserOrProxyOwnerID(mxid) {
* @param {string} content
* @param {{id: string, name: string}[]} attachments
* @param {({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} pendingFiles
* @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
*/
async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles) {
async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, mxcDownloader) {
if (!content.includes("<::>")) return content // No unknown emojis, nothing to do
// Remove known and unknown emojis from the end of the message
const r = /<a?:[a-zA-Z0-9_]*:[0-9]*>\s*$/
@ -313,7 +314,7 @@ async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
content = content.replace(r, "")
}
// Create a sprite sheet of known and unknown emojis from the end of the message
const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis)
const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis, mxcDownloader)
// Attach it
const name = "emojis.png"
attachments.push({id: String(attachments.length), name})
@ -421,7 +422,7 @@ const attachmentEmojis = new Map([
/**
* @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event
* @param {import("discord-api-types/v10").APIGuild} guild
* @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di simple-as-nails dependency injection for the matrix API
* @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"], mxcDownloader: (mxc: string) => Promise<Buffer | undefined>}} di simple-as-nails dependency injection for the matrix API
*/
async function eventToMessage(event, guild, di) {
/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
@ -717,7 +718,7 @@ async function eventToMessage(event, guild, di) {
if (replyLine && content.startsWith("> ")) content = "\n" + content
// SPRITE SHEET EMOJIS FEATURE:
content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader)
} else {
// Looks like we're using the plaintext body!
content = event.content.body

11
m2d/converters/event-to-message.test.js
View file

@ -1,6 +1,7 @@
const assert = require("assert").strict
const {test} = require("supertape")
const {eventToMessage} = require("./event-to-message")
const {_mockGetAndConvertEmoji} = require("./emoji-sheet")
const data = require("../../test/data")
const {MatrixServerError} = require("../../matrix/mreq")
const {db, select, discord} = require("../../passthrough")
@ -3534,7 +3535,7 @@ slow()("event2message: unknown emoji at the end is reuploaded as a sprite sheet"
},
event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe"
})
}, {}, {mxcDownloader: _mockGetAndConvertEmoji})
const testResult = {
content: messages.messagesToSend[0].content,
fileName: messages.messagesToSend[0].pendingFiles[0].name,
@ -3559,7 +3560,7 @@ slow()("event2message: known emoji from an unreachable server at the end is reup
},
event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe"
})
}, {}, {mxcDownloader: _mockGetAndConvertEmoji})
const testResult = {
content: messages.messagesToSend[0].content,
fileName: messages.messagesToSend[0].pendingFiles[0].name,
@ -3584,7 +3585,7 @@ slow()("event2message: known and unknown emojis in the end are reuploaded as a s
},
event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe"
})
}, {}, {mxcDownloader: _mockGetAndConvertEmoji})
const testResult = {
content: messages.messagesToSend[0].content,
fileName: messages.messagesToSend[0].pendingFiles[0].name,
@ -3609,7 +3610,7 @@ slow()("event2message: all unknown chess emojis are reuploaded as a sprite sheet
},
event_id: "$Me6iE8C8CZyrDEOYYrXKSYRuuh_25Jj9kZaNrf7LKr4",
room_id: "!maggESguZBqGBZtSnr:cadence.moe"
})
}, {}, {mxcDownloader: _mockGetAndConvertEmoji})
const testResult = {
content: messages.messagesToSend[0].content,
fileName: messages.messagesToSend[0].pendingFiles[0].name,
@ -3618,6 +3619,6 @@ slow()("event2message: all unknown chess emojis are reuploaded as a sprite sheet
t.deepEqual(testResult, {
content: "testing",
fileName: "emojis.png",
fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAASAAAAAwCAYAAACxIqevAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAgAElEQVR4nOV9B1xUV9r3JMbEGBQLbRodhukDg2jWZP02"
fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAYAAAABgCAYAAAAU9KWJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAgAElEQVR4nOx9B3xT1/W/UkImYKZtLdt4a0uWMaQkzS9t"
})
})