diff --git a/package.json b/package.json index d0a154c..cce8204 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,6 @@ "setup": "node --enable-source-maps scripts/setup.js", "addbot": "node addbot.js", "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js | tap-dot", - "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot", "cover": "c8 -o test/coverage --skip-full -x db/migrations -x src/m2d/event-dispatcher.js -x src/matrix/file.js -x src/matrix/api.js -x src/d2m/converters/rlottie-wasm.js -r html -r text supertape --no-check-assertions-count --format fail --no-worker test/test.js -- --slow" } } diff --git a/scripts/start-server.js b/scripts/start-server.js index 0d4753a..44edbcb 100755 --- a/scripts/start-server.js +++ b/scripts/start-server.js @@ -34,5 +34,9 @@ passthrough.select = orm.select console.log("Discord gateway started") sync.require("../src/web/server") + discord.cloud.once("ready", () => { + as.listen() + }) + require("../src/stdin") })() diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 9ac174e..684b79d 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -348,26 +348,38 @@ function getUserOrProxyOwnerMention(mxid) { /** * At the time of this executing, we know what the end of message emojis are, and we know that at least one of them is unknown. - * This function will strip them from the content and generate the correct pending file of the sprite sheet. + * This function will strip them from the content and add a link that Discord will preview with a sprite sheet of emojis. * @param {string} content - * @param {{id: string, filename: string}[]} attachments - * @param {({name: string, mxc: string} | {name: string, mxc: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} pendingFiles - * @param {(mxc: string) => Promise} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock. + * @returns {string} new content with emoji sheet link */ -async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, mxcDownloader) { +function linkEndOfMessageSpriteSheet(content) { if (!content.includes("<::>")) return content // No unknown emojis, nothing to do // Remove known and unknown emojis from the end of the message const r = /\s*$/ + while (content.match(r)) { content = content.replace(r, "") } - // Create a sprite sheet of known and unknown emojis from the end of the message - const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis, mxcDownloader) - // Attach it - const filename = "emojis.png" - attachments.push({id: String(attachments.length), filename}) - pendingFiles.push({name: filename, buffer}) - return content + + // Use a markdown link to hide the URL. If this is the only thing in the message, Discord will hide it entirely, same as lone URLs. Good for us. + content = content.trimEnd() + content += " [\u2800](" // U+2800 Braille Pattern Blank is invisible on all known platforms but is digitally not a whitespace character + const afterLink = ")" + + // Make emojis URL params + const params = new URLSearchParams() + for (const mxc of endOfMessageEmojis) { + // We can do up to 2000 chars max. (In this maximal case it will get chunked to a separate message.) Ignore additional emojis. + const withoutMxc = mxUtils.makeMxcPublic(mxc) + const emojisLength = params.toString().length + encodeURIComponent(withoutMxc).length + 2 + if (content.length + emojisLength + afterLink.length > 2000) { + break + } + params.append("e", withoutMxc) + } + + const url = `${reg.ooye.bridge_origin}/download/sheet?${params.toString()}` + return content + url + afterLink } /** @@ -524,7 +536,7 @@ async function getL1L2ReplyLine(called = false) { * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_Start | Ty.Event.Outer_Org_Matrix_Msc3381_Poll_End} event * @param {DiscordTypes.APIGuild} guild * @param {DiscordTypes.APIGuildTextChannel} channel - * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, mxcDownloader: (mxc: string) => Promise, pollEnd?: {messageID: string}}} di simple-as-nails dependency injection for the matrix API + * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, pollEnd?: {messageID: string}}} di simple-as-nails dependency injection for the matrix API */ async function eventToMessage(event, guild, channel, di) { let displayName = event.sender @@ -937,7 +949,7 @@ async function eventToMessage(event, guild, channel, di) { if (replyLine && content.startsWith("> ")) content = "\n" + content // SPRITE SHEET EMOJIS FEATURE: - content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader) + content = await linkEndOfMessageSpriteSheet(content) } else { // Looks like we're using the plaintext body! content = event.content.body diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 551cbd0..f667d70 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -1,22 +1,11 @@ const assert = require("assert").strict -const fs = require("fs") const {test} = require("supertape") const DiscordTypes = require("discord-api-types/v10") const {eventToMessage} = require("./event-to-message") -const {convertImageStream} = require("./emoji-sheet") const data = require("../../../test/data") const {MatrixServerError} = require("../../matrix/mreq") const {select, discord} = require("../../passthrough") -/* c8 ignore next 7 */ -function slow() { - if (process.argv.includes("--slow")) { - return test - } else { - return test.skip - } -} - /** * @param {string} roomID * @param {string} eventID @@ -49,25 +38,6 @@ function sameFirstContentAndWhitespace(t, a, b) { t.equal(a2, b2) } -/** - * MOCK: Gets the emoji from the filesystem and converts to uncompressed PNG data. - * @param {string} mxc a single mxc:// URL - * @returns {Promise} uncompressed PNG data, or undefined if the downloaded emoji is not valid -*/ -async function mockGetAndConvertEmoji(mxc) { - const id = mxc.match(/\/([^./]*)$/)?.[1] - let s - if (fs.existsSync(`test/res/${id}.png`)) { - s = fs.createReadStream(`test/res/${id}.png`) - } else { - s = fs.createReadStream(`test/res/${id}.gif`) - } - return convertImageStream(s, () => { - s.pause() - s.emit("end") - }) -} - test("event2message: body is used when there is no formatted_body", async t => { t.deepEqual( await eventToMessage({ @@ -5335,102 +5305,122 @@ test("event2message: table", async t => { ) }) -slow()("event2message: unknown emoji at the end is reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "wrong body", - format: "org.matrix.custom.html", - formatted_body: 'a b \":ms_robot_grin:\"' - }, - event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "a b", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAAAPoAAAD6AG1e1JrAAALoklEQVR4nM1ZaVBU2RU+LZSIGnAvFUtcRkSk6abpbkDH" - }) +test("event2message: unknown emoji at the end is used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: 'a b \":ms_robot_grin:\"' + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "a b [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FRLMgJGfgTPjIQtvvWZsYjhjy)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) -slow()("event2message: known emoji from an unreachable server at the end is reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "wrong body", - format: "org.matrix.custom.html", - formatted_body: 'a b \":emoji_from_unreachable_server:\"' - }, - event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "a b", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAOoUlEQVR4nM1aCXBbx3l+Eu8bN0CAuO+TAHGTFAmAJHgT" - }) +test("event2message: known emoji from an unreachable server at the end is used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: 'a b \":emoji_from_unreachable_server:\"' + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "a b [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FbZFuuUSEebJYXUMSxuuSuLTa)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) -slow()("event2message: known and unknown emojis in the end are reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "wrong body", - format: "org.matrix.custom.html", - formatted_body: 'known unknown: \":hippo:\" \":ms_robot_dress:\" and known unknown: \":hipposcope:\" \":ms_robot_cat:\"' - }, - event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "known unknown: <:hippo:230201364309868544> [:ms_robot_dress:](https://bridge.example.org/download/matrix/cadence.moe/wcouHVjbKJJYajkhJLsyeJAA) and known unknown:", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAYAAADuFn/PAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAT/klEQVR4nOVcC3CVRZbuS2KAIMpDQt5PQkIScm/uvYRX" - }) +test("event2message: known and unknown emojis in the end are used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: 'known unknown: \":hippo:\" \":ms_robot_dress:\" and known unknown: \":hipposcope:\" \":ms_robot_cat:\"' + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "known unknown: <:hippo:230201364309868544> [:ms_robot_dress:](https://bridge.example.org/download/matrix/cadence.moe/wcouHVjbKJJYajkhJLsyeJAA) and known unknown: [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FWbYqNlACRuicynBfdnPYtmvc&e=cadence.moe%2FHYcztccFIPgevDvoaWNsEtGJ)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) -slow()("event2message: all unknown chess emojis are reuploaded as a sprite sheet", async t => { - const messages = await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "testing :chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black::chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black:", - format: "org.matrix.custom.html", - formatted_body: "testing \":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"\":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"" - }, - event_id: "$Me6iE8C8CZyrDEOYYrXKSYRuuh_25Jj9kZaNrf7LKr4", - room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" - }, {}, {}, {mxcDownloader: mockGetAndConvertEmoji}) - const testResult = { - content: messages.messagesToSend[0].content, - fileName: messages.messagesToSend[0].pendingFiles[0].name, - fileContentStart: messages.messagesToSend[0].pendingFiles[0].buffer.subarray(0, 90).toString("base64") - } - t.deepEqual(testResult, { - content: "testing", - fileName: "emojis.png", - fileContentStart: "iVBORw0KGgoAAAANSUhEUgAAAYAAAABgCAYAAAAU9KWJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAgAElEQVR4nOx9B3hUVdr/KIpKL2nT0pPpLRNQkdXddV1c" - }) +test("event2message: all unknown chess emojis are used for sprite sheet", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "testing :chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black::chess_good_move::chess_incorrect::chess_blund::chess_brilliant_move::chess_blundest::chess_draw_black:", + format: "org.matrix.custom.html", + formatted_body: "testing \":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"\":chess_good_move:\"\":chess_incorrect:\"\":chess_blund:\"\":chess_brilliant_move:\"\":chess_blundest:\"\":chess_draw_black:\"" + }, + event_id: "$Me6iE8C8CZyrDEOYYrXKSYRuuh_25Jj9kZaNrf7LKr4", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "testing [\u2800](https://bridge.example.org/download/sheet?e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj&e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj)", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }], + ensureJoined: [] + } + ) }) diff --git a/src/matrix/utils.js b/src/matrix/utils.js index d89c968..9e447e7 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -205,6 +205,19 @@ async function getViaServersQuery(roomID, api) { return qs } +function generatePermittedMediaHash(mxc) { + assert(hasher, "xxhash is not ready yet") + const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) + if (!mediaParts) return undefined + + const serverAndMediaID = `${mediaParts[1]}/${mediaParts[2]}` + const unsignedHash = hasher.h64(serverAndMediaID) + const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range + db.prepare("INSERT OR IGNORE INTO media_proxy (permitted_hash) VALUES (?)").run(signedHash) + + return serverAndMediaID +} + /** * Since the introduction of authenticated media, this can no longer just be the /_matrix/media/r0/download URL * because Discord and Discord users cannot use those URLs. Media now has to be proxied through the bridge. @@ -219,6 +232,16 @@ async function getViaServersQuery(roomID, api) { * @returns {string | undefined} */ function getPublicUrlForMxc(mxc) { + const serverAndMediaID = makeMxcPublic(mxc) + if(!serverAndMediaID) return undefined + return `${reg.ooye.bridge_origin}/download/matrix/${serverAndMediaID}` +} + +/** + * @param {string} mxc + * @returns {string | undefined} mxc URL with protocol stripped, e.g. "cadence.moe/abcdef1234" + */ +function makeMxcPublic(mxc) { assert(hasher, "xxhash is not ready yet") const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) if (!mediaParts) return undefined @@ -228,7 +251,7 @@ function getPublicUrlForMxc(mxc) { const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range db.prepare("INSERT OR IGNORE INTO media_proxy (permitted_hash) VALUES (?)").run(signedHash) - return `${reg.ooye.bridge_origin}/download/matrix/${serverAndMediaID}` + return serverAndMediaID } /** @@ -358,6 +381,7 @@ async function setUserPowerCascade(spaceID, mxid, power, api) { module.exports.bot = bot module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord +module.exports.makeMxcPublic = makeMxcPublic module.exports.getPublicUrlForMxc = getPublicUrlForMxc module.exports.getEventIDHash = getEventIDHash module.exports.MatrixStringBuilder = MatrixStringBuilder diff --git a/src/web/routes/download-matrix.js b/src/web/routes/download-matrix.js index 8f790c5..bb6b850 100644 --- a/src/web/routes/download-matrix.js +++ b/src/web/routes/download-matrix.js @@ -1,7 +1,7 @@ // @ts-check const assert = require("assert/strict") -const {defineEventHandler, getValidatedRouterParams, setResponseStatus, setResponseHeader, sendStream, createError, H3Event} = require("h3") +const {defineEventHandler, getValidatedRouterParams, setResponseStatus, setResponseHeader, createError, H3Event, getValidatedQuery} = require("h3") const {z} = require("zod") /** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore @@ -11,10 +11,18 @@ require("xxhash-wasm")().then(h => hasher = h) const {sync, as, select} = require("../../passthrough") +/** @type {import("../../m2d/actions/emoji-sheet")} */ +const emojiSheet = sync.require("../../m2d/actions/emoji-sheet") +/** @type {import("../../m2d/converters/emoji-sheet")} */ +const emojiSheetConverter = sync.require("../../m2d/converters/emoji-sheet") + const schema = { params: z.object({ server_name: z.string(), media_id: z.string() + }), + sheet: z.object({ + e: z.array(z.string()).or(z.string()) }) } @@ -27,10 +35,16 @@ function getAPI(event) { return event.context.api || sync.require("../../matrix/api") } -as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(async event => { - const params = await getValidatedRouterParams(event, schema.params.parse) +/** + * @param {H3Event} event + * @returns {typeof emojiSheet["getAndConvertEmoji"]} + */ +function getMxcDownloader(event) { + /* c8 ignore next */ + return event.context.mxcDownloader || emojiSheet.getAndConvertEmoji +} - const serverAndMediaID = `${params.server_name}/${params.media_id}` +function verifyMediaHash(serverAndMediaID) { const unsignedHash = hasher.h64(serverAndMediaID) const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range @@ -41,7 +55,12 @@ as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(asyn data: `The file you requested isn't permitted by this media proxy.` }) } +} +as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(async event => { + const params = await getValidatedRouterParams(event, schema.params.parse) + + verifyMediaHash(`${params.server_name}/${params.media_id}`) const api = getAPI(event) const res = await api.getMedia(`mxc://${params.server_name}/${params.media_id}`) @@ -53,3 +72,21 @@ as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(asyn setResponseHeader(event, "Transfer-Encoding", "chunked") return res.body })) + +as.router.get(`/download/sheet`, defineEventHandler(async event => { + const query = await getValidatedQuery(event, schema.sheet.parse) + + /** remember that these have no mxc:// protocol in the string for space reasons */ + let mxcs = query.e + if (!Array.isArray(mxcs)) { + mxcs = [mxcs] + } + + for (const serverAndMediaID of mxcs) { + verifyMediaHash(serverAndMediaID) + } + + const buffer = await emojiSheetConverter.compositeMatrixEmojis(mxcs.map(s => `mxc://${s}`), getMxcDownloader(event)) + setResponseHeader(event, "Content-Type", "image/png") + return buffer +})) diff --git a/src/web/routes/download-matrix.test.js b/src/web/routes/download-matrix.test.js index 421d2da..49a6349 100644 --- a/src/web/routes/download-matrix.test.js +++ b/src/web/routes/download-matrix.test.js @@ -1,5 +1,7 @@ // @ts-check +const fs = require("fs") +const {convertImageStream} = require("../../m2d/converters/emoji-sheet") const tryToCatch = require("try-to-catch") const {test} = require("supertape") const {router} = require("../../../test/web") @@ -33,3 +35,52 @@ test("web download matrix: works if a known attachment", async t => { t.equal(event.node.res.statusCode, 200) t.equal(event.node.res.getHeader("content-type"), "image/png") }) + +/** + * MOCK: Gets the emoji from the filesystem and converts to uncompressed PNG data. + * @param {string} mxc a single mxc:// URL + * @returns {Promise} uncompressed PNG data, or undefined if the downloaded emoji is not valid +*/ +async function mockGetAndConvertEmoji(mxc) { + const id = mxc.match(/\/([^./]*)$/)?.[1] + let s + if (fs.existsSync(`test/res/${id}.png`)) { + s = fs.createReadStream(`test/res/${id}.png`) + } else { + s = fs.createReadStream(`test/res/${id}.gif`) + } + return convertImageStream(s, () => { + s.pause() + s.emit("end") + }) +} + +test("web sheet: single emoji", async t => { + const event = {} + const sheet = await router.test("get", "/download/sheet?e=cadence.moe%2FRLMgJGfgTPjIQtvvWZsYjhjy", { + event, + mxcDownloader: mockGetAndConvertEmoji + }) + t.equal(event.node.res.statusCode, 200) + t.equal(sheet.subarray(0, 90).toString("base64"), "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAACXBIWXMAAAPoAAAD6AG1e1JrAAALoklEQVR4nM1ZaVBU2RU+LZSIGnAvFUtcRkSk6abpbkDH") +}) + +test("web sheet: multiple sources", async t => { + const event = {} + const sheet = await router.test("get", "/download/sheet?e=cadence.moe%2FWbYqNlACRuicynBfdnPYtmvc&e=cadence.moe%2FHYcztccFIPgevDvoaWNsEtGJ", { + event, + mxcDownloader: mockGetAndConvertEmoji + }) + t.equal(event.node.res.statusCode, 200) + t.equal(sheet.subarray(0, 90).toString("base64"), "iVBORw0KGgoAAAANSUhEUgAAAGAAAAAwCAYAAADuFn/PAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAT/klEQVR4nOVcC3CVRZbuS2KAIMpDQt5PQkIScm/uvYRX") +}) + +test("web sheet: big sheet", async t => { + const event = {} + const sheet = await router.test("get", "/download/sheet?e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj&e=cadence.moe%2FlHfmJpzgoNyNtYHdAmBHxXix&e=cadence.moe%2FMtRdXixoKjKKOyHJGWLsWLNU&e=cadence.moe%2FHXfFuougamkURPPMflTJRxGc&e=cadence.moe%2FikYKbkhGhMERAuPPbsnQzZiX&e=cadence.moe%2FAYPpqXzVJvZdzMQJGjioIQBZ&e=cadence.moe%2FUVuzvpVUhqjiueMxYXJiFEAj", { + event, + mxcDownloader: mockGetAndConvertEmoji + }) + t.equal(event.node.res.statusCode, 200) + t.equal(sheet.subarray(0, 90).toString("base64"), "iVBORw0KGgoAAAANSUhEUgAAAYAAAABgCAYAAAAU9KWJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAgAElEQVR4nOx9B3hUVdr/KIpKL2nT0pPpLRNQkdXddV1c") +}) diff --git a/test/test.js b/test/test.js index 0bb1da4..e05b687 100644 --- a/test/test.js +++ b/test/test.js @@ -75,47 +75,45 @@ const file = sync.require("../src/matrix/file") file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) } ;(async () => { - /* c8 ignore start - maybe download some more test files in slow mode */ - if (process.argv.includes("--slow")) { - test("test files: download", async t => { - /** @param {{url: string, to: string}[]} files */ - async function allReporter(files) { - return new Promise(resolve => { - let resolved = 0 - const report = files.map(file => file.to.split("/").slice(-1)[0][0]) - files.map(download).forEach((p, i) => { - p.then(() => { - report[i] = green(".") - process.stderr.write("\r" + report.join("")) - if (++resolved === files.length) resolve(null) - }) + /* c8 ignore start - download some more test files in slow mode */ + test("test files: download", async t => { + /** @param {{url: string, to: string}[]} files */ + async function allReporter(files) { + return new Promise(resolve => { + let resolved = 0 + const report = files.map(file => file.to.split("/").slice(-1)[0][0]) + files.map(download).forEach((p, i) => { + p.then(() => { + report[i] = green(".") + process.stderr.write("\r" + report.join("")) + if (++resolved === files.length) resolve(null) }) }) - } - async function download({url, to}) { - if (await fs.existsSync(to)) return - const res = await fetch(url) - // @ts-ignore - await res.body.pipeTo(Writable.toWeb(fs.createWriteStream(to, {encoding: "binary"}))) - } - await allReporter([ - {url: "https://cadence.moe/friends/ooye_test/RLMgJGfgTPjIQtvvWZsYjhjy.png", to: "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png"}, - {url: "https://cadence.moe/friends/ooye_test/bZFuuUSEebJYXUMSxuuSuLTa.png", to: "test/res/bZFuuUSEebJYXUMSxuuSuLTa.png"}, - {url: "https://cadence.moe/friends/ooye_test/qWmbXeRspZRLPcjseyLmeyXC.png", to: "test/res/qWmbXeRspZRLPcjseyLmeyXC.png"}, - {url: "https://cadence.moe/friends/ooye_test/wcouHVjbKJJYajkhJLsyeJAA.png", to: "test/res/wcouHVjbKJJYajkhJLsyeJAA.png"}, - {url: "https://cadence.moe/friends/ooye_test/WbYqNlACRuicynBfdnPYtmvc.gif", to: "test/res/WbYqNlACRuicynBfdnPYtmvc.gif"}, - {url: "https://cadence.moe/friends/ooye_test/HYcztccFIPgevDvoaWNsEtGJ.png", to: "test/res/HYcztccFIPgevDvoaWNsEtGJ.png"}, - {url: "https://cadence.moe/friends/ooye_test/lHfmJpzgoNyNtYHdAmBHxXix.png", to: "test/res/lHfmJpzgoNyNtYHdAmBHxXix.png"}, - {url: "https://cadence.moe/friends/ooye_test/MtRdXixoKjKKOyHJGWLsWLNU.png", to: "test/res/MtRdXixoKjKKOyHJGWLsWLNU.png"}, - {url: "https://cadence.moe/friends/ooye_test/HXfFuougamkURPPMflTJRxGc.png", to: "test/res/HXfFuougamkURPPMflTJRxGc.png"}, - {url: "https://cadence.moe/friends/ooye_test/ikYKbkhGhMERAuPPbsnQzZiX.png", to: "test/res/ikYKbkhGhMERAuPPbsnQzZiX.png"}, - {url: "https://cadence.moe/friends/ooye_test/AYPpqXzVJvZdzMQJGjioIQBZ.png", to: "test/res/AYPpqXzVJvZdzMQJGjioIQBZ.png"}, - {url: "https://cadence.moe/friends/ooye_test/UVuzvpVUhqjiueMxYXJiFEAj.png", to: "test/res/UVuzvpVUhqjiueMxYXJiFEAj.png"}, - {url: "https://ezgif.com/images/format-demo/butterfly.gif", to: "test/res/butterfly.gif"}, - {url: "https://ezgif.com/images/format-demo/butterfly.png", to: "test/res/butterfly.png"}, - ]) - }, {timeout: 60000}) - } + }) + } + async function download({url, to}) { + if (await fs.existsSync(to)) return + const res = await fetch(url) + // @ts-ignore + await res.body.pipeTo(Writable.toWeb(fs.createWriteStream(to, {encoding: "binary"}))) + } + await allReporter([ + {url: "https://cadence.moe/friends/ooye_test/RLMgJGfgTPjIQtvvWZsYjhjy.png", to: "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png"}, + {url: "https://cadence.moe/friends/ooye_test/bZFuuUSEebJYXUMSxuuSuLTa.png", to: "test/res/bZFuuUSEebJYXUMSxuuSuLTa.png"}, + {url: "https://cadence.moe/friends/ooye_test/qWmbXeRspZRLPcjseyLmeyXC.png", to: "test/res/qWmbXeRspZRLPcjseyLmeyXC.png"}, + {url: "https://cadence.moe/friends/ooye_test/wcouHVjbKJJYajkhJLsyeJAA.png", to: "test/res/wcouHVjbKJJYajkhJLsyeJAA.png"}, + {url: "https://cadence.moe/friends/ooye_test/WbYqNlACRuicynBfdnPYtmvc.gif", to: "test/res/WbYqNlACRuicynBfdnPYtmvc.gif"}, + {url: "https://cadence.moe/friends/ooye_test/HYcztccFIPgevDvoaWNsEtGJ.png", to: "test/res/HYcztccFIPgevDvoaWNsEtGJ.png"}, + {url: "https://cadence.moe/friends/ooye_test/lHfmJpzgoNyNtYHdAmBHxXix.png", to: "test/res/lHfmJpzgoNyNtYHdAmBHxXix.png"}, + {url: "https://cadence.moe/friends/ooye_test/MtRdXixoKjKKOyHJGWLsWLNU.png", to: "test/res/MtRdXixoKjKKOyHJGWLsWLNU.png"}, + {url: "https://cadence.moe/friends/ooye_test/HXfFuougamkURPPMflTJRxGc.png", to: "test/res/HXfFuougamkURPPMflTJRxGc.png"}, + {url: "https://cadence.moe/friends/ooye_test/ikYKbkhGhMERAuPPbsnQzZiX.png", to: "test/res/ikYKbkhGhMERAuPPbsnQzZiX.png"}, + {url: "https://cadence.moe/friends/ooye_test/AYPpqXzVJvZdzMQJGjioIQBZ.png", to: "test/res/AYPpqXzVJvZdzMQJGjioIQBZ.png"}, + {url: "https://cadence.moe/friends/ooye_test/UVuzvpVUhqjiueMxYXJiFEAj.png", to: "test/res/UVuzvpVUhqjiueMxYXJiFEAj.png"}, + {url: "https://ezgif.com/images/format-demo/butterfly.gif", to: "test/res/butterfly.gif"}, + {url: "https://ezgif.com/images/format-demo/butterfly.png", to: "test/res/butterfly.png"}, + ]) + }, {timeout: 60000}) /* c8 ignore stop */ const p = migrate.migrate(db) @@ -135,15 +133,6 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("./addbot.test") require("../src/db/orm.test") require("../src/web/server.test") - require("../src/web/routes/download-discord.test") - require("../src/web/routes/download-matrix.test") - require("../src/web/routes/guild.test") - require("../src/web/routes/guild-settings.test") - require("../src/web/routes/info.test") - require("../src/web/routes/link.test") - require("../src/web/routes/log-in-with-matrix.test") - require("../src/web/routes/oauth.test") - require("../src/web/routes/password.test") require("../src/discord/utils.test") require("../src/matrix/kstate.test") require("../src/matrix/api.test") @@ -178,4 +167,13 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../src/discord/interactions/permissions.test") require("../src/discord/interactions/privacy.test") require("../src/discord/interactions/reactions.test") + require("../src/web/routes/download-discord.test") + require("../src/web/routes/download-matrix.test") + require("../src/web/routes/guild.test") + require("../src/web/routes/guild-settings.test") + require("../src/web/routes/info.test") + require("../src/web/routes/link.test") + require("../src/web/routes/log-in-with-matrix.test") + require("../src/web/routes/oauth.test") + require("../src/web/routes/password.test") })() diff --git a/test/web.js b/test/web.js index 463c6b1..250694a 100644 --- a/test/web.js +++ b/test/web.js @@ -51,7 +51,7 @@ class Router { /** * @param {string} method * @param {string} inputUrl - * @param {{event?: any, params?: any, body?: any, sessionData?: any, getOauth2Token?: any, getClient?: (string) => {user: {getGuilds: () => Promise}}, api?: Partial, snow?: {[k in keyof SnowTransfer]?: Partial}, createRoom?: Partial, createSpace?: Partial, headers?: any}} [options] + * @param {{event?: any, params?: any, body?: any, sessionData?: any, getOauth2Token?: any, getClient?: (string) => {user: {getGuilds: () => Promise}}, api?: Partial, snow?: {[k in keyof SnowTransfer]?: Partial}, createRoom?: Partial, createSpace?: Partial, mxcDownloader?: import("../src/m2d/actions/emoji-sheet")["getAndConvertEmoji"], headers?: any}} [options] */ async test(method, inputUrl, options = {}) { const url = new URL(inputUrl, "http://a") @@ -83,6 +83,7 @@ class Router { }, context: { api: options.api, + mxcDownloader: options.mxcDownloader, params: options.params, snow: options.snow, createRoom: options.createRoom,