From 24a3b9b0f49d5f252ee81104fb884512ebaca42e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 6 Jun 2024 12:12:48 +1200 Subject: [PATCH] Refactor kstate resource uploading --- d2m/actions/create-room.js | 6 +-- d2m/actions/create-space.js | 18 +++---- d2m/actions/create-space.test.js | 39 ++++++++++++++++ m2d/converters/event-to-message.test.js | 62 ++++++++++++++++++++++++- matrix/kstate.js | 34 +++++++++++++- matrix/kstate.test.js | 51 ++++++++++++++++++-- test/test.js | 4 ++ 7 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 d2m/actions/create-space.test.js diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js index 4559ffa..051e889 100644 --- a/d2m/actions/create-room.js +++ b/d2m/actions/create-room.js @@ -51,8 +51,8 @@ async function roomToKState(roomID) { * @param {string} roomID * @param {any} kstate */ -function applyKStateDiffToRoom(roomID, kstate) { - const events = ks.kstateToState(kstate) +async function applyKStateDiffToRoom(roomID, kstate) { + const events = await ks.kstateToState(kstate) return Promise.all(events.map(({type, state_key, content}) => api.sendState(roomID, type, state_key, content) )) @@ -220,7 +220,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel], invite: [], - initial_state: ks.kstateToState(kstate), + initial_state: await ks.kstateToState(kstate), ...spaceCreationContent }) diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index 3bad2a1..dbefd6d 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -45,7 +45,7 @@ async function createSpace(guild, kstate) { creation_content: { type: "m.space" }, - initial_state: ks.kstateToState(kstate) + initial_state: await ks.kstateToState(kstate) }) }) db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, roomID) @@ -57,15 +57,14 @@ async function createSpace(guild, kstate) { * @param {number} privacyLevel */ async function guildToKState(guild, privacyLevel) { - const avatarEventContent = {} - if (guild.icon) { - avatarEventContent.discord_path = file.guildIcon(guild) - avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API - } - + assert.equal(typeof privacyLevel, "number") const guildKState = { "m.room.name/": {name: guild.name}, - "m.room.avatar/": avatarEventContent, + "m.room.avatar/": { + $if: guild.icon, + discord_path: file.guildIcon(guild), + url: {$url: file.guildIcon(guild)} + }, "m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]}, "m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]}, "m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]}, @@ -123,7 +122,8 @@ async function _syncSpace(guild, shouldActuallySync) { // don't try to update rooms with custom avatars though const roomsWithCustomAvatars = select("channel_room", "room_id", {}, "WHERE custom_avatar IS NOT NULL").pluck().all() - const childRooms = ks.kstateToState(spaceKState).filter(({type, state_key, content}) => { + const state = await ks.kstateToState(spaceKState) + const childRooms = state.filter(({type, state_key, content}) => { return type === "m.space.child" && "via" in content && !roomsWithCustomAvatars.includes(state_key) }).map(({state_key}) => state_key) diff --git a/d2m/actions/create-space.test.js b/d2m/actions/create-space.test.js new file mode 100644 index 0000000..368bb32 --- /dev/null +++ b/d2m/actions/create-space.test.js @@ -0,0 +1,39 @@ +// @ts-check + +const mixin = require("mixin-deep") +const {guildToKState, ensureSpace} = require("./create-space") +const {kstateStripConditionals, kstateUploadMxc} = require("../../matrix/kstate") +const {test} = require("supertape") +const testData = require("../../test/data") + +const passthrough = require("../../passthrough") +const {db} = passthrough + +test("guild2space: can generate kstate for a guild, passing privacy level 0", async t => { + t.deepEqual( + await kstateUploadMxc(kstateStripConditionals(await guildToKState(testData.guild.general, 0))), + { + "m.room.avatar/": { + discord_path: "/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024", + url: "mxc://cadence.moe/zKXGZhmImMHuGQZWJEFKJbsF" + }, + "m.room.guest_access/": { + guest_access: "can_join" + }, + "m.room.history_visibility/": { + history_visibility: "invited" + }, + "m.room.join_rules/": { + join_rule: "invite" + }, + "m.room.name/": { + name: "Psychonauts 3" + }, + "m.room.power_levels/": { + users: { + "@test_auto_invite:example.org": 100 + }, + }, + } + ) +}) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index db44393..6d750a5 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -584,7 +584,7 @@ test("event2message: code blocks work", async t => { msgtype: "m.text", body: "wrong body", format: "org.matrix.custom.html", - formatted_body: "

preceding

\n
code block\n
\n

following code is inline

\n" + formatted_body: "

preceding

\n
code block\n
\n

following code is inline

" }, event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", origin_server_ts: 1688301929913, @@ -641,6 +641,66 @@ test("event2message: code block contents are formatted correctly and not escaped ) }) +test("event2message: code blocks use double backtick as delimiter when necessary", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: "backtick in ` the middle, backtick at the edge`" + }, + event_id: "$pGkWQuGVmrPNByrFELxhzI6MCBgJecr5I2J3z88Gc2s", + room_id: "!BpMdOUkWWhFxmTrENV:cadence.moe" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "``backtick in ` the middle``, `` backtick at the edge` ``", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + +test("event2message: inline code is converted to code block if it contains both delimiters", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: "` one two ``" + }, + event_id: "$pGkWQuGVmrPNByrFELxhzI6MCBgJecr5I2J3z88Gc2s", + room_id: "!BpMdOUkWWhFxmTrENV:cadence.moe" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "``` ` one two `` ```", + avatar_url: undefined, + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + test("event2message: code blocks are uploaded as attachments instead if they contain incompatible backticks", async t => { t.deepEqual( await eventToMessage({ diff --git a/matrix/kstate.js b/matrix/kstate.js index 3c4189d..eca9522 100644 --- a/matrix/kstate.js +++ b/matrix/kstate.js @@ -4,7 +4,12 @@ const assert = require("assert").strict const mixin = require("mixin-deep") const {isDeepStrictEqual} = require("util") -/** Mutates the input. */ +const passthrough = require("../passthrough") +const {sync} = passthrough +/** @type {import("./file")} */ +const file = sync.require("./file") + +/** Mutates the input. Not recursive - can only include or exclude entire state events. */ function kstateStripConditionals(kstate) { for (const [k, content] of Object.entries(kstate)) { // conditional for whether a key is even part of the kstate (doing this declaratively on json is hard, so represent it as a property instead.) @@ -16,9 +21,33 @@ function kstateStripConditionals(kstate) { return kstate } -function kstateToState(kstate) { +/** Mutates the input. Works recursively through object tree. */ +async function kstateUploadMxc(obj) { + const promises = [] + function inner(obj) { + for (const [k, v] of Object.entries(obj)) { + if (v == null || typeof v !== "object") continue + + if (v.$url) { + promises.push( + file.uploadDiscordFileToMxc(v.$url) + .then(mxc => obj[k] = mxc) + ) + } + + inner(v) + } + } + inner(obj) + await Promise.all(promises) + return obj +} + +/** Automatically strips conditionals and uploads URLs to mxc. */ +async function kstateToState(kstate) { const events = [] kstateStripConditionals(kstate) + await kstateUploadMxc(kstate) for (const [k, content] of Object.entries(kstate)) { const slashIndex = k.indexOf("/") assert(slashIndex > 0) @@ -74,6 +103,7 @@ function diffKState(actual, target) { } module.exports.kstateStripConditionals = kstateStripConditionals +module.exports.kstateUploadMxc = kstateUploadMxc module.exports.kstateToState = kstateToState module.exports.stateToKState = stateToKState module.exports.diffKState = diffKState diff --git a/matrix/kstate.test.js b/matrix/kstate.test.js index 239de75..0538450 100644 --- a/matrix/kstate.test.js +++ b/matrix/kstate.test.js @@ -1,5 +1,5 @@ const assert = require("assert") -const {kstateToState, stateToKState, diffKState, kstateStripConditionals} = require("./kstate") +const {kstateToState, stateToKState, diffKState, kstateStripConditionals, kstateUploadMxc} = require("./kstate") const {test} = require("supertape") test("kstate strip: strips false conditions", t => { @@ -21,8 +21,53 @@ test("kstate strip: keeps true conditions while removing $if", t => { }) }) -test("kstate2state: general", t => { - t.deepEqual(kstateToState({ +test("kstateUploadMxc: sets the mxc", async t => { + const input = { + "m.room.avatar/": { + url: {$url: "https://cdn.discordapp.com/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"}, + test1: { + test2: { + test3: {$url: "https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg"} + } + } + } + } + await kstateUploadMxc(input) + t.deepEqual(input, { + "m.room.avatar/": { + url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl", + test1: { + test2: { + test3: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR" + } + } + } + }) +}) + +test("kstateUploadMxc and strip: work together", async t => { + const input = { + "m.room.avatar/yes": { + $if: true, + url: {$url: "https://cdn.discordapp.com/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"} + }, + "m.room.avatar/no": { + $if: false, + url: {$url: "https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024"} + }, + } + kstateStripConditionals(input) + await kstateUploadMxc(input) + t.deepEqual(input, { + "m.room.avatar/yes": { + url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl" + } + }) +}) + + +test("kstate2state: general", async t => { + t.deepEqual(await kstateToState({ "m.room.name/": {name: "test name"}, "m.room.member/@cadence:cadence.moe": {membership: "join"}, "uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"} diff --git a/test/test.js b/test/test.js index 58459f6..396dea9 100644 --- a/test/test.js +++ b/test/test.js @@ -21,6 +21,9 @@ const reg = require("../matrix/read-registration") reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded reg.ooye.server_name = "cadence.moe" reg.ooye.invite = ["@test_auto_invite:example.org"] +reg.id = "baby" // don't actually take authenticated actions on the server +reg.as_token = "baby" +reg.hs_token = "baby" const sync = new HeatSync({watchFS: false}) @@ -117,6 +120,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../matrix/read-registration.test") require("../matrix/txnid.test") require("../d2m/actions/create-room.test") + require("../d2m/actions/create-space.test") require("../d2m/actions/register-user.test") require("../d2m/converters/edit-to-changes.test") require("../d2m/converters/emoji-to-key.test")