Refactor kstate resource uploading

This commit is contained in:
Cadence Ember 2024-06-06 12:12:48 +12:00
parent 07a133eba9
commit 24a3b9b0f4
7 changed files with 196 additions and 18 deletions

View file

@ -51,8 +51,8 @@ async function roomToKState(roomID) {
* @param {string} roomID
* @param {any} kstate
*/
function applyKStateDiffToRoom(roomID, kstate) {
const events = ks.kstateToState(kstate)
async function applyKStateDiffToRoom(roomID, kstate) {
const events = await ks.kstateToState(kstate)
return Promise.all(events.map(({type, state_key, content}) =>
api.sendState(roomID, type, state_key, content)
))
@ -220,7 +220,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
invite: [],
initial_state: ks.kstateToState(kstate),
initial_state: await ks.kstateToState(kstate),
...spaceCreationContent
})

View file

@ -45,7 +45,7 @@ async function createSpace(guild, kstate) {
creation_content: {
type: "m.space"
},
initial_state: ks.kstateToState(kstate)
initial_state: await ks.kstateToState(kstate)
})
})
db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, roomID)
@ -57,15 +57,14 @@ async function createSpace(guild, kstate) {
* @param {number} privacyLevel
*/
async function guildToKState(guild, privacyLevel) {
const avatarEventContent = {}
if (guild.icon) {
avatarEventContent.discord_path = file.guildIcon(guild)
avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
}
assert.equal(typeof privacyLevel, "number")
const guildKState = {
"m.room.name/": {name: guild.name},
"m.room.avatar/": avatarEventContent,
"m.room.avatar/": {
$if: guild.icon,
discord_path: file.guildIcon(guild),
url: {$url: file.guildIcon(guild)}
},
"m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
"m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]},
"m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]},
@ -123,7 +122,8 @@ async function _syncSpace(guild, shouldActuallySync) {
// don't try to update rooms with custom avatars though
const roomsWithCustomAvatars = select("channel_room", "room_id", {}, "WHERE custom_avatar IS NOT NULL").pluck().all()
const childRooms = ks.kstateToState(spaceKState).filter(({type, state_key, content}) => {
const state = await ks.kstateToState(spaceKState)
const childRooms = state.filter(({type, state_key, content}) => {
return type === "m.space.child" && "via" in content && !roomsWithCustomAvatars.includes(state_key)
}).map(({state_key}) => state_key)

View file

@ -0,0 +1,39 @@
// @ts-check
const mixin = require("mixin-deep")
const {guildToKState, ensureSpace} = require("./create-space")
const {kstateStripConditionals, kstateUploadMxc} = require("../../matrix/kstate")
const {test} = require("supertape")
const testData = require("../../test/data")
const passthrough = require("../../passthrough")
const {db} = passthrough
test("guild2space: can generate kstate for a guild, passing privacy level 0", async t => {
t.deepEqual(
await kstateUploadMxc(kstateStripConditionals(await guildToKState(testData.guild.general, 0))),
{
"m.room.avatar/": {
discord_path: "/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024",
url: "mxc://cadence.moe/zKXGZhmImMHuGQZWJEFKJbsF"
},
"m.room.guest_access/": {
guest_access: "can_join"
},
"m.room.history_visibility/": {
history_visibility: "invited"
},
"m.room.join_rules/": {
join_rule: "invite"
},
"m.room.name/": {
name: "Psychonauts 3"
},
"m.room.power_levels/": {
users: {
"@test_auto_invite:example.org": 100
},
},
}
)
})

View file

@ -584,7 +584,7 @@ test("event2message: code blocks work", async t => {
msgtype: "m.text",
body: "wrong body",
format: "org.matrix.custom.html",
formatted_body: "<p>preceding</p>\n<pre><code>code block\n</code></pre>\n<p>following <code>code</code> is inline</p>\n"
formatted_body: "<p>preceding</p>\n<pre><code>code block\n</code></pre>\n<p>following <code>code</code> is inline</p>"
},
event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
origin_server_ts: 1688301929913,
@ -641,6 +641,66 @@ test("event2message: code block contents are formatted correctly and not escaped
)
})
test("event2message: code blocks use double backtick as delimiter when necessary", async t => {
t.deepEqual(
await eventToMessage({
type: "m.room.message",
sender: "@cadence:cadence.moe",
content: {
msgtype: "m.text",
body: "wrong body",
format: "org.matrix.custom.html",
formatted_body: "<code>backtick in ` the middle</code>, <code>backtick at the edge`</code>"
},
event_id: "$pGkWQuGVmrPNByrFELxhzI6MCBgJecr5I2J3z88Gc2s",
room_id: "!BpMdOUkWWhFxmTrENV:cadence.moe"
}),
{
ensureJoined: [],
messagesToDelete: [],
messagesToEdit: [],
messagesToSend: [{
username: "cadence [they]",
content: "``backtick in ` the middle``, `` backtick at the edge` ``",
avatar_url: undefined,
allowed_mentions: {
parse: ["users", "roles"]
}
}]
}
)
})
test("event2message: inline code is converted to code block if it contains both delimiters", async t => {
t.deepEqual(
await eventToMessage({
type: "m.room.message",
sender: "@cadence:cadence.moe",
content: {
msgtype: "m.text",
body: "wrong body",
format: "org.matrix.custom.html",
formatted_body: "<code>` one two ``</code>"
},
event_id: "$pGkWQuGVmrPNByrFELxhzI6MCBgJecr5I2J3z88Gc2s",
room_id: "!BpMdOUkWWhFxmTrENV:cadence.moe"
}),
{
ensureJoined: [],
messagesToDelete: [],
messagesToEdit: [],
messagesToSend: [{
username: "cadence [they]",
content: "``` ` one two `` ```",
avatar_url: undefined,
allowed_mentions: {
parse: ["users", "roles"]
}
}]
}
)
})
test("event2message: code blocks are uploaded as attachments instead if they contain incompatible backticks", async t => {
t.deepEqual(
await eventToMessage({

View file

@ -4,7 +4,12 @@ const assert = require("assert").strict
const mixin = require("mixin-deep")
const {isDeepStrictEqual} = require("util")
/** Mutates the input. */
const passthrough = require("../passthrough")
const {sync} = passthrough
/** @type {import("./file")} */
const file = sync.require("./file")
/** Mutates the input. Not recursive - can only include or exclude entire state events. */
function kstateStripConditionals(kstate) {
for (const [k, content] of Object.entries(kstate)) {
// conditional for whether a key is even part of the kstate (doing this declaratively on json is hard, so represent it as a property instead.)
@ -16,9 +21,33 @@ function kstateStripConditionals(kstate) {
return kstate
}
function kstateToState(kstate) {
/** Mutates the input. Works recursively through object tree. */
async function kstateUploadMxc(obj) {
const promises = []
function inner(obj) {
for (const [k, v] of Object.entries(obj)) {
if (v == null || typeof v !== "object") continue
if (v.$url) {
promises.push(
file.uploadDiscordFileToMxc(v.$url)
.then(mxc => obj[k] = mxc)
)
}
inner(v)
}
}
inner(obj)
await Promise.all(promises)
return obj
}
/** Automatically strips conditionals and uploads URLs to mxc. */
async function kstateToState(kstate) {
const events = []
kstateStripConditionals(kstate)
await kstateUploadMxc(kstate)
for (const [k, content] of Object.entries(kstate)) {
const slashIndex = k.indexOf("/")
assert(slashIndex > 0)
@ -74,6 +103,7 @@ function diffKState(actual, target) {
}
module.exports.kstateStripConditionals = kstateStripConditionals
module.exports.kstateUploadMxc = kstateUploadMxc
module.exports.kstateToState = kstateToState
module.exports.stateToKState = stateToKState
module.exports.diffKState = diffKState

View file

@ -1,5 +1,5 @@
const assert = require("assert")
const {kstateToState, stateToKState, diffKState, kstateStripConditionals} = require("./kstate")
const {kstateToState, stateToKState, diffKState, kstateStripConditionals, kstateUploadMxc} = require("./kstate")
const {test} = require("supertape")
test("kstate strip: strips false conditions", t => {
@ -21,8 +21,53 @@ test("kstate strip: keeps true conditions while removing $if", t => {
})
})
test("kstate2state: general", t => {
t.deepEqual(kstateToState({
test("kstateUploadMxc: sets the mxc", async t => {
const input = {
"m.room.avatar/": {
url: {$url: "https://cdn.discordapp.com/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"},
test1: {
test2: {
test3: {$url: "https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg"}
}
}
}
}
await kstateUploadMxc(input)
t.deepEqual(input, {
"m.room.avatar/": {
url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl",
test1: {
test2: {
test3: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR"
}
}
}
})
})
test("kstateUploadMxc and strip: work together", async t => {
const input = {
"m.room.avatar/yes": {
$if: true,
url: {$url: "https://cdn.discordapp.com/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"}
},
"m.room.avatar/no": {
$if: false,
url: {$url: "https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024"}
},
}
kstateStripConditionals(input)
await kstateUploadMxc(input)
t.deepEqual(input, {
"m.room.avatar/yes": {
url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl"
}
})
})
test("kstate2state: general", async t => {
t.deepEqual(await kstateToState({
"m.room.name/": {name: "test name"},
"m.room.member/@cadence:cadence.moe": {membership: "join"},
"uk.half-shot.bridge/org.matrix.appservice-irc://irc/epicord.net/#general": {creator: "@cadence:cadence.moe"}

View file

@ -21,6 +21,9 @@ const reg = require("../matrix/read-registration")
reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded
reg.ooye.server_name = "cadence.moe"
reg.ooye.invite = ["@test_auto_invite:example.org"]
reg.id = "baby" // don't actually take authenticated actions on the server
reg.as_token = "baby"
reg.hs_token = "baby"
const sync = new HeatSync({watchFS: false})
@ -117,6 +120,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
require("../matrix/read-registration.test")
require("../matrix/txnid.test")
require("../d2m/actions/create-room.test")
require("../d2m/actions/create-space.test")
require("../d2m/actions/register-user.test")
require("../d2m/converters/edit-to-changes.test")
require("../d2m/converters/emoji-to-key.test")