Finish room diffing and syncing. All tests pass

d2m/actions/create-room.js

@@ -1,8 +1,6 @@
 // @ts-check
 
 const assert = require("assert").strict
-const {test} = require("supertape")
-const testData = require("../../test/data")
 const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
@@ -12,37 +10,62 @@ const mreq = sync.require("../../matrix/mreq")
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
 
+function kstateStripConditionals(kstate) {
+	for (const [k, content] of Object.entries(kstate)) {
+		if ("$if" in content) {
+			if (content.$if) delete content.$if
+			else delete kstate[k]
+		}
+	}
+	return kstate
+}
+
 function kstateToState(kstate) {
-	return Object.entries(kstate).map(([k, content]) => {
-		console.log(k)
+	const events = []
+	for (const [k, content] of Object.entries(kstate)) {
+		// conditional for whether a key is even part of the kstate (doing this declaratively on json is hard, so represent it as a property instead.)
+		if ("$if" in content && !content.$if) continue
+		delete content.$if
+
 		const [type, state_key] = k.split("/")
 		assert.ok(typeof type === "string")
 		assert.ok(typeof state_key === "string")
-		return {type, state_key, content}
-	})
+		events.push({type, state_key, content})
+	}
+	return events
 }
 
-test("kstate2state: general", t => {
-	t.deepEqual(kstateToState({
-		"m.room.name/": {name: "test name"},
-		"m.room.member/@cadence:cadence.moe": {membership: "join"}
-	}), [
-		{
-			type: "m.room.name",
-			state_key: "",
-			content: {
-				name: "test name"
-			}
-		},
-		{
-			type: "m.room.member",
-			state_key: "@cadence:cadence.moe",
-			content: {
-				membership: "join"
-			}
-		}
-	])
-})
+/**
+ * @param {import("../../types").Event.BaseStateEvent[]} events
+ * @returns {any}
+ */
+function stateToKState(events) {
+	const kstate = {}
+	for (const event of events) {
+		kstate[event.type + "/" + event.state_key] = event.content
+	}
+	return kstate
+}
+
+/**
+ * @param {string} roomID
+ */
+async function roomToKState(roomID) {
+	/** @type {import("../../types").Event.BaseStateEvent[]} */
+	const root = await mreq.mreq("GET", `/client/v3/rooms/${roomID}/state`)
+	return stateToKState(root)
+}
+
+/**
+ * @params {string} roomID
+ * @params {any} kstate
+ */
+function applyKStateDiffToRoom(roomID, kstate) {
+	const events = kstateToState(kstate)
+	return Promise.all(events.map(({type, state_key, content}) =>
+		mreq.mreq("PUT", `/client/v3/rooms/${roomID}/state/${type}/${state_key}`, content)
+	))
+}
 
 function diffKState(actual, target) {
 	const diff = {}
@@ -60,39 +83,11 @@ function diffKState(actual, target) {
 			// not present, needs to be added
 			diff[key] = target[key]
 		}
+		// keys that are missing in "actual" will not be deleted on "target" (no action)
 	}
 	return diff
 }
 
-test("diffKState: detects edits", t => {
-	t.deepEqual(
-		diffKState({
-			"m.room.name/": {name: "test name"},
-			"same/": {a: 2}
-		}, {
-			"m.room.name/": {name: "edited name"},
-			"same/": {a: 2}
-		}),
-		{
-			"m.room.name/": {name: "edited name"}
-		}
-	)
-})
-
-test("diffKState: detects new properties", t => {
-	t.deepEqual(
-		diffKState({
-			"m.room.name/": {name: "test name"},
-		}, {
-			"m.room.name/": {name: "test name"},
-			"new/": {a: 2}
-		}),
-		{
-			"new/": {a: 2}
-		}
-	)
-})
-
 /**
  * @param {import("discord-api-types/v10").APIGuildTextChannel} channel
  * @param {import("discord-api-types/v10").APIGuild} guild
@@ -107,14 +102,14 @@ async function channelToKState(channel, guild) {
 		avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path)
 	}
 
-	const kstate = {
+	const channelKState = {
 		"m.room.name/": {name: channel.name},
-		"m.room.topic/": {topic: channel.topic || undefined},
+		"m.room.topic/": {$if: channel.topic, topic: channel.topic},
 		"m.room.avatar/": avatarEventContent,
 		"m.room.guest_access/": {guest_access: "can_join"},
 		"m.room.history_visibility/": {history_visibility: "invited"},
-		[`m.space.parent/${spaceID}`]: { // TODO: put the proper server here
-			via: ["cadence.moe"],
+		[`m.space.parent/${spaceID}`]: {
+			via: ["cadence.moe"], // TODO: put the proper server here
 			canonical: true
 		},
 		"m.room.join_rules/": {
@@ -126,13 +121,9 @@ async function channelToKState(channel, guild) {
 		}
 	}
 
-	return {spaceID, kstate}
+	return {spaceID, channelKState}
 }
 
-test("channel2room: general", async t => {
-	t.deepEqual(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.kstate), {expected: true, ...testData.room.general})
-})
-
 /**
  * @param {import("discord-api-types/v10").APIGuildTextChannel} channel
  * @param guild
@@ -140,7 +131,7 @@ test("channel2room: general", async t => {
  * @param {any} kstate
  */
 async function createRoom(channel, guild, spaceID, kstate) {
-	/** @type {import("../../types").R_RoomCreated} */
+	/** @type {import("../../types").R.RoomCreated} */
 	const root = await mreq.mreq("POST", "/client/v3/createRoom", {
 		name: channel.name,
 		topic: channel.topic || undefined,
@@ -159,20 +150,46 @@ async function createRoom(channel, guild, spaceID, kstate) {
 }
 
 /**
- * @param {import("discord-api-types/v10").APIGuildTextChannel} channel
+ * @param {import("discord-api-types/v10").APIGuildChannel} channel
  */
-async function syncRoom(channel) {
+function channelToGuild(channel) {
 	const guildID = channel.guild_id
 	assert(guildID)
 	const guild = discord.guilds.get(guildID)
 	assert(guild)
+	return guild
+}
 
-	const {spaceID, kstate} = await channelToKState(channel, guild)
+/**
+ * @param {string} channelID
+ */
+async function syncRoom(channelID) {
+	/** @ts-ignore @type {import("discord-api-types/v10").APIGuildChannel} */
+	const channel = discord.channels.get(channelID)
+	assert.ok(channel)
+	const guild = channelToGuild(channel)
+
+	const {spaceID, channelKState} = await channelToKState(channel, guild)
 
 	/** @type {string?} */
 	const existing = db.prepare("SELECT room_id from channel_room WHERE channel_id = ?").pluck().get(channel.id)
 	if (!existing) {
-		createRoom(channel, guild, spaceID, kstate)
+		return createRoom(channel, guild, spaceID, channelKState)
+	} else {
+		// sync channel state to room
+		const roomKState = await roomToKState(existing)
+		const roomDiff = diffKState(roomKState, channelKState)
+		const roomApply = applyKStateDiffToRoom(existing, roomDiff)
+
+		// sync room as space member
+		const spaceKState = await roomToKState(spaceID)
+		const spaceDiff = diffKState(spaceKState, {
+			[`m.space.child/${existing}`]: {
+				via: ["cadence.moe"] // TODO: use the proper server
+			}
+		})
+		const spaceApply = applyKStateDiffToRoom(spaceID, spaceDiff)
+		return Promise.all([roomApply, spaceApply])
 	}
 }
 
@@ -180,14 +197,15 @@ async function createAllForGuild(guildID) {
 	const channelIDs = discord.guildChannelMap.get(guildID)
 	assert.ok(channelIDs)
 	for (const channelID of channelIDs) {
-		const channel = discord.channels.get(channelID)
-		assert.ok(channel)
-		const existing = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(channel.id)
-		if (channel.type === DiscordTypes.ChannelType.GuildText && !existing) {
-			await createRoom(channel)
-		}
+		await syncRoom(channelID).then(r => console.log(`synced ${channelID}:`, r))
 	}
 }
 
 module.exports.createRoom = createRoom
+module.exports.syncRoom = syncRoom
 module.exports.createAllForGuild = createAllForGuild
+module.exports.kstateToState = kstateToState
+module.exports.stateToKState = stateToKState
+module.exports.diffKState = diffKState
+module.exports.channelToKState = channelToKState
+module.exports.kstateStripConditionals = kstateStripConditionals

d2m/actions/create-room.test.js

@@ -0,0 +1,83 @@
+const {kstateToState, stateToKState, diffKState, channelToKState, kstateStripConditionals} = require("./create-room")
+const {test} = require("supertape")
+const testData = require("../../test/data")
+
+test("kstate2state: general", t => {
+	t.deepEqual(kstateToState({
+		"m.room.name/": {name: "test name"},
+		"m.room.member/@cadence:cadence.moe": {membership: "join"}
+	}), [
+		{
+			type: "m.room.name",
+			state_key: "",
+			content: {
+				name: "test name"
+			}
+		},
+		{
+			type: "m.room.member",
+			state_key: "@cadence:cadence.moe",
+			content: {
+				membership: "join"
+			}
+		}
+	])
+})
+
+test("state2kstate: general", t => {
+	t.deepEqual(stateToKState([
+		{
+			type: "m.room.name",
+			state_key: "",
+			content: {
+				name: "test name"
+			}
+		},
+		{
+			type: "m.room.member",
+			state_key: "@cadence:cadence.moe",
+			content: {
+				membership: "join"
+			}
+		}
+	]), {
+		"m.room.name/": {name: "test name"},
+		"m.room.member/@cadence:cadence.moe": {membership: "join"}
+	})
+})
+
+test("diffKState: detects edits", t => {
+	t.deepEqual(
+		diffKState({
+			"m.room.name/": {name: "test name"},
+			"same/": {a: 2}
+		}, {
+			"m.room.name/": {name: "edited name"},
+			"same/": {a: 2}
+		}),
+		{
+			"m.room.name/": {name: "edited name"}
+		}
+	)
+})
+
+test("diffKState: detects new properties", t => {
+	t.deepEqual(
+		diffKState({
+			"m.room.name/": {name: "test name"},
+		}, {
+			"m.room.name/": {name: "test name"},
+			"new/": {a: 2}
+		}),
+		{
+			"new/": {a: 2}
+		}
+	)
+})
+
+test("channel2room: general", async t => {
+	t.deepEqual(
+		kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
+		testData.room.general
+	)
+})

d2m/actions/create-space.js

@@ -37,7 +37,7 @@ function createSpace(guild) {
 				}
 			}
 		]
-	}).then(/** @param {import("../../types").R_RoomCreated} root */ root => {
+	}).then(/** @param {import("../../types").R.RoomCreated} root */ root => {
 		db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, root.room_id)
 		return root
 	})

d2m/actions/register-user.js

@@ -1,7 +1,7 @@
 // @ts-check
 
 const reg = require("../../matrix/read-registration.js")
-const fetch = require("node-fetch")
+const fetch = require("node-fetch").default
 
 fetch("https://matrix.cadence.moe/_matrix/client/v3/register", {
 	method: "POST",

index.js

@@ -22,6 +22,3 @@ passthrough.discord = discord
 
 	require("./stdin")
 })()
-
-// process.on("unhandledRejection", console.error)
-// process.on("uncaughtException", console.error)

matrix/file.js

@@ -36,7 +36,7 @@ async function uploadDiscordFileToMxc(path) {
 		const body = res.body
 
 		// Upload to Matrix
-		/** @type {import("../types").R_FileUploaded} */
+		/** @type {import("../types").R.FileUploaded} */
 		const root = await mreq.mreq("POST", "/media/v3/upload", body, {
 			headers: {
 				"Content-Type": res.headers.get("content-type")

matrix/mreq.js

@@ -10,8 +10,9 @@ const reg = sync.require("./read-registration.js")
 
 const baseUrl = "https://matrix.cadence.moe/_matrix"
 
-class MatrixServerError {
+class MatrixServerError extends Error {
 	constructor(data) {
+		super(data.error || data.errcode)
 		this.data = data
 		/** @type {string} */
 		this.errcode = data.errcode
@@ -35,7 +36,7 @@ async function mreq(method, url, body, extra = {}) {
 		}
 	}, extra)
 
-	console.log(baseUrl + url, opts)
+	// console.log(baseUrl + url, opts)
 	const res = await fetch(baseUrl + url, opts)
 	const root = await res.json()
 

package.json

@@ -24,13 +24,15 @@
     "matrix-js-sdk": "^24.1.0",
     "mixin-deep": "^2.0.1",
     "node-fetch": "^2.6.7",
-    "snowtransfer": "^0.7.0",
-    "supertape": "^8.3.0"
+    "snowtransfer": "^0.7.0"
   },
   "devDependencies": {
-    "@types/node": "^18.16.0"
+    "@types/node": "^18.16.0",
+    "@types/node-fetch": "^2.6.3",
+    "supertape": "^8.3.0",
+    "tap-dot": "github:cloudrac3r/tap-dot#223a4e67a6f7daf015506a12a7af74605f06c7f4"
   },
   "scripts": {
-    "test": "supertape --format short test/test.js"
+    "test": "FORCE_COLOR=true supertape --format tap test/test.js | tap-dot"
   }
 }

stdin.js

@@ -43,7 +43,7 @@ async function customEval(input, _context, _filename, callback) {
 		const output = util.inspect(result, false, depth, true)
 		return callback(null, output)
 	} catch (e) {
-		return callback(null, util.inspect(e, true, 100, true))
+		return callback(null, util.inspect(e, false, 100, true))
 	}
 }
 

test/data.js

@@ -39,7 +39,7 @@ module.exports = {
 			},
 			"m.room.avatar/": {
 				discord_path: "/icons/112760669178241024/a_f83622e09ead74f0c5c527fe241f8f8c.png?size=1024",
-				url: "mxc://cadence.moe/sZtPwbfOIsvfSoWCWPrGnzql"
+				url: "mxc://cadence.moe/zKXGZhmImMHuGQZWJEFKJbsF"
 			}
 		}
 	},

test/test.js

@@ -12,4 +12,4 @@ const sync = new HeatSync({persistent: false})
 
 Object.assign(passthrough, { config, sync, db })
 
-require("../d2m/actions/create-room")
+require("../d2m/actions/create-room.test")

types.d.ts

@@ -8,17 +8,42 @@ export type AppServiceRegistrationConfig = {
 	rate_limited: boolean
 }
 
-export type M_Room_Message_content = {
-	msgtype: "m.text"
-	body: string
-	formatted_body?: "org.matrix.custom.html"
-	format?: string
+namespace Event {
+	export type BaseStateEvent = {
+		type: string
+		room_id: string
+		sender: string
+		content: any
+		state_key: string
+		origin_server_ts: number
+		unsigned: any
+		event_id: string
+		user_id: string
+		age: number
+		replaces_state: string
+		prev_content?: any
+	}
+
+	export type M_Room_Message = {
+		msgtype: "m.text"
+		body: string
+		formatted_body?: "org.matrix.custom.html"
+		format?: string
+	}
+
+	export type M_Room_Member = {
+		membership: string
+		display_name?: string
+		avatar_url?: string
+	}
 }
 
-export type R_RoomCreated = {
-	room_id: string
-}
+namespace R {
+	export type RoomCreated = {
+		room_id: string
+	}
 
-export type R_FileUploaded = {
-	content_uri: string
+	export type FileUploaded = {
+		content_uri: string
+	}
 }