d->m: Make PK members appear real

d2m/actions/announce-thread.js

@@ -8,6 +8,8 @@ const {discord, sync, db, select} = passthrough
 const threadToAnnouncement = sync.require("../converters/thread-to-announcement")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
+/** @type {import("./register-user")} */
+const registerUser = sync.require("./register-user")
 
 /**
  * @param {string} parentRoomID
@@ -15,10 +17,10 @@ const api = sync.require("../../matrix/api")
  * @param {import("discord-api-types/v10").APIThreadChannel} thread
  */
 async function announceThread(parentRoomID, threadRoomID, thread) {
-	const creatorMxid = select("sim", "mxid", {user_id: thread.owner_id}).pluck().get()
+	assert(thread.owner_id)
-
+	// @ts-ignore
+	const creatorMxid = await registerUser.ensureSimJoined({id: thread.owner_id}, parentRoomID)
 	const content = await threadToAnnouncement.threadToAnnouncement(parentRoomID, threadRoomID, creatorMxid, thread, {api})
-
 	await api.sendEvent(parentRoomID, "m.room.message", content, creatorMxid)
 }
 

d2m/actions/delete-message.js

@@ -4,21 +4,25 @@ const passthrough = require("../../passthrough")
 const {sync, db, select, from} = passthrough
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
+/** @type {import("./speedbump")} */
+const speedbump = sync.require("./speedbump")
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data
  */
 async function deleteMessage(data) {
-	const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
+	const row = select("channel_room", ["room_id", "speedbump_id", "speedbump_checked"], {channel_id: data.channel_id}).get()
-	if (!roomID) return
+	if (!row) return
 
 	const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all()
 	db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id)
 	db.prepare("DELETE FROM event_message WHERE message_id = ?").run(data.id)
 	for (const eventID of eventsToRedact) {
 		// Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs
-		await api.redactEvent(roomID, eventID)
+		await api.redactEvent(row.room_id, eventID)
 	}
+
+	speedbump.updateCache(data.channel_id, row.speedbump_id, row.speedbump_checked)
 }
 
 /**

d2m/actions/edit-message.js

@@ -1,18 +1,32 @@
 // @ts-check
 
+const assert = require("assert").strict
+
 const passthrough = require("../../passthrough")
 const {sync, db, select} = passthrough
 /** @type {import("../converters/edit-to-changes")} */
 const editToChanges = sync.require("../converters/edit-to-changes")
+/** @type {import("./register-pk-user")} */
+const registerPkUser = sync.require("./register-pk-user")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 
 /**
  * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
  * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
-async function editMessage(message, guild) {
+async function editMessage(message, guild, row) {
-	const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api)
+	let {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api)
+
+	if (row && row.speedbump_webhook_id === message.webhook_id) {
+		// Handle the PluralKit public instance
+		if (row.speedbump_id === "466378653216014359") {
+			const root = await registerPkUser.fetchMessage(message.id)
+			assert(root.member)
+			senderMxid = await registerPkUser.ensureSimJoined(root.member, roomID)
+		}
+	}
 
 	// 1. Replace all the things.
 	for (const {oldID, newContent} of eventsToReplace) {

d2m/actions/register-pk-user.js

@@ -0,0 +1,139 @@
+// @ts-check
+
+const assert = require("assert")
+const reg = require("../../matrix/read-registration")
+const Ty = require("../../types")
+const fetch = require("node-fetch").default
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select} = passthrough
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("./register-user")} */
+const registerUser = sync.require("./register-user")
+
+/**
+ * A sim is an account that is being simulated by the bridge to copy events from the other side.
+ * @param {Ty.PkMember} member
+ * @returns mxid
+ */
+async function createSim(member) {
+	// Choose sim name
+	const simName = "_pk_" + member.id
+	const localpart = reg.ooye.namespace_prefix + simName
+	const mxid = `@${localpart}:${reg.ooye.server_name}`
+
+	// Save chosen name in the database forever
+	db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(member.uuid, simName, localpart, mxid)
+
+	// Register matrix user with that name
+	try {
+		await api.register(localpart)
+	} catch (e) {
+		// If user creation fails, manually undo the database change. Still isn't perfect, but should help.
+		// (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.)
+		db.prepare("DELETE FROM sim WHERE user_id = ?").run(member.uuid)
+		throw e
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user.
+ * If there is already a sim, use that one. If there isn't one yet, register a new sim.
+ * @param {Ty.PkMember} member
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSim(member) {
+	let mxid = null
+	const existing = select("sim", "mxid", {user_id: member.uuid}).pluck().get()
+	if (existing) {
+		mxid = existing
+	} else {
+		mxid = await createSim(member)
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user and is joined to the room.
+ * @param {Ty.PkMember} member
+ * @param {string} roomID
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSimJoined(member, roomID) {
+	// Ensure room ID is really an ID, not an alias
+	assert.ok(roomID[0] === "!")
+
+	// Ensure user
+	const mxid = await ensureSim(member)
+
+	// Ensure joined
+	const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get()
+	if (!existing) {
+		try {
+			await api.inviteToRoom(roomID, mxid)
+			await api.joinRoom(roomID, mxid)
+		} catch (e) {
+			if (e.message.includes("is already in the room.")) {
+				// Sweet!
+			} else {
+				throw e
+			}
+		}
+		db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid)
+	}
+	return mxid
+}
+
+/**
+ * @param {Ty.PkMember} member
+ */
+async function memberToStateContent(member) {
+	const displayname = member.display_name || member.name
+	const avatar = member.avatar_url || member.webhook_avatar_url
+
+	const content = {
+		displayname,
+		membership: "join",
+		"moe.cadence.ooye.pk_member": member
+	}
+	if (avatar) content.avatar_url = await file.uploadDiscordFileToMxc(avatar)
+
+	return content
+}
+
+/**
+ * Sync profile data for a sim user. This function follows the following process:
+ * 1. Join the sim to the room if needed
+ * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
+ * 3. Compare against the previously known state content, which is helpfully stored in the database
+ * 4. If the state content has changed, send it to Matrix and update it in the database for next time
+ * @param {Ty.PkMember} member
+ * @returns {Promise<string>} mxid of the updated sim
+ */
+async function syncUser(member, roomID) {
+	const mxid = await ensureSimJoined(member, roomID)
+	const content = await memberToStateContent(member)
+	const currentHash = registerUser._hashProfileContent(content)
+	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
+	// only do the actual sync if the hash has changed since we last looked
+	if (existingHash !== currentHash) {
+		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
+	}
+	return mxid
+}
+
+/** @returns {Promise<{member?: Ty.PkMember}>} */
+function fetchMessage(messageID) {
+	return fetch(`https://api.pluralkit.me/v2/messages/${messageID}`).then(res => res.json())
+}
+
+module.exports._memberToStateContent = memberToStateContent
+module.exports.ensureSim = ensureSim
+module.exports.ensureSimJoined = ensureSimJoined
+module.exports.syncUser = syncUser
+module.exports.fetchMessage = fetchMessage

d2m/actions/register-user.js

@@ -123,7 +123,7 @@ async function memberToStateContent(user, member, guildID) {
 	return content
 }
 
-function hashProfileContent(content) {
+function _hashProfileContent(content) {
 	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
 	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
 	return signedHash
@@ -142,7 +142,7 @@ function hashProfileContent(content) {
 async function syncUser(user, member, guildID, roomID) {
 	const mxid = await ensureSimJoined(user, roomID)
 	const content = await memberToStateContent(user, member, guildID)
-	const currentHash = hashProfileContent(content)
+	const currentHash = _hashProfileContent(content)
 	const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
 	// only do the actual sync if the hash has changed since we last looked
 	if (existingHash !== currentHash) {
@@ -179,6 +179,7 @@ async function syncAllUsersInRoom(roomID) {
 }
 
 module.exports._memberToStateContent = memberToStateContent
+module.exports._hashProfileContent = _hashProfileContent
 module.exports.ensureSim = ensureSim
 module.exports.ensureSimJoined = ensureSimJoined
 module.exports.syncUser = syncUser

d2m/actions/send-message.js

@@ -10,6 +10,8 @@ const messageToEvent = sync.require("../converters/message-to-event")
 const api = sync.require("../../matrix/api")
 /** @type {import("./register-user")} */
 const registerUser = sync.require("./register-user")
+/** @type {import("./register-pk-user")} */
+const registerPkUser = sync.require("./register-pk-user")
 /** @type {import("../actions/create-room")} */
 const createRoom = sync.require("../actions/create-room")
 /** @type {import("../../discord/utils")} */
@@ -18,8 +20,9 @@ const dUtils = sync.require("../../discord/utils")
 /**
  * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
  * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
-async function sendMessage(message, guild) {
+async function sendMessage(message, guild, row) {
 	const roomID = await createRoom.ensureRoom(message.channel_id)
 
 	let senderMxid = null
@@ -29,6 +32,13 @@ async function sendMessage(message, guild) {
 		} else { // well, good enough...
 			senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
 		}
+	} else if (row && row.speedbump_webhook_id === message.webhook_id) {
+		// Handle the PluralKit public instance
+		if (row.speedbump_id === "466378653216014359") {
+			const root = await registerPkUser.fetchMessage(message.id)
+			assert(root.member) // Member is null if member was deleted. We just got this message, so member surely exists.
+			senderMxid = await registerPkUser.syncUser(root.member, roomID)
+		}
 	}
 
 	const events = await messageToEvent.messageToEvent(message, guild, {}, {api})

d2m/actions/speedbump.js

@@ -0,0 +1,53 @@
+// @ts-check
+
+const DiscordTypes = require("discord-api-types/v10")
+const passthrough = require("../../passthrough")
+const {discord, db} = passthrough
+
+const SPEEDBUMP_SPEED = 4000 // 4 seconds delay
+const SPEEDBUMP_UPDATE_FREQUENCY = 2 * 60 * 60 // 2 hours
+
+/** @type {Set<any>} */
+const KNOWN_BOTS = new Set([
+	"466378653216014359" // PluralKit
+])
+
+/**
+ * Fetch new speedbump data for the channel and put it in the database as cache
+ * @param {string} channelID
+ * @param {string?} speedbumpID
+ * @param {number?} speedbumpChecked
+ */
+async function updateCache(channelID, speedbumpID, speedbumpChecked) {
+	const now = Math.floor(Date.now() / 1000)
+	if (speedbumpChecked && now - speedbumpChecked < SPEEDBUMP_UPDATE_FREQUENCY) return
+	const webhooks = await discord.snow.webhook.getChannelWebhooks(channelID)
+	const found = webhooks.find(b => KNOWN_BOTS.has(b.application_id))
+	const foundApplication = found?.application_id
+	const foundWebhook = found?.id
+	db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_webhook_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(foundApplication, foundWebhook, now, channelID)
+}
+
+/** @type {Set<string>} set of messageID */
+const bumping = new Set()
+
+/**
+ * Slow down a message. After it passes the speedbump, return whether it's okay or if it's been deleted.
+ * @param {string} messageID
+ */
+async function doSpeedbump(messageID) {
+	bumping.add(messageID)
+	await new Promise(resolve => setTimeout(resolve, SPEEDBUMP_SPEED))
+	return !bumping.delete(messageID)
+}
+
+/**
+ * @param {string} messageID
+ */
+function onMessageDelete(messageID) {
+	bumping.delete(messageID)
+}
+
+module.exports.updateCache = updateCache
+module.exports.doSpeedbump = doSpeedbump
+module.exports.onMessageDelete = onMessageDelete

d2m/discord-client.js

@@ -47,7 +47,16 @@ class DiscordClient {
 		if (listen !== "no") {
 			this.cloud.on("event", message => discordPackets.onPacket(this, message, listen))
 		}
-		this.cloud.on("error", console.error)
+
+		const addEventLogger = (eventName, logName) => {
+			this.cloud.on(eventName, (...args) => {
+				const d = new Date().toISOString().slice(0, 19)
+				console.error(`[${d} Client ${logName}]`, ...args)
+			})
+		}
+		addEventLogger("error", "Error")
+		addEventLogger("disconnected", "Disconnected")
+		addEventLogger("ready", "Ready")
 	}
 }
 

d2m/event-dispatcher.js

@@ -31,6 +31,8 @@ const dUtils = sync.require("../discord/utils")
 const discordCommandHandler = sync.require("../discord/discord-command-handler")
 /** @type {import("../m2d/converters/utils")} */
 const mxUtils = require("../m2d/converters/utils")
+/** @type {import("./actions/speedbump")} */
+const speedbump = sync.require("./actions/speedbump")
 
 /** @type {any} */ // @ts-ignore bad types from semaphore
 const Semaphore = require("@chriscdn/promise-semaphore")
@@ -234,19 +236,22 @@ module.exports = {
 	 */
 	async onMessageCreate(client, message) {
 		if (message.author.username === "Deleted User") return // Nothing we can do for deleted users.
-		if (message.webhook_id) {
-			const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get()
-			if (row) {
-				// The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
-				return
-			}
-		}
 		const channel = client.channels.get(message.channel_id)
 		if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
 		const guild = client.guilds.get(channel.guild_id)
 		assert(guild)
 
-		await sendMessage.sendMessage(message, guild),
+		const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: message.channel_id}).get()
+		if (message.webhook_id) {
+			const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get()
+			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
+		} else if (row) {
+			const affected = await speedbump.doSpeedbump(message.id)
+			if (affected) return
+		}
+
+		// @ts-ignore
+		await sendMessage.sendMessage(message, guild, row),
 		await discordCommandHandler.execute(message, channel, guild)
 	},
 
@@ -255,13 +260,16 @@ module.exports = {
 	 * @param {DiscordTypes.GatewayMessageUpdateDispatchData} data
 	 */
 	async onMessageUpdate(client, data) {
+		const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: data.channel_id}).get()
 		if (data.webhook_id) {
 			const row = select("webhook", "webhook_id", {webhook_id: data.webhook_id}).pluck().get()
-			if (row) {
+			if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
-				// The update was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it.
+		} else if (row) {
-				return
+			// Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from.
-			}
+			const affected = await speedbump.doSpeedbump(data.id)
+			if (affected) return
 		}
+
 		// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
 		// If the message content is a string then it includes all interesting fields and is meaningful.
 		if (typeof data.content === "string") {
@@ -272,7 +280,8 @@ module.exports = {
 			if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages.
 			const guild = client.guilds.get(channel.guild_id)
 			assert(guild)
-			await editMessage.editMessage(message, guild)
+			// @ts-ignore
+			await editMessage.editMessage(message, guild, row)
 		}
 	},
 
@@ -299,6 +308,7 @@ module.exports = {
 	 * @param {DiscordTypes.GatewayMessageDeleteDispatchData} data
 	 */
 	async onMessageDelete(client, data) {
+		speedbump.onMessageDelete(data.id)
 		await deleteMessage.deleteMessage(data)
 	},
 

db/migrations/0009-add-speedbump-id.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE channel_room ADD COLUMN speedbump_id TEXT;
+ALTER TABLE channel_room ADD COLUMN speedbump_webhook_id TEXT;
+ALTER TABLE channel_room ADD COLUMN speedbump_checked INTEGER;
+
+COMMIT;

db/orm-defs.d.ts

@@ -7,6 +7,9 @@ export type Models = {
 		thread_parent: string | null
 		custom_avatar: string | null
 		last_bridged_pin_timestamp: number | null
+		speedbump_id: string | null
+		speedbump_webhook_id: string | null
+		speedbump_checked: number | null
 	}
 
 	event_message: {

docs/pluralkit-notetaking.md

@@ -85,7 +85,9 @@ OOYE's speedbump will prevent the edit command appearing at all on Matrix-side,
 
 ## Database schema
 
-TBD
+* channel_room
+	+ speedbump_id - the ID of the webhook that may be proxying in this channel
+	+ speedbump_checked - time in unix seconds when the webhooks were last queried
 
 ## Unsolved problems
 

m2d/actions/send-event.js

@@ -75,7 +75,7 @@ async function sendEvent(event) {
 
 	// no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it
 
-	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow})
+	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch})
 
 	messagesToEdit = await Promise.all(messagesToEdit.map(async e => {
 		e.message = await resolvePendingFiles(e.message)

m2d/converters/event-to-message.js

@@ -322,7 +322,7 @@ async function handleRoomOrMessageLinks(input, di) {
 /**
  * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event
  * @param {import("discord-api-types/v10").APIGuild} guild
- * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: typeof fetch}} di simple-as-nails dependency injection for the matrix API
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di simple-as-nails dependency injection for the matrix API
  */
 async function eventToMessage(event, guild, di) {
 	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */

m2d/converters/event-to-message.test.js

@@ -581,10 +581,6 @@ test("event2message: ordered list start attribute works", async t => {
 			room_id: '!cBxtVRxDlZvSVhJXVK:cadence.moe',
 			sender: '@Milan:tchncs.de',
 			type: 'm.room.message',
-		}, {}, {
-			api: {
-				getStateEvent: async () => ({displayname: "Milan"})
-			}
 		}),
 		{
 			ensureJoined: [],
@@ -2194,7 +2190,7 @@ test("event2message: mentioning events falls back to original link when the chan
 			}
 		}, {}, {
 			api: {
-				/* c8 skip next 3 */
+				/* c8 ignore next 3 */
 				async getEvent() {
 					t.fail("getEvent should not be called because it should quit early due to no channel-guild")
 				}

stdin.js

@@ -17,6 +17,7 @@ const file = sync.require("./matrix/file")
 const sendEvent = sync.require("./m2d/actions/send-event")
 const eventDispatcher = sync.require("./d2m/event-dispatcher")
 const updatePins = sync.require("./d2m/actions/update-pins")
+const speedbump = sync.require("./d2m/actions/speedbump")
 const ks = sync.require("./matrix/kstate")
 const guildID = "112760669178241024"
 

types.d.ts

@@ -34,6 +34,26 @@ export type WebhookCreds = {
 	token: string
 }
 
+export type PkMember = {
+	id: string
+	uuid: string
+	name: string
+	display_name: string | null
+	color: string | null
+	birthday: string | null
+	pronouns: string | null
+	avatar_url: string | null
+	webhook_avatar_url: string | null
+	banner: string | null
+	description: string | null
+	created: string | null
+	keep_proxy: boolean
+	tts: boolean
+	autoproxy_enabled: boolean | null
+	message_count: number | null
+	last_message_timestamp: string
+}
+
 export namespace Event {
 	export type Outer<T> = {
 		type: string