catch up on missed d->m messages when logging in

d2m/actions/create-room.js

@@ -190,6 +190,33 @@ async function _syncRoom(channelID, shouldActuallySync) {
 	}
 }
 
+async function _unbridgeRoom(channelID) {
+	/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
+	const channel = discord.channels.get(channelID)
+	assert.ok(channel)
+	const roomID = db.prepare("SELECT room_id from channel_room WHERE channel_id = ?").pluck().get(channelID)
+	assert.ok(roomID)
+	const spaceID = db.prepare("SELECT space_id FROM guild_space WHERE guild_id = ?").pluck().get(channel.guild_id)
+	assert.ok(spaceID)
+
+	// remove room from being a space member
+	await api.sendState(spaceID, "m.space.child", roomID, {})
+
+	// send a notification in the room
+	await api.sendEvent(roomID, "m.room.message", {
+		msgtype: "m.notice",
+		body: "⚠️ This room was removed from the bridge."
+	})
+
+	// leave room
+	await api.leaveRoom(roomID)
+
+	// delete room from database
+	const {changes} = db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channelID)
+	assert.equal(changes, 1)
+}
+
+
 /**
  * @param {DiscordTypes.APIGuildTextChannel} channel
  * @param {string} spaceID
@@ -237,3 +264,4 @@ module.exports.syncRoom = syncRoom
 module.exports.createAllForGuild = createAllForGuild
 module.exports.channelToKState = channelToKState
 module.exports._convertNameAndTopic = convertNameAndTopic
+module.exports._unbridgeRoom = _unbridgeRoom

d2m/actions/register-user.test.js

@@ -8,7 +8,7 @@ test("member2state: general", async t => {
 		await _memberToStateContent(testData.member.sheep.user, testData.member.sheep, testData.guild.general.id),
 		{
 			avatar_url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl",
-			displayname: "The Expert's Submarine | aprilsong",
+			displayname: "The Expert's Submarine",
 			membership: "join",
 			"moe.cadence.ooye.member": {
 				avatar: "/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024"

d2m/actions/send-message.js

@@ -22,8 +22,11 @@ async function sendMessage(message, guild) {
 
 	let senderMxid = null
 	if (!message.webhook_id) {
-		assert(message.member)
+		if (message.member) { // available on a gateway message create event
-		senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID)
+			senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID)
+		} else { // well, good enough...
+			senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
+		}
 	}
 
 	const events = await messageToEvent.messageToEvent(message, guild, {}, {api})
@@ -35,7 +38,7 @@ async function sendMessage(message, guild) {
 		const eventWithoutType = {...event}
 		delete eventWithoutType.$type
 
-		const eventID = await api.sendEvent(roomID, eventType, event, senderMxid)
+		const eventID = await api.sendEvent(roomID, eventType, event, senderMxid, new Date(message.timestamp).getTime())
 		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, message.channel_id, eventPart) // source 1 = discord
 
 		eventPart = 1 // TODO: use more intelligent algorithm to determine whether primary or supporting

d2m/converters/message-to-event.embeds.test.js

@@ -0,0 +1,40 @@
+const {test} = require("supertape")
+const {messageToEvent} = require("./message-to-event")
+const data = require("../../test/data")
+const Ty = require("../../types")
+
+/**
+ * @param {string} roomID
+ * @param {string} eventID
+ * @returns {(roomID: string, eventID: string) => Promise<Ty.Event.Outer<Ty.Event.M_Room_Message>>}
+ */
+function mockGetEvent(t, roomID_in, eventID_in, outer) {
+	return async function(roomID, eventID) {
+		t.equal(roomID, roomID_in)
+		t.equal(eventID, eventID_in)
+		return new Promise(resolve => {
+			setTimeout(() => {
+				resolve({
+					event_id: eventID_in,
+					room_id: roomID_in,
+					origin_server_ts: 1680000000000,
+					unsigned: {
+						age: 2245,
+						transaction_id: "$local.whatever"
+					},
+					...outer
+				})
+			})
+		})
+	}
+}
+
+test("message2event embeds: nothing but a field", async t => {
+	const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {})
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.text",
+		body: "Amanda"
+	}])
+})

d2m/converters/message-to-event.js

@@ -108,6 +108,13 @@ async function messageToEvent(message, guild, options = {}, di) {
 		addMention(repliedToEventSenderMxid)
 	}
 
+	let msgtype = "m.text"
+	// Handle message type 4, channel name changed
+	if (message.type === DiscordTypes.MessageType.ChannelNameChange) {
+		msgtype = "m.emote"
+		message.content = "changed the channel name to **" + message.content + "**"
+	}
+
 	// Text content appears first
 	if (message.content) {
 		let content = message.content
@@ -188,7 +195,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 		const newTextMessageEvent = {
 			$type: "m.room.message",
 			"m.mentions": mentions,
-			msgtype: "m.text",
+			msgtype,
 			body: body
 		}
 
@@ -239,6 +246,22 @@ async function messageToEvent(message, guild, options = {}, di) {
 					size: attachment.size
 				}
 			}
+		} else if (attachment.content_type?.startsWith("video/") && attachment.width && attachment.height) {
+			return {
+				$type: "m.room.message",
+				"m.mentions": mentions,
+				msgtype: "m.video",
+				url: await file.uploadDiscordFileToMxc(attachment.url),
+				external_url: attachment.url,
+				body: attachment.description || attachment.filename,
+				filename: attachment.filename,
+				info: {
+					mimetype: attachment.content_type,
+					w: attachment.width,
+					h: attachment.height,
+					size: attachment.size
+				}
+			}
 		} else {
 			return {
 				$type: "m.room.message",

d2m/converters/message-to-event.test.js

@@ -330,4 +330,14 @@ test("message2event: very large attachment is linked instead of being uploaded",
 	}])
 })
 
-// TODO: read "edits of replies" in the spec
+test("message2event: type 4 channel name change", async t => {
+	const events = await messageToEvent(data.special_message.thread_name_change, data.guild.general)
+	t.deepEqual(events, [{
+		$type: "m.room.message",
+		"m.mentions": {},
+		msgtype: "m.emote",
+		body: "changed the channel name to **worming**",
+		format: "org.matrix.custom.html",
+		formatted_body: "changed the channel name to <strong>worming</strong>"
+	}])
+})

d2m/converters/user-to-mxid.test.js

@@ -16,6 +16,10 @@ test("user2name: works on emojis", t => {
    t.equal(userToSimName({username: "🍪 Cookie Monster 🍪", discriminator: "0001"}), "cookie_monster")
 })
 
+test("user2name: works on single emoji at the end", t => {
+   t.equal(userToSimName({username: "Amanda 🎵", discriminator: "2192"}), "amanda")
+})
+
 test("user2name: works on crazy name", t => {
    t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7//")
 })
@@ -34,4 +38,4 @@ test("user2name: uses ID if name becomes too short", t => {
 
 test("user2name: uses ID when name has only disallowed characters", t => {
    t.equal(userToSimName({username: "!@#$%^&*", discriminator: "0001", id: "9"}), "9")
-})
+})

d2m/discord-packets.js

@@ -41,6 +41,7 @@ const utils = {
 				arr.push(thread.id)
 				client.channels.set(thread.id, thread)
 			}
+			eventDispatcher.checkMissedMessages(client, message.d)
 
 
 		} else if (message.t === "GUILD_DELETE") {

d2m/event-dispatcher.js

@@ -63,6 +63,42 @@ module.exports = {
 		})
 	},
 
+	/**
+	 * When logging back in, check if we missed any conversations in any channels. Bridge up to 49 missed messages per channel.
+	 * If more messages were missed, only the latest missed message will be posted. TODO: Consider bridging more, or post a warning when skipping history?
+	 * This can ONLY detect new messages, not any other kind of event. Any missed edits, deletes, reactions, etc will not be bridged.
+	 * @param {import("./discord-client")} client
+	 * @param {import("discord-api-types/v10").GatewayGuildCreateDispatchData} guild
+	 */
+	async checkMissedMessages(client, guild) {
+		if (guild.unavailable) return
+		const bridgedChannels = db.prepare("SELECT channel_id FROM channel_room").pluck().all()
+		const prepared = db.prepare("SELECT message_id FROM event_message WHERE channel_id = ? AND message_id = ?").pluck()
+		for (const channel of guild.channels.concat(guild.threads)) {
+			if (!bridgedChannels.includes(channel.id)) continue
+			if (!channel.last_message_id) continue
+			const latestWasBridged = prepared.get(channel.id, channel.last_message_id)
+			if (latestWasBridged) continue
+
+			/** More recent messages come first. */
+			console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
+			const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50})
+			let latestBridgedMessageIndex = messages.findIndex(m => {
+				return prepared.get(channel.id, m.id)
+			})
+			console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
+			if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date.
+			for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) {
+				const simulatedGatewayDispatchData = {
+					guild_id: guild.id,
+					mentions: [],
+					...messages[i]
+				}
+				await module.exports.onMessageCreate(client, simulatedGatewayDispatchData)
+			}
+		}
+	},
+
 	/**
 	 * @param {import("./discord-client")} client
 	 * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
@@ -85,7 +121,7 @@ module.exports = {
 
 	/**
 	 * @param {import("./discord-client")} client
-	 * @param {import("discord-api-types/v10").GatewayMessageUpdateDispatchData} message
+	 * @param {import("discord-api-types/v10").GatewayMessageUpdateDispatchData} data
 	 */
 	async onMessageUpdate(client, data) {
 		if (data.webhook_id) {

db/data-for-test.sql

@@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS "channel_room" (
 	"room_id"	TEXT NOT NULL UNIQUE,
 	"name"	TEXT,
 	"nick"	TEXT,
+	"thread_parent"	TEXT,
 	PRIMARY KEY("channel_id")
 );
 CREATE TABLE IF NOT EXISTS "event_message" (
@@ -54,10 +55,10 @@ BEGIN TRANSACTION;
 INSERT INTO guild_space (guild_id, space_id) VALUES
 ('112760669178241024', '!jjWAGMeQdNrVZSSfvz:cadence.moe');
 
-INSERT INTO channel_room (channel_id, room_id, name, nick, is_thread) VALUES
+INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES
-('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL, 0),
+('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL),
-('497161350934560778', '!edUxjVdzgUvXDUIQCK:cadence.moe', 'amanda-spam', NULL, 0),
+('497161350934560778', '!edUxjVdzgUvXDUIQCK:cadence.moe', 'amanda-spam', NULL, NULL),
-('160197704226439168', '!uCtjHhfGlYbVnPVlkG:cadence.moe', 'the-stanley-parable-channel', 'bots', 0);
+('160197704226439168', '!uCtjHhfGlYbVnPVlkG:cadence.moe', 'the-stanley-parable-channel', 'bots', NULL);
 
 INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES
 ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'),

m2d/actions/channel-webhook.js

@@ -41,9 +41,8 @@ async function ensureWebhook(channelID, forceCreate = false) {
 async function withWebhook(channelID, callback) {
 	const webhook = await ensureWebhook(channelID, false)
 	return callback(webhook).catch(e => {
-		console.error(e)
 		// TODO: check if the error was webhook-related and if webhook.created === false, then: const webhook = ensureWebhook(channelID, true); return callback(webhook)
-		throw new Error(e)
+		throw e
 	})
 }
 

m2d/converters/event-to-message.js

@@ -30,6 +30,12 @@ function eventToMessage(event) {
 			username: displayName,
 			avatar_url: avatarURL
 		})
+	} else if (event.content.msgtype === "m.emote") {
+		messages.push({
+			content: `*${displayName} ${event.content.body}*`,
+			username: displayName,
+			avatar_url: avatarURL
+		})
 	}
 
 	return messages

m2d/converters/event-to-message.test.js

@@ -21,7 +21,7 @@ test("event2message: janky test", t => {
 			}
 		}),
 		[{
-			username: "cadence:cadence.moe",
+			username: "cadence",
 			content: "test",
 			avatar_url: undefined
 		}]

stdin.js

@@ -13,6 +13,7 @@ const registerUser = sync.require("./d2m/actions/register-user")
 const mreq = sync.require("./matrix/mreq")
 const api = sync.require("./matrix/api")
 const sendEvent = sync.require("./m2d/actions/send-event")
+const eventDispatcher = sync.require("./d2m/event-dispatcher")
 const guildID = "112760669178241024"
 
 const extraContext = {}

test/data.js

@@ -864,7 +864,7 @@ module.exports = {
 			components: [],
 			channel_id: "910283343378120754",
 			author: {
-			  	username: "kumaccino",
+				username: "kumaccino",
 				public_flags: 128,
 				id: "113340068197859328",
 				global_name: "kumaccino",
@@ -876,6 +876,72 @@ module.exports = {
 			guild_id: "112760669178241024"
 		}
 	},
+	message_with_embeds: {
+		nothing_but_a_field: {
+			guild_id: "497159726455455754",
+			mentions: [],
+			id: "1141934888862351440",
+			type: 20,
+			content: "",
+			channel_id: "497161350934560778",
+			author: {
+				id: "1109360903096369153",
+				username: "Amanda 🎵",
+				avatar: "d56cd1b26e043ae512edae2214962faa",
+				discriminator: "2192",
+				public_flags: 524288,
+				flags: 524288,
+				bot: true,
+				banner: null,
+				accent_color: null,
+				global_name: null,
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [
+				{
+					type: "rich",
+					color: 3092790,
+					fields: [
+						{
+							name: "Amanda 🎵#2192 <:online:606664341298872324>\nwillow tree, branch 0",
+							value: "**❯ Uptime:**\n3m 55s\n**❯ Memory:**\n64.45MB",
+							inline: false
+						}
+					]
+				}
+			],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-08-18T03:21:33.629000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: [],
+			application_id: "1109360903096369153",
+			interaction: {
+				id: "1141934887608254475",
+				type: 2,
+				name: "stats",
+				user: {
+					id: "320067006521147393",
+					username: "papiophidian",
+					avatar: "47a19b0445069b826e136da4df4259bb",
+					discriminator: "0",
+					public_flags: 4194880,
+					flags: 4194880,
+					banner: null,
+					accent_color: null,
+					global_name: "PapiOphidian",
+					avatar_decoration_data: null,
+					banner_color: null
+				}
+			},
+			webhook_id: "1109360903096369153"
+		}
+	},
 	message_update: {
 		edit_by_webhook: {
 			application_id: "684280192553844747",
@@ -1277,5 +1343,38 @@ module.exports = {
 			],
 			guild_id: "112760669178241024"
 		}
+	},
+	special_message: {
+		thread_name_change: {
+			id: "1142391602799710298",
+			type: 4,
+			content: "worming",
+			channel_id: "1142271000067706880",
+			author: {
+				id: "772659086046658620",
+				username: "cadence.worm",
+				avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+				discriminator: "0",
+				public_flags: 0,
+				flags: 0,
+				banner: null,
+				accent_color: null,
+				global_name: "cadence",
+				avatar_decoration_data: null,
+				banner_color: null
+			},
+			attachments: [],
+			embeds: [],
+			mentions: [],
+			mention_roles: [],
+			pinned: false,
+			mention_everyone: false,
+			tts: false,
+			timestamp: "2023-08-19T09:36:22.717000+00:00",
+			edited_timestamp: null,
+			flags: 0,
+			components: [],
+			position: 12
+		}
 	}
 }

test/test.js

@@ -21,6 +21,7 @@ require("../matrix/kstate.test")
 require("../matrix/api.test")
 require("../matrix/read-registration.test")
 require("../d2m/converters/message-to-event.test")
+require("../d2m/converters/message-to-event.embeds.test")
 require("../d2m/converters/edit-to-changes.test")
 require("../d2m/actions/create-room.test")
 require("../d2m/converters/user-to-mxid.test")