Add experimental backfills script

scripts/backfill.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+// @ts-check
+
+console.log("-=- This script is experimental. It WILL mess up the room history on Matrix. -=-")
+console.log()
+
+const {channel: channelID} = require("minimist")(process.argv.slice(2), {string: ["channel"]})
+if (!channelID) {
+	console.error("Usage: ./scripts/backfill.js --channel=<channel id here>")
+	process.exit(1)
+}
+
+const assert = require("assert/strict")
+const sqlite = require("better-sqlite3")
+const backfill = new sqlite("scripts/backfill.db")
+backfill.prepare("CREATE TABLE IF NOT EXISTS backfill (channel_id TEXT NOT NULL, message_id INTEGER NOT NULL, PRIMARY KEY (channel_id, message_id))").run()
+
+const HeatSync = require("heatsync")
+
+const {reg} = require("../src/matrix/read-registration")
+const passthrough = require("../src/passthrough")
+
+const sync = new HeatSync({watchFS: false})
+const db = new sqlite("ooye.db")
+Object.assign(passthrough, {sync, db})
+
+const DiscordClient = require("../src/d2m/discord-client")
+
+const discord = new DiscordClient(reg.ooye.discord_token, "half")
+passthrough.discord = discord
+
+const orm = sync.require("../src/db/orm")
+passthrough.from = orm.from
+passthrough.select = orm.select
+
+/** @type {import("../src/d2m/event-dispatcher")}*/
+const eventDispatcher = sync.require("../src/d2m/event-dispatcher")
+
+const roomID = passthrough.select("channel_room", "room_id", {channel_id: channelID}).pluck().get()
+if (!roomID) {
+	console.error("Please choose a channel that's already bridged.")
+	process.exit(1)
+}
+
+;(async () => {
+	await discord.cloud.connect()
+	console.log("Connected, waiting for data about requested channel...")
+
+	discord.cloud.on("event", event)
+})()
+
+const preparedInsert = backfill.prepare("INSERT INTO backfill (channel_id, message_id) VALUES (?, ?)")
+
+async function event(event) {
+	if (event.t !== "GUILD_CREATE") return
+	const channel = event.d.channels.find(c => c.id === channelID)
+	if (!channel) return
+	const guild_id = event.d.id
+
+	let last = backfill.prepare("SELECT cast(max(message_id) as TEXT) FROM backfill WHERE channel_id = ?").pluck().get(channelID) || "0"
+	console.log(`OK, processing messages for #${channel.name}, continuing from ${last}`)
+
+	while (last) {
+		const messages = await discord.snow.channel.getChannelMessages(channelID, {limit: 50, after: String(last)})
+		messages.reverse() // More recent messages come first -> More recent messages come last
+		for (const message of messages) {
+			const simulatedGatewayDispatchData = {
+				guild_id,
+				backfill: true,
+				...message
+			}
+			await eventDispatcher.onMessageCreate(discord, simulatedGatewayDispatchData)
+			preparedInsert.run(channelID, message.id)
+		}
+		last = messages.at(-1)?.id
+	}
+
+	process.exit()
+}

src/d2m/actions/register-user.js

@@ -97,12 +97,12 @@ async function ensureSimJoined(user, roomID) {
 
 /**
  * @param {DiscordTypes.APIUser} user
- * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
+ * @param {Omit<DiscordTypes.APIGuildMember, "user"> | undefined} member
  */
 async function memberToStateContent(user, member, guildID) {
 	let displayname = user.username
 	if (user.global_name) displayname = user.global_name
-	if (member.nick) displayname = member.nick
+	if (member?.nick) displayname = member.nick
 
 	const content = {
 		displayname,
@@ -117,7 +117,7 @@ async function memberToStateContent(user, member, guildID) {
 		}
 	}
 
-	if (member.avatar || user.avatar) {
+	if (member?.avatar || user.avatar) {
 		// const avatarPath = file.userAvatar(user) // the user avatar only
 		const avatarPath = file.memberAvatar(guildID, user, member) // the member avatar or the user avatar
 		content["moe.cadence.ooye.member"].avatar = avatarPath
@@ -130,12 +130,14 @@ async function memberToStateContent(user, member, guildID) {
 /**
  * https://gitdab.com/cadence/out-of-your-element/issues/9
  * @param {DiscordTypes.APIUser} user
- * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
+ * @param {Omit<DiscordTypes.APIGuildMember, "user"> | undefined} member
  * @param {DiscordTypes.APIGuild} guild
  * @param {DiscordTypes.APIGuildChannel} channel
  * @returns {number} 0 to 100
  */
 function memberToPowerLevel(user, member, guild, channel) {
+	if (!member) return 0
+
 	const permissions = utils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites)
 	/*
 	 * PL 100 = Administrator = People who can brick the room. RATIONALE:
@@ -179,7 +181,7 @@ function _hashProfileContent(content, powerLevel) {
  * 4. Compare against the previously known state content, which is helpfully stored in the database
  * 5. If the state content or power level have changed, send them to Matrix and update them in the database for next time
  * @param {DiscordTypes.APIUser} user
- * @param {Omit<DiscordTypes.APIGuildMember, "user">} member
+ * @param {Omit<DiscordTypes.APIGuildMember, "user"> | undefined} member
  * @param {DiscordTypes.APIGuildChannel} channel
  * @param {DiscordTypes.APIGuild} guild
  * @param {string} roomID

src/d2m/actions/send-message.js

@@ -31,10 +31,8 @@ async function sendMessage(message, channel, guild, row) {
 	if (!dUtils.isWebhookMessage(message)) {
 		if (message.author.id === discord.application.id) {
 			// no need to sync the bot's own user
-		} else if (message.member) { // available on a gateway message create event
+		} else {
 			senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID)
-		} else { // well, good enough...
-			senderMxid = await registerUser.ensureSimJoined(message.author, roomID)
 		}
 	} else if (row && row.speedbump_webhook_id === message.webhook_id) {
 		// Handle the PluralKit public instance

src/matrix/file.js

@@ -98,8 +98,8 @@ function userAvatar(user) {
 }
 
 function memberAvatar(guildID, user, member) {
-	if (!member.avatar) return userAvatar(user)
+	if (!member?.avatar) return userAvatar(user)
-	return `/guilds/${guildID}/users/${user.id}/avatars/${member.avatar}.png?size=${IMAGE_SIZE}`
+	return `/guilds/${guildID}/users/${user.id}/avatars/${member?.avatar}.png?size=${IMAGE_SIZE}`
 }
 
 function emoji(emojiID, animated) {