From ec1550bc97a4cd0cb14a5dc24829c108f8931119 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 7 Mar 2025 10:40:45 +1300 Subject: [PATCH] Add experimental backfills script --- scripts/backfill.js | 79 ++++++++++++++++++++++++++++++++ src/d2m/actions/register-user.js | 12 +++-- src/d2m/actions/send-message.js | 4 +- src/matrix/file.js | 4 +- 4 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 scripts/backfill.js diff --git a/scripts/backfill.js b/scripts/backfill.js new file mode 100644 index 0000000..12d9da3 --- /dev/null +++ b/scripts/backfill.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// @ts-check + +console.log("-=- This script is experimental. It WILL mess up the room history on Matrix. -=-") +console.log() + +const {channel: channelID} = require("minimist")(process.argv.slice(2), {string: ["channel"]}) +if (!channelID) { + console.error("Usage: ./scripts/backfill.js --channel=") + process.exit(1) +} + +const assert = require("assert/strict") +const sqlite = require("better-sqlite3") +const backfill = new sqlite("scripts/backfill.db") +backfill.prepare("CREATE TABLE IF NOT EXISTS backfill (channel_id TEXT NOT NULL, message_id INTEGER NOT NULL, PRIMARY KEY (channel_id, message_id))").run() + +const HeatSync = require("heatsync") + +const {reg} = require("../src/matrix/read-registration") +const passthrough = require("../src/passthrough") + +const sync = new HeatSync({watchFS: false}) +const db = new sqlite("ooye.db") +Object.assign(passthrough, {sync, db}) + +const DiscordClient = require("../src/d2m/discord-client") + +const discord = new DiscordClient(reg.ooye.discord_token, "half") +passthrough.discord = discord + +const orm = sync.require("../src/db/orm") +passthrough.from = orm.from +passthrough.select = orm.select + +/** @type {import("../src/d2m/event-dispatcher")}*/ +const eventDispatcher = sync.require("../src/d2m/event-dispatcher") + +const roomID = passthrough.select("channel_room", "room_id", {channel_id: channelID}).pluck().get() +if (!roomID) { + console.error("Please choose a channel that's already bridged.") + process.exit(1) +} + +;(async () => { + await discord.cloud.connect() + console.log("Connected, waiting for data about requested channel...") + + discord.cloud.on("event", event) +})() + +const preparedInsert = backfill.prepare("INSERT INTO backfill (channel_id, message_id) VALUES (?, ?)") + +async function event(event) { + if (event.t !== "GUILD_CREATE") return + const channel = event.d.channels.find(c => c.id === channelID) + if (!channel) return + const guild_id = event.d.id + + let last = backfill.prepare("SELECT cast(max(message_id) as TEXT) FROM backfill WHERE channel_id = ?").pluck().get(channelID) || "0" + console.log(`OK, processing messages for #${channel.name}, continuing from ${last}`) + + while (last) { + const messages = await discord.snow.channel.getChannelMessages(channelID, {limit: 50, after: String(last)}) + messages.reverse() // More recent messages come first -> More recent messages come last + for (const message of messages) { + const simulatedGatewayDispatchData = { + guild_id, + backfill: true, + ...message + } + await eventDispatcher.onMessageCreate(discord, simulatedGatewayDispatchData) + preparedInsert.run(channelID, message.id) + } + last = messages.at(-1)?.id + } + + process.exit() +} diff --git a/src/d2m/actions/register-user.js b/src/d2m/actions/register-user.js index c3bd16c..2765909 100644 --- a/src/d2m/actions/register-user.js +++ b/src/d2m/actions/register-user.js @@ -97,12 +97,12 @@ async function ensureSimJoined(user, roomID) { /** * @param {DiscordTypes.APIUser} user - * @param {Omit} member + * @param {Omit | undefined} member */ async function memberToStateContent(user, member, guildID) { let displayname = user.username if (user.global_name) displayname = user.global_name - if (member.nick) displayname = member.nick + if (member?.nick) displayname = member.nick const content = { displayname, @@ -117,7 +117,7 @@ async function memberToStateContent(user, member, guildID) { } } - if (member.avatar || user.avatar) { + if (member?.avatar || user.avatar) { // const avatarPath = file.userAvatar(user) // the user avatar only const avatarPath = file.memberAvatar(guildID, user, member) // the member avatar or the user avatar content["moe.cadence.ooye.member"].avatar = avatarPath @@ -130,12 +130,14 @@ async function memberToStateContent(user, member, guildID) { /** * https://gitdab.com/cadence/out-of-your-element/issues/9 * @param {DiscordTypes.APIUser} user - * @param {Omit} member + * @param {Omit | undefined} member * @param {DiscordTypes.APIGuild} guild * @param {DiscordTypes.APIGuildChannel} channel * @returns {number} 0 to 100 */ function memberToPowerLevel(user, member, guild, channel) { + if (!member) return 0 + const permissions = utils.getPermissions(member.roles, guild.roles, user.id, channel.permission_overwrites) /* * PL 100 = Administrator = People who can brick the room. RATIONALE: @@ -179,7 +181,7 @@ function _hashProfileContent(content, powerLevel) { * 4. Compare against the previously known state content, which is helpfully stored in the database * 5. If the state content or power level have changed, send them to Matrix and update them in the database for next time * @param {DiscordTypes.APIUser} user - * @param {Omit} member + * @param {Omit | undefined} member * @param {DiscordTypes.APIGuildChannel} channel * @param {DiscordTypes.APIGuild} guild * @param {string} roomID diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index 743c15a..15d749c 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -31,10 +31,8 @@ async function sendMessage(message, channel, guild, row) { if (!dUtils.isWebhookMessage(message)) { if (message.author.id === discord.application.id) { // no need to sync the bot's own user - } else if (message.member) { // available on a gateway message create event + } else { senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID) - } else { // well, good enough... - senderMxid = await registerUser.ensureSimJoined(message.author, roomID) } } else if (row && row.speedbump_webhook_id === message.webhook_id) { // Handle the PluralKit public instance diff --git a/src/matrix/file.js b/src/matrix/file.js index a37a732..002f34f 100644 --- a/src/matrix/file.js +++ b/src/matrix/file.js @@ -98,8 +98,8 @@ function userAvatar(user) { } function memberAvatar(guildID, user, member) { - if (!member.avatar) return userAvatar(user) - return `/guilds/${guildID}/users/${user.id}/avatars/${member.avatar}.png?size=${IMAGE_SIZE}` + if (!member?.avatar) return userAvatar(user) + return `/guilds/${guildID}/users/${user.id}/avatars/${member?.avatar}.png?size=${IMAGE_SIZE}` } function emoji(emojiID, animated) {