allow migrating from old to new bridge

This commit is contained in:
Cadence Ember 2023-09-04 01:38:30 +12:00
parent 5185ae45ab
commit 39458bd2bf
13 changed files with 214 additions and 42 deletions

View file

@ -78,7 +78,10 @@ async function syncSpace(guildID) {
const guildKState = await guildToKState(guild)
if (!spaceID) return
if (!spaceID) {
const spaceID = await createSpace(guild, guildKState)
return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here.
}
console.log(`[space sync] to matrix: ${guild.name}`)
@ -123,7 +126,10 @@ async function syncSpaceFully(guildID) {
const guildKState = await guildToKState(guild)
if (!spaceID) return
if (!spaceID) {
const spaceID = await createSpace(guild, guildKState)
return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here.
}
console.log(`[space sync] to matrix: ${guild.name}`)

View file

@ -12,9 +12,9 @@ const discordPackets = sync.require("./discord-packets")
class DiscordClient {
/**
* @param {string} discordToken
* @param {boolean} listen whether to set up the event listeners for OOYE to operate
* @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
*/
constructor(discordToken, listen = true) {
constructor(discordToken, listen = "full") {
this.discordToken = discordToken
this.snow = new SnowTransfer(discordToken)
this.cloud = new CloudStorm(discordToken, {
@ -44,8 +44,8 @@ class DiscordClient {
this.guilds = new Map()
/** @type {Map<string, Array<string>>} */
this.guildChannelMap = new Map()
if (listen) {
this.cloud.on("event", message => discordPackets.onPacket(this, message))
if (listen !== "no") {
this.cloud.on("event", message => discordPackets.onPacket(this, message, listen))
}
this.cloud.on("error", console.error)
}

View file

@ -10,8 +10,9 @@ const utils = {
/**
* @param {import("./discord-client")} client
* @param {import("cloudstorm").IGatewayMessage} message
* @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
*/
async onPacket(client, message) {
async onPacket(client, message, listen) {
// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
/** @type {typeof import("./event-dispatcher")} */
const eventDispatcher = sync.require("./event-dispatcher")
@ -41,7 +42,9 @@ const utils = {
arr.push(thread.id)
client.channels.set(thread.id, thread)
}
eventDispatcher.checkMissedMessages(client, message.d)
if (listen === "full") {
eventDispatcher.checkMissedMessages(client, message.d)
}
} else if (message.t === "GUILD_UPDATE") {
const guild = client.guilds.get(message.d.id)
@ -90,35 +93,37 @@ const utils = {
}
// Event dispatcher for OOYE bridge operations
try {
if (message.t === "GUILD_UPDATE") {
await eventDispatcher.onGuildUpdate(client, message.d)
if (listen === "full") {
try {
if (message.t === "GUILD_UPDATE") {
await eventDispatcher.onGuildUpdate(client, message.d)
} else if (message.t === "CHANNEL_UPDATE") {
await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false)
} else if (message.t === "CHANNEL_UPDATE") {
await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false)
} else if (message.t === "THREAD_CREATE") {
// @ts-ignore
await eventDispatcher.onThreadCreate(client, message.d)
} else if (message.t === "THREAD_CREATE") {
// @ts-ignore
await eventDispatcher.onThreadCreate(client, message.d)
} else if (message.t === "THREAD_UPDATE") {
await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true)
} else if (message.t === "THREAD_UPDATE") {
await eventDispatcher.onChannelOrThreadUpdate(client, message.d, true)
} else if (message.t === "MESSAGE_CREATE") {
await eventDispatcher.onMessageCreate(client, message.d)
} else if (message.t === "MESSAGE_CREATE") {
await eventDispatcher.onMessageCreate(client, message.d)
} else if (message.t === "MESSAGE_UPDATE") {
await eventDispatcher.onMessageUpdate(client, message.d)
} else if (message.t === "MESSAGE_UPDATE") {
await eventDispatcher.onMessageUpdate(client, message.d)
} else if (message.t === "MESSAGE_DELETE") {
await eventDispatcher.onMessageDelete(client, message.d)
} else if (message.t === "MESSAGE_DELETE") {
await eventDispatcher.onMessageDelete(client, message.d)
} else if (message.t === "MESSAGE_REACTION_ADD") {
await eventDispatcher.onReactionAdd(client, message.d)
} else if (message.t === "MESSAGE_REACTION_ADD") {
await eventDispatcher.onReactionAdd(client, message.d)
}
} catch (e) {
// Let OOYE try to handle errors too
eventDispatcher.onError(client, e, message)
}
} catch (e) {
// Let OOYE try to handle errors too
eventDispatcher.onError(client, e, message)
}
}
}

View file

@ -23,10 +23,6 @@ const discordCommandHandler = sync.require("./discord-command-handler")
let lastReportedEvent = 0
function isGuildAllowed(guildID) {
return ["112760669178241024", "497159726455455754", "1100319549670301727"].includes(guildID)
}
// Grab Discord events we care about for the bridge, check them, and pass them on
module.exports = {
@ -93,12 +89,22 @@ module.exports = {
if (latestWasBridged) continue
/** More recent messages come first. */
console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50})
// console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
let messages
try {
messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50})
} catch (e) {
if (e.message === `{"message": "Missing Access", "code": 50001}`) { // pathetic error handling from SnowTransfer
console.log(`[check missed messages] no permissions to look back in channel ${channel.name} (${channel.id})`)
continue // Sucks.
} else {
throw e // Sucks more.
}
}
let latestBridgedMessageIndex = messages.findIndex(m => {
return prepared.get(m.id)
})
console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
// console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date.
for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) {
const simulatedGatewayDispatchData = {

View file

@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db})
const DiscordClient = require("./d2m/discord-client")
const discord = new DiscordClient(config.discordToken, true)
const discord = new DiscordClient(config.discordToken, "full")
passthrough.discord = discord
const as = require("./m2d/appservice")

View file

@ -43,5 +43,24 @@ async function mreq(method, url, body, extra = {}) {
return root
}
/**
* JavaScript doesn't have Racket-like parameters with dynamic scoping, so
* do NOT do anything else at the same time as this.
* @template T
* @param {string} token
* @param {(...arg: any[]) => Promise<T>} callback
* @returns {Promise<T>}
*/
async function withAccessToken(token, callback) {
const prevToken = reg.as_token
reg.as_token = token
try {
return await callback()
} finally {
reg.as_token = prevToken
}
}
module.exports.MatrixServerError = MatrixServerError
module.exports.mreq = mreq
module.exports.withAccessToken = withAccessToken

6
package-lock.json generated
View file

@ -9,6 +9,7 @@
"version": "1.0.0",
"license": "MIT",
"dependencies": {
"@chriscdn/promise-semaphore": "^2.0.1",
"better-sqlite3": "^8.3.0",
"chunk-text": "^2.0.1",
"cloudstorm": "^0.8.0",
@ -51,6 +52,11 @@
"integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==",
"dev": true
},
"node_modules/@chriscdn/promise-semaphore": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.1.tgz",
"integrity": "sha512-C0Ku5DNZFbafbSRXagidIaRgzhgGmSHk4aAgPpmmHEostazBiSaMryovC/Aix3vRLNuaeGDKN/DHoNECmMD6jg=="
},
"node_modules/@cloudcmd/stub": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@cloudcmd/stub/-/stub-4.0.1.tgz",

View file

@ -15,6 +15,7 @@
"author": "Cadence, PapiOphidian",
"license": "MIT",
"dependencies": {
"@chriscdn/promise-semaphore": "^2.0.1",
"better-sqlite3": "^8.3.0",
"chunk-text": "^2.0.1",
"cloudstorm": "^0.8.0",

View file

@ -23,9 +23,9 @@ const sync = new HeatSync({watchFS: false})
Object.assign(passthrough, {config, sync})
const DiscordClient = require("../d2m/discord-client", false)
const DiscordClient = require("../d2m/discord-client")
const discord = new DiscordClient(config.discordToken, false)
const discord = new DiscordClient(config.discordToken, "no")
passthrough.discord = discord
;(async () => {

View file

@ -0,0 +1,124 @@
// @ts-check
const assert = require("assert").strict
/** @type {any} */ // @ts-ignore bad types from semaphore
const Semaphore = require("@chriscdn/promise-semaphore")
const sqlite = require("better-sqlite3")
const HeatSync = require("heatsync")
const config = require("../config")
const passthrough = require("../passthrough")
const sync = new HeatSync({watchFS: false})
/** @type {import("../matrix/read-registration")} */
const reg = sync.require("../matrix/read-registration")
assert(reg.old_bridge)
const oldAT = reg.old_bridge.as_token
const newAT = reg.as_token
const oldDB = new sqlite(reg.old_bridge.database)
const db = new sqlite("db/ooye.db")
db.exec(`CREATE TABLE IF NOT EXISTS migration (
discord_channel TEXT NOT NULL,
migrated INTEGER NOT NULL,
PRIMARY KEY("discord_channel")
) WITHOUT ROWID;`)
Object.assign(passthrough, {config, sync, db})
const DiscordClient = require("../d2m/discord-client")
const discord = new DiscordClient(config.discordToken, "half")
passthrough.discord = discord
/** @type {import("../d2m/actions/create-space")} */
const createSpace = sync.require("../d2m/actions/create-space")
/** @type {import("../d2m/actions/create-room")} */
const createRoom = sync.require("../d2m/actions/create-room")
/** @type {import("../matrix/mreq")} */
const mreq = sync.require("../matrix/mreq")
/** @type {import("../matrix/api")} */
const api = sync.require("../matrix/api")
const sema = new Semaphore()
;(async () => {
await discord.cloud.connect()
console.log("Discord gateway started")
discord.cloud.on("event", event => onPacket(discord, event))
})()
/** @param {DiscordClient} discord */
function onPacket(discord, event) {
if (event.t === "GUILD_CREATE") {
const guild = event.d
if (["1100319549670301727", "112760669178241024", "497159726455455754"].includes(guild.id)) return
sema.request(() => migrateGuild(guild))
}
}
const newBridgeMxid = `@${reg.sender_localpart}:${reg.ooye.server_name}`
/** @param {import("discord-api-types/v10").GatewayGuildCreateDispatchData} guild */
async function migrateGuild(guild) {
console.log(`START MIGRATION of ${guild.name} (${guild.id})`)
// Step 1: Create a new space for the guild (createSpace)
const spaceID = await createSpace.syncSpace(guild.id)
let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id)
const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all()
oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel))
console.log("Found these rooms which can be migrated:")
console.log(oldRooms)
for (const row of oldRooms) {
const roomID = row.matrix_id
const channel = discord.channels.get(row.discord_channel)
assert(channel)
// Step 2: (Using old bridge access token) Join the new bridge to the old rooms and give it PL 100
console.log(`-- Joining channel ${channel.name}...`)
await mreq.withAccessToken(oldAT, async () => {
try {
await api.inviteToRoom(roomID, newBridgeMxid)
} catch (e) {
if (e.message.includes("is already in the room")) {
// Great!
} else {
throw e
}
}
await api.setUserPower(roomID, newBridgeMxid, 100)
})
await api.joinRoom(roomID)
// Step 3: Remove the old bridge's aliases
console.log(`-- -- Deleting aliases...`)
await mreq.withAccessToken(oldAT, async () => { // have to run as old application service since the AS owns its aliases
const aliases = (await mreq.mreq("GET", `/client/v3/rooms/${roomID}/aliases`)).aliases
for (const alias of aliases) {
if (alias.match(/^#?_?discord/)) {
await mreq.mreq("DELETE", `/client/v3/directory/room/${alias.replace(/#/g, "%23")}`)
}
}
await api.sendState(roomID, "m.room.canonical_alias", "", {})
})
// Step 4: Add old rooms to new database; they are now also the new rooms
db.prepare("REPLACE INTO channel_room (channel_id, room_id, name) VALUES (?, ?, ?)").run(channel.id, row.matrix_id, channel.name)
console.log(`-- -- Added to database`)
// Step 5: Call syncRoom for each room
await createRoom.syncRoom(row.discord_channel)
console.log(`-- -- Finished syncing`)
db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
}
// Step 5: Call syncSpace to make sure everything is up to date
await createSpace.syncSpace(guild.id)
console.log(`Finished migrating ${guild.name} to Out Of Your Element`)
}

View file

@ -13,7 +13,7 @@ Object.assign(passthrough, {config, sync, db})
const DiscordClient = require("../d2m/discord-client")
const discord = new DiscordClient(config.discordToken, false)
const discord = new DiscordClient(config.discordToken, "no")
passthrough.discord = discord
;(async () => {

View file

@ -25,7 +25,7 @@ module.exports = {
"m.room.name/": {name: "main"},
"m.room.topic/": {topic: "#collective-unconscious | https://docs.google.com/document/d/blah/edit | I spread, pipe, and whip because it is my will. :headstone:\n\nChannel ID: 112760669178241024\nGuild ID: 112760669178241024"},
"m.room.guest_access/": {guest_access: "can_join"},
"m.room.history_visibility/": {history_visibility: "invited"},
"m.room.history_visibility/": {history_visibility: "shared"},
"m.space.parent/!jjWAGMeQdNrVZSSfvz:cadence.moe": {
via: ["cadence.moe"],
canonical: true
@ -45,7 +45,8 @@ module.exports = {
events: {
"m.room.avatar": 0
}
}
},
"chat.schildi.hide_ui/read_receipts": {hidden: true}
}
},
guild: {

4
types.d.ts vendored
View file

@ -21,6 +21,10 @@ export type AppServiceRegistrationConfig = {
max_file_size: number
server_name: string
}
old_bridge?: {
as_token: string
database: string
}
}
export type WebhookCreds = {