From 1d99b91ef7b702fa5ff9896f971e7bacc1d7d984 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 1 Oct 2023 01:24:05 +1300 Subject: [PATCH] Add database migrations --- d2m/actions/register-user.js | 12 +- db/migrate.js | 42 +++++ db/migrations/0001-schema.sql | 4 + .../0002-optimise-profile-content.sql | 4 + ...js => 0002-optimise-profile-content.up.js} | 10 +- db/orm-utils.d.ts | 5 +- scripts/migrate-from-old-bridge.js | 6 +- start.js | 2 + test/data.js | 156 +++++++++++++++++- test/ooye-test-data.sql | 2 +- test/test.js | 53 ++++-- 11 files changed, 263 insertions(+), 33 deletions(-) create mode 100644 db/migrate.js rename db/migrations/{0002-optimise-profile-content.js => 0002-optimise-profile-content.up.js} (60%) diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js index 64e0ceb..05f8518 100644 --- a/d2m/actions/register-user.js +++ b/d2m/actions/register-user.js @@ -11,6 +11,10 @@ const api = sync.require("../../matrix/api") const file = sync.require("../../matrix/file") /** @type {import("../converters/user-to-mxid")} */ const userToMxid = sync.require("../converters/user-to-mxid") +/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore +let hasher = null +// @ts-ignore +require("xxhash-wasm")().then(h => hasher = h) /** * A sim is an account that is being simulated by the bridge to copy events from the other side. @@ -120,7 +124,9 @@ async function memberToStateContent(user, member, guildID) { } function hashProfileContent(content) { - return `${content.displayname}\u0000${content.avatar_url}` + const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`) + const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range + return signedHash } /** @@ -137,11 +143,11 @@ async function syncUser(user, member, guildID, roomID) { const mxid = await ensureSimJoined(user, roomID) const content = await memberToStateContent(user, member, guildID) const currentHash = hashProfileContent(content) - const existingHash = select("sim_member", "profile_event_content_hash", "WHERE room_id = ? AND mxid = ?").pluck().get(roomID, mxid) + const existingHash = select("sim_member", "hashed_profile_content", "WHERE room_id = ? AND mxid = ?").safeIntegers().pluck().get(roomID, mxid) // only do the actual sync if the hash has changed since we last looked if (existingHash !== currentHash) { await api.sendState(roomID, "m.room.member", mxid, content, mxid) - db.prepare("UPDATE sim_member SET profile_event_content_hash = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) + db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) } return mxid } diff --git a/db/migrate.js b/db/migrate.js new file mode 100644 index 0000000..7c1faf9 --- /dev/null +++ b/db/migrate.js @@ -0,0 +1,42 @@ +// @ts-check + +const fs = require("fs") +const {join} = require("path") + +async function migrate(db) { + let files = fs.readdirSync(join(__dirname, "migrations")) + files = files.sort() + db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run() + let progress = db.prepare("SELECT * FROM migration").pluck().get() + if (!progress) { + progress = "" + db.prepare("INSERT INTO migration VALUES ('')").run() + } + + let migrationRan = false + + for (const filename of files) { + if (progress >= filename) continue + console.log(`Applying database migration ${filename}`) + if (filename.endsWith(".sql")) { + const sql = fs.readFileSync(join(__dirname, "migrations", filename), "utf8") + db.exec(sql) + } else if (filename.endsWith(".js")) { + await require("./" + join("migrations", filename))(db) + } else { + continue + } + + migrationRan = true + db.transaction(() => { + db.prepare("DELETE FROM migration").run() + db.prepare("INSERT INTO migration VALUES (?)").run(filename) + })() + } + + if (migrationRan) { + console.log("Database migrations all done.") + } +} + +module.exports.migrate = migrate diff --git a/db/migrations/0001-schema.sql b/db/migrations/0001-schema.sql index 13fbc1b..b02fac3 100644 --- a/db/migrations/0001-schema.sql +++ b/db/migrations/0001-schema.sql @@ -1,3 +1,5 @@ +BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS "sim" ( "discord_id" TEXT NOT NULL, "sim_name" TEXT NOT NULL UNIQUE, @@ -86,3 +88,5 @@ CREATE TABLE IF NOT EXISTS "reaction" ( "encoded_emoji" TEXT NOT NULL, PRIMARY KEY ("hashed_event_id") ) WITHOUT ROWID; + +COMMIT; diff --git a/db/migrations/0002-optimise-profile-content.sql b/db/migrations/0002-optimise-profile-content.sql index 6deeb9f..c648b24 100644 --- a/db/migrations/0002-optimise-profile-content.sql +++ b/db/migrations/0002-optimise-profile-content.sql @@ -1,3 +1,5 @@ +BEGIN TRANSACTION; + -- Change hashed_profile_content column affinity to INTEGER CREATE TABLE "new_sim_member" ( @@ -13,4 +15,6 @@ DROP TABLE sim_member; ALTER TABLE new_sim_member RENAME TO sim_member; +COMMIT; + VACUUM; diff --git a/db/migrations/0002-optimise-profile-content.js b/db/migrations/0002-optimise-profile-content.up.js similarity index 60% rename from db/migrations/0002-optimise-profile-content.js rename to db/migrations/0002-optimise-profile-content.up.js index 22acdfa..a8619cf 100644 --- a/db/migrations/0002-optimise-profile-content.js +++ b/db/migrations/0002-optimise-profile-content.up.js @@ -1,13 +1,13 @@ module.exports = async function(db) { const hasher = await require("xxhash-wasm")() - const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member").pluck().all() + const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all() const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?") db.transaction(() => { - for (const s of contents) { - if (!Buffer.isBuffer(s)) s = Buffer.from(s) - const unsignedHash = hasher.h64(eventID) + for (let s of contents) { + let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s)) + const unsignedHash = hasher.h64Raw(b) const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range - stmt.run(s, signedHash) + stmt.run(signedHash, s) } })() } diff --git a/db/orm-utils.d.ts b/db/orm-utils.d.ts index 58eeb28..e0c2cb4 100644 --- a/db/orm-utils.d.ts +++ b/db/orm-utils.d.ts @@ -54,7 +54,7 @@ export type Models = { sim_member: { mxid: string room_id: string - profile_event_content_hash: any + hashed_profile_content: number } webhook: { @@ -79,8 +79,9 @@ export type Models = { export type Prepared = { pluck: () => Prepared + safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}> all: (..._: any[]) => Row[] - get: (..._: any[]) => Row? + get: (..._: any[]) => Row | null } export type AllKeys = U extends any ? keyof U : never diff --git a/scripts/migrate-from-old-bridge.js b/scripts/migrate-from-old-bridge.js index 84daad7..aec345d 100644 --- a/scripts/migrate-from-old-bridge.js +++ b/scripts/migrate-from-old-bridge.js @@ -20,7 +20,7 @@ const newAT = reg.as_token const oldDB = new sqlite(reg.old_bridge.database) const db = new sqlite("db/ooye.db") -db.exec(`CREATE TABLE IF NOT EXISTS migration ( +db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration ( discord_channel TEXT NOT NULL, migrated INTEGER NOT NULL, PRIMARY KEY("discord_channel") @@ -69,7 +69,7 @@ async function migrateGuild(guild) { const spaceID = await createSpace.syncSpace(guild) let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id) - const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all() + const migrated = db.prepare("SELECT discord_channel FROM half_shot_migration WHERE migrated = 1").pluck().all() oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel)) console.log("Found these rooms which can be migrated:") console.log(oldRooms) @@ -128,7 +128,7 @@ async function migrateGuild(guild) { await createRoom.syncRoom(row.discord_channel) console.log(`-- -- Finished syncing`) - db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id) + db.prepare("INSERT INTO half_shot_migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id) } // Step 5: Call syncSpace to make sure everything is up to date diff --git a/start.js b/start.js index 49cfadb..819281f 100644 --- a/start.js +++ b/start.js @@ -1,6 +1,7 @@ // @ts-check const sqlite = require("better-sqlite3") +const migrate = require("./db/migrate") const HeatSync = require("heatsync") const config = require("./config") @@ -30,6 +31,7 @@ discord.snow.requestHandler.on("requestError", data => { }) ;(async () => { + await migrate.migrate(db) await discord.cloud.connect() console.log("Discord gateway started") diff --git a/test/data.js b/test/data.js index 8cfa717..f82b4ae 100644 --- a/test/data.js +++ b/test/data.js @@ -93,7 +93,28 @@ module.exports = { afk_timeout: 300, id: "112760669178241024", icon: "a_f83622e09ead74f0c5c527fe241f8f8c", - emojis: [], + emojis: [ + { + version: 0, + roles: [], + require_colons: true, + name: "hippo", + managed: false, + id: "230201364309868544", + available: true, + animated: false + }, + { + version: 0, + roles: [], + require_colons: true, + name: "hipposcope", + managed: false, + id: "393635038903926784", + available: true, + animated: true + } + ], premium_subscription_count: 14, roles: [], discovery_splash: null, @@ -1133,6 +1154,139 @@ module.exports = { } }, webhook_id: "1109360903096369153" + }, + reply_with_only_embed: { + type: 19, + tts: false, + timestamp: "2023-09-29T20:44:42.606000+00:00", + referenced_message: { + type: 19, + tts: false, + timestamp: "2023-09-29T20:44:42.204000+00:00", + pinned: false, + message_reference: { + message_id: "1157413453921787924", + guild_id: "1150201337112449045", + channel_id: "1150208267285434429" + }, + mentions: [ + { + username: "goat_six", + public_flags: 64, + id: "334539029879980040", + global_name: "GoatSixx", + discriminator: "0", + avatar_decoration_data: null, + avatar: "fd87e077c6ebe4239ce573bae083ed66" + } + ], + mention_roles: [], + mention_everyone: false, + id: "1157417694728044624", + flags: 0, + embeds: [ + { + url: "https://twitter.com/dynastic/status/1707484191963648161", + type: "rich", + timestamp: "2023-09-28T19:55:29.543000+00:00", + reference_id: "1157417694728044624", + footer: { + text: "Twitter", + proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png", + icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png" + }, + fields: [ + { value: "119", name: "Retweets", inline: true }, + { value: "5581", name: "Likes", inline: true } + ], + description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?", + color: 1942002, + author: { + url: "https://twitter.com/dynastic", + proxy_icon_url: "https://images-ext-2.discordapp.net/external/06UZNFT37nepFbzmK2FN4q-9DO_UeSaOaZQICSiMexU/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg", + name: "dynastic (@dynastic)", + icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg" + } + } + ], + edited_timestamp: null, + content: "https://twitter.com/dynastic/status/1707484191963648161", + components: [], + channel_id: "1150208267285434429", + author: { + username: "pokemongod", + public_flags: 0, + id: "66255093481082800", + global_name: "PokemonGod", + discriminator: "0", + avatar_decoration_data: null, + avatar: "0cab06c4256499749cbdd4561c629f84" + }, + attachments: [] + }, + pinned: false, + message_reference: { + message_id: "1157417694728044624", + guild_id: "1150201337112449045", + channel_id: "1150208267285434429" + }, + mentions: [], + mention_roles: [], + mention_everyone: false, + member: { + roles: [ "1153875112832008212" ], + premium_since: null, + pending: false, + nick: null, + mute: false, + joined_at: "2023-09-20T02:07:44.874994+00:00", + flags: 0, + deaf: false, + communication_disabled_until: null, + avatar: null + }, + id: "1157417696414150778", + flags: 0, + embeds: [ + { + url: "https://twitter.com/i/status/1707484191963648161", + type: "rich", + timestamp: "2023-09-28T19:55:29+00:00", + footer: { + text: "Twitter", + proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png", + icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png" + }, + fields: [ + { value: "119", name: "Retweets", inline: true }, + { value: "5581", name: "Likes", inline: true } + ], + description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?", + color: 1942002, + author: { + url: "https://twitter.com/i/user/719631291747078145", + proxy_icon_url: "https://images-ext-1.discordapp.net/external/6LgXrIifZ-MhwPPiwqAomgoy93d932jZiJqLCAf79Fw/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg", + name: "dynastic (@dynastic)", + icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg" + } + } + ], + edited_timestamp: null, + content: "", + components: [], + channel_id: "1150208267285434429", + author: { + username: "Twitter Video Embeds", + public_flags: 65536, + id: "842601826674540574", + global_name: null, + discriminator: "4945", + bot: true, + avatar_decoration_data: null, + avatar: "6ed5bf10f953b22d47893b4655705b30" + }, + attachments: [], + guild_id: "1150201337112449045" } }, message_update: { diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 2a56381..4724e30 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -17,7 +17,7 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES ('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'), ('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');; -INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES +INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES ('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL); INSERT INTO message_channel (message_id, channel_id) VALUES diff --git a/test/test.js b/test/test.js index c2ec57b..76fe658 100644 --- a/test/test.js +++ b/test/test.js @@ -1,8 +1,12 @@ // @ts-check const fs = require("fs") +const {join} = require("path") const sqlite = require("better-sqlite3") +const migrate = require("../db/migrate") const HeatSync = require("heatsync") +const {test} = require("supertape") +const data = require("./data") const config = require("../config") const passthrough = require("../passthrough") @@ -11,12 +15,15 @@ const db = new sqlite(":memory:") const reg = require("../matrix/read-registration") reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded -db.exec(fs.readFileSync("db/ooye-schema.sql", "utf8")) -db.exec(fs.readFileSync("db/ooye-test-data.sql", "utf8")) - const sync = new HeatSync({watchFS: false}) -Object.assign(passthrough, { config, sync, db }) +const discord = { + guilds: new Map([ + [data.guild.general.id, data.guild.general] + ]) +} + +Object.assign(passthrough, { discord, config, sync, db }) const orm = sync.require("../db/orm") passthrough.from = orm.from @@ -25,17 +32,27 @@ passthrough.select = orm.select const file = sync.require("../matrix/file") file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) } -require("../db/orm.test") -require("../matrix/kstate.test") -require("../matrix/api.test") -require("../matrix/read-registration.test") -require("../matrix/txnid.test") -require("../d2m/converters/message-to-event.test") -require("../d2m/converters/message-to-event.embeds.test") -require("../d2m/converters/edit-to-changes.test") -require("../d2m/converters/thread-to-announcement.test") -require("../d2m/actions/create-room.test") -require("../d2m/converters/user-to-mxid.test") -require("../d2m/actions/register-user.test") -require("../m2d/converters/event-to-message.test") -require("../m2d/converters/utils.test") +;(async () => { + const p = migrate.migrate(db) + test("migrate: migration works", async t => { + await p + t.pass("it did not throw an error") + }) + await p + db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8")) + require("../db/orm.test") + require("../matrix/kstate.test") + require("../matrix/api.test") + require("../matrix/file.test") + require("../matrix/read-registration.test") + require("../matrix/txnid.test") + require("../d2m/converters/message-to-event.test") + require("../d2m/converters/message-to-event.embeds.test") + require("../d2m/converters/edit-to-changes.test") + require("../d2m/converters/thread-to-announcement.test") + require("../d2m/actions/create-room.test") + require("../d2m/converters/user-to-mxid.test") + require("../d2m/actions/register-user.test") + require("../m2d/converters/event-to-message.test") + require("../m2d/converters/utils.test") +})()