Add database migrations

d2m/actions/register-user.js

@@ -11,6 +11,10 @@ const api = sync.require("../../matrix/api")
 const file = sync.require("../../matrix/file")
 /** @type {import("../converters/user-to-mxid")} */
 const userToMxid = sync.require("../converters/user-to-mxid")
+/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
+let hasher = null
+// @ts-ignore
+require("xxhash-wasm")().then(h => hasher = h)
 
 /**
  * A sim is an account that is being simulated by the bridge to copy events from the other side.
@@ -120,7 +124,9 @@ async function memberToStateContent(user, member, guildID) {
 }
 
 function hashProfileContent(content) {
-	return `${content.displayname}\u0000${content.avatar_url}`
+	const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
+	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
+	return signedHash
 }
 
 /**
@@ -137,11 +143,11 @@ async function syncUser(user, member, guildID, roomID) {
 	const mxid = await ensureSimJoined(user, roomID)
 	const content = await memberToStateContent(user, member, guildID)
 	const currentHash = hashProfileContent(content)
-	const existingHash = select("sim_member", "profile_event_content_hash", "WHERE room_id = ? AND mxid = ?").pluck().get(roomID, mxid)
+	const existingHash = select("sim_member", "hashed_profile_content", "WHERE room_id = ? AND mxid = ?").safeIntegers().pluck().get(roomID, mxid)
 	// only do the actual sync if the hash has changed since we last looked
 	if (existingHash !== currentHash) {
 		await api.sendState(roomID, "m.room.member", mxid, content, mxid)
-		db.prepare("UPDATE sim_member SET profile_event_content_hash = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
+		db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
 	}
 	return mxid
 }

db/migrate.js

@@ -0,0 +1,42 @@
+// @ts-check
+
+const fs = require("fs")
+const {join} = require("path")
+
+async function migrate(db) {
+	let files = fs.readdirSync(join(__dirname, "migrations"))
+	files = files.sort()
+	db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run()
+	let progress = db.prepare("SELECT * FROM migration").pluck().get()
+	if (!progress) {
+		progress = ""
+		db.prepare("INSERT INTO migration VALUES ('')").run()
+	}
+
+	let migrationRan = false
+
+	for (const filename of files) {
+		if (progress >= filename) continue
+		console.log(`Applying database migration ${filename}`)
+		if (filename.endsWith(".sql")) {
+			const sql = fs.readFileSync(join(__dirname, "migrations", filename), "utf8")
+			db.exec(sql)
+		} else if (filename.endsWith(".js")) {
+			await require("./" + join("migrations", filename))(db)
+		} else {
+			continue
+		}
+
+		migrationRan = true
+		db.transaction(() => {
+			db.prepare("DELETE FROM migration").run()
+			db.prepare("INSERT INTO migration VALUES (?)").run(filename)
+		})()
+	}
+
+	if (migrationRan) {
+		console.log("Database migrations all done.")
+	}
+}
+
+module.exports.migrate = migrate

db/migrations/0001-schema.sql

@@ -1,3 +1,5 @@
+BEGIN TRANSACTION;
+
 CREATE TABLE IF NOT EXISTS "sim" (
 	"discord_id"	TEXT NOT NULL,
 	"sim_name"	TEXT NOT NULL UNIQUE,
@@ -86,3 +88,5 @@ CREATE TABLE IF NOT EXISTS "reaction" (
 	"encoded_emoji"	TEXT NOT NULL,
 	PRIMARY KEY ("hashed_event_id")
 ) WITHOUT ROWID;
+
+COMMIT;

db/migrations/0002-optimise-profile-content.sql

@@ -1,3 +1,5 @@
+BEGIN TRANSACTION;
+
 -- Change hashed_profile_content column affinity to INTEGER
 
 CREATE TABLE "new_sim_member" (
@@ -13,4 +15,6 @@ DROP TABLE sim_member;
 
 ALTER TABLE new_sim_member RENAME TO sim_member;
 
+COMMIT;
+
 VACUUM;

db/migrations/0002-optimise-profile-content.up.js

@@ -1,13 +1,13 @@
 module.exports = async function(db) {
 	const hasher = await require("xxhash-wasm")()
-	const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member").pluck().all()
+	const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all()
 	const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?")
 	db.transaction(() => {
-		for (const s of contents) {
+		for (let s of contents) {
-			if (!Buffer.isBuffer(s)) s = Buffer.from(s)
+			let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s))
-			const unsignedHash = hasher.h64(eventID)
+			const unsignedHash = hasher.h64Raw(b)
 			const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
-			stmt.run(s, signedHash)
+			stmt.run(signedHash, s)
 		}
 	})()
 }

db/orm-utils.d.ts

@@ -54,7 +54,7 @@ export type Models = {
 	sim_member: {
 		mxid: string
 		room_id: string
-		profile_event_content_hash: any
+		hashed_profile_content: number
 	}
 
 	webhook: {
@@ -79,8 +79,9 @@ export type Models = {
 
 export type Prepared<Row> = {
 	pluck: () => Prepared<Row[keyof Row]>
+	safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
 	all: (..._: any[]) => Row[]
-	get: (..._: any[]) => Row?
+	get: (..._: any[]) => Row | null
 }
 
 export type AllKeys<U> = U extends any ? keyof U : never

scripts/migrate-from-old-bridge.js

@@ -20,7 +20,7 @@ const newAT = reg.as_token
 const oldDB = new sqlite(reg.old_bridge.database)
 const db = new sqlite("db/ooye.db")
 
-db.exec(`CREATE TABLE IF NOT EXISTS migration (
+db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration (
 	discord_channel	TEXT NOT NULL,
 	migrated	INTEGER NOT NULL,
 	PRIMARY KEY("discord_channel")
@@ -69,7 +69,7 @@ async function migrateGuild(guild) {
 	const spaceID = await createSpace.syncSpace(guild)
 
 	let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id)
-	const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all()
+	const migrated = db.prepare("SELECT discord_channel FROM half_shot_migration WHERE migrated = 1").pluck().all()
 	oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel))
 	console.log("Found these rooms which can be migrated:")
 	console.log(oldRooms)
@@ -128,7 +128,7 @@ async function migrateGuild(guild) {
 		await createRoom.syncRoom(row.discord_channel)
 		console.log(`-- -- Finished syncing`)
 
-		db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
+		db.prepare("INSERT INTO half_shot_migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
 	}
 
 	// Step 5: Call syncSpace to make sure everything is up to date

start.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 const sqlite = require("better-sqlite3")
+const migrate = require("./db/migrate")
 const HeatSync = require("heatsync")
 
 const config = require("./config")
@@ -30,6 +31,7 @@ discord.snow.requestHandler.on("requestError", data => {
 })
 
 ;(async () => {
+	await migrate.migrate(db)
 	await discord.cloud.connect()
 	console.log("Discord gateway started")
 

test/data.js

@@ -93,7 +93,28 @@ module.exports = {
 			afk_timeout: 300,
 			id: "112760669178241024",
 			icon: "a_f83622e09ead74f0c5c527fe241f8f8c",
-			emojis: [],
+			emojis: [
+				{
+					version: 0,
+					roles: [],
+					require_colons: true,
+					name: "hippo",
+					managed: false,
+					id: "230201364309868544",
+					available: true,
+					animated: false
+				},
+				{
+					version: 0,
+					roles: [],
+					require_colons: true,
+					name: "hipposcope",
+					managed: false,
+					id: "393635038903926784",
+					available: true,
+					animated: true
+				}
+			],
 			premium_subscription_count: 14,
 			roles: [],
 			discovery_splash: null,
@@ -1133,6 +1154,139 @@ module.exports = {
 				}
 			},
 			webhook_id: "1109360903096369153"
+		},
+		reply_with_only_embed: {
+			type: 19,
+			tts: false,
+			timestamp: "2023-09-29T20:44:42.606000+00:00",
+			referenced_message: {
+				type: 19,
+				tts: false,
+				timestamp: "2023-09-29T20:44:42.204000+00:00",
+				pinned: false,
+				message_reference: {
+					message_id: "1157413453921787924",
+					guild_id: "1150201337112449045",
+					channel_id: "1150208267285434429"
+				},
+				mentions: [
+					{
+						username: "goat_six",
+						public_flags: 64,
+						id: "334539029879980040",
+						global_name: "GoatSixx",
+						discriminator: "0",
+						avatar_decoration_data: null,
+						avatar: "fd87e077c6ebe4239ce573bae083ed66"
+					}
+				],
+				mention_roles: [],
+				mention_everyone: false,
+				id: "1157417694728044624",
+				flags: 0,
+				embeds: [
+					{
+						url: "https://twitter.com/dynastic/status/1707484191963648161",
+						type: "rich",
+						timestamp: "2023-09-28T19:55:29.543000+00:00",
+						reference_id: "1157417694728044624",
+						footer: {
+							text: "Twitter",
+							proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png",
+							icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png"
+						},
+						fields: [
+							{ value: "119", name: "Retweets", inline: true },
+							{ value: "5581", name: "Likes", inline: true }
+						],
+						description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?",
+						color: 1942002,
+						author: {
+							url: "https://twitter.com/dynastic",
+							proxy_icon_url: "https://images-ext-2.discordapp.net/external/06UZNFT37nepFbzmK2FN4q-9DO_UeSaOaZQICSiMexU/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg",
+							name: "dynastic (@dynastic)",
+							icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg"
+						}
+					}
+				],
+				edited_timestamp: null,
+				content: "https://twitter.com/dynastic/status/1707484191963648161",
+				components: [],
+				channel_id: "1150208267285434429",
+				author: {
+					username: "pokemongod",
+					public_flags: 0,
+					id: "66255093481082800",
+					global_name: "PokemonGod",
+					discriminator: "0",
+					avatar_decoration_data: null,
+					avatar: "0cab06c4256499749cbdd4561c629f84"
+				},
+				attachments: []
+			},
+			pinned: false,
+			message_reference: {
+				message_id: "1157417694728044624",
+				guild_id: "1150201337112449045",
+				channel_id: "1150208267285434429"
+			},
+			mentions: [],
+			mention_roles: [],
+			mention_everyone: false,
+			member: {
+				roles: [ "1153875112832008212" ],
+				premium_since: null,
+				pending: false,
+				nick: null,
+				mute: false,
+				joined_at: "2023-09-20T02:07:44.874994+00:00",
+				flags: 0,
+				deaf: false,
+				communication_disabled_until: null,
+				avatar: null
+			},
+			id: "1157417696414150778",
+			flags: 0,
+			embeds: [
+				{
+					url: "https://twitter.com/i/status/1707484191963648161",
+					type: "rich",
+					timestamp: "2023-09-28T19:55:29+00:00",
+					footer: {
+						text: "Twitter",
+						proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png",
+						icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png"
+					},
+					fields: [
+						{ value: "119", name: "Retweets", inline: true },
+						{ value: "5581", name: "Likes", inline: true }
+					],
+					description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?",
+					color: 1942002,
+					author: {
+						url: "https://twitter.com/i/user/719631291747078145",
+						proxy_icon_url: "https://images-ext-1.discordapp.net/external/6LgXrIifZ-MhwPPiwqAomgoy93d932jZiJqLCAf79Fw/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg",
+						name: "dynastic (@dynastic)",
+						icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg"
+					}
+				}
+			],
+			edited_timestamp: null,
+			content: "",
+			components: [],
+			channel_id: "1150208267285434429",
+			author: {
+				username: "Twitter Video Embeds",
+				public_flags: 65536,
+				id: "842601826674540574",
+				global_name: null,
+				discriminator: "4945",
+				bot: true,
+				avatar_decoration_data: null,
+				avatar: "6ed5bf10f953b22d47893b4655705b30"
+			},
+			attachments: [],
+			guild_id: "1150201337112449045"
 		}
 	},
 	message_update: {

test/ooye-test-data.sql

@@ -17,7 +17,7 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES
 ('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'),
 ('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');;
 
-INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES
+INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES
 ('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL);
 
 INSERT INTO message_channel (message_id, channel_id) VALUES

test/test.js

@@ -1,8 +1,12 @@
 // @ts-check
 
 const fs = require("fs")
+const {join} = require("path")
 const sqlite = require("better-sqlite3")
+const migrate = require("../db/migrate")
 const HeatSync = require("heatsync")
+const {test} = require("supertape")
+const data = require("./data")
 
 const config = require("../config")
 const passthrough = require("../passthrough")
@@ -11,12 +15,15 @@ const db = new sqlite(":memory:")
 const reg = require("../matrix/read-registration")
 reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded
 
-db.exec(fs.readFileSync("db/ooye-schema.sql", "utf8"))
-db.exec(fs.readFileSync("db/ooye-test-data.sql", "utf8"))
-
 const sync = new HeatSync({watchFS: false})
 
-Object.assign(passthrough, { config, sync, db })
+const discord = {
+	guilds: new Map([
+		[data.guild.general.id, data.guild.general]
+	])
+}
+
+Object.assign(passthrough, { discord, config, sync, db })
 
 const orm = sync.require("../db/orm")
 passthrough.from = orm.from
@@ -25,17 +32,27 @@ passthrough.select = orm.select
 const file = sync.require("../matrix/file")
 file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) }
 
-require("../db/orm.test")
+;(async () => {
-require("../matrix/kstate.test")
+	const p = migrate.migrate(db)
-require("../matrix/api.test")
+	test("migrate: migration works", async t => {
-require("../matrix/read-registration.test")
+		await p
-require("../matrix/txnid.test")
+		t.pass("it did not throw an error")
-require("../d2m/converters/message-to-event.test")
+	})
-require("../d2m/converters/message-to-event.embeds.test")
+	await p
-require("../d2m/converters/edit-to-changes.test")
+	db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8"))
-require("../d2m/converters/thread-to-announcement.test")
+	require("../db/orm.test")
-require("../d2m/actions/create-room.test")
+	require("../matrix/kstate.test")
-require("../d2m/converters/user-to-mxid.test")
+	require("../matrix/api.test")
-require("../d2m/actions/register-user.test")
+	require("../matrix/file.test")
-require("../m2d/converters/event-to-message.test")
+	require("../matrix/read-registration.test")
-require("../m2d/converters/utils.test")
+	require("../matrix/txnid.test")
+	require("../d2m/converters/message-to-event.test")
+	require("../d2m/converters/message-to-event.embeds.test")
+	require("../d2m/converters/edit-to-changes.test")
+	require("../d2m/converters/thread-to-announcement.test")
+	require("../d2m/actions/create-room.test")
+	require("../d2m/converters/user-to-mxid.test")
+	require("../d2m/actions/register-user.test")
+	require("../m2d/converters/event-to-message.test")
+	require("../m2d/converters/utils.test")
+})()