Add database migrations

db/migrate.js

@@ -0,0 +1,42 @@
+// @ts-check
+
+const fs = require("fs")
+const {join} = require("path")
+
+async function migrate(db) {
+	let files = fs.readdirSync(join(__dirname, "migrations"))
+	files = files.sort()
+	db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run()
+	let progress = db.prepare("SELECT * FROM migration").pluck().get()
+	if (!progress) {
+		progress = ""
+		db.prepare("INSERT INTO migration VALUES ('')").run()
+	}
+
+	let migrationRan = false
+
+	for (const filename of files) {
+		if (progress >= filename) continue
+		console.log(`Applying database migration ${filename}`)
+		if (filename.endsWith(".sql")) {
+			const sql = fs.readFileSync(join(__dirname, "migrations", filename), "utf8")
+			db.exec(sql)
+		} else if (filename.endsWith(".js")) {
+			await require("./" + join("migrations", filename))(db)
+		} else {
+			continue
+		}
+
+		migrationRan = true
+		db.transaction(() => {
+			db.prepare("DELETE FROM migration").run()
+			db.prepare("INSERT INTO migration VALUES (?)").run(filename)
+		})()
+	}
+
+	if (migrationRan) {
+		console.log("Database migrations all done.")
+	}
+}
+
+module.exports.migrate = migrate

db/migrations/0001-schema.sql

@@ -1,3 +1,5 @@
+BEGIN TRANSACTION;
+
 CREATE TABLE IF NOT EXISTS "sim" (
 	"discord_id"	TEXT NOT NULL,
 	"sim_name"	TEXT NOT NULL UNIQUE,
@@ -86,3 +88,5 @@ CREATE TABLE IF NOT EXISTS "reaction" (
 	"encoded_emoji"	TEXT NOT NULL,
 	PRIMARY KEY ("hashed_event_id")
 ) WITHOUT ROWID;
+
+COMMIT;

db/migrations/0002-optimise-profile-content.sql

@@ -1,3 +1,5 @@
+BEGIN TRANSACTION;
+
 -- Change hashed_profile_content column affinity to INTEGER
 
 CREATE TABLE "new_sim_member" (
@@ -13,4 +15,6 @@ DROP TABLE sim_member;
 
 ALTER TABLE new_sim_member RENAME TO sim_member;
 
+COMMIT;
+
 VACUUM;

db/migrations/0002-optimise-profile-content.up.js

@@ -1,13 +1,13 @@
 module.exports = async function(db) {
 	const hasher = await require("xxhash-wasm")()
-	const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member").pluck().all()
+	const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all()
 	const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?")
 	db.transaction(() => {
-		for (const s of contents) {
-			if (!Buffer.isBuffer(s)) s = Buffer.from(s)
-			const unsignedHash = hasher.h64(eventID)
+		for (let s of contents) {
+			let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s))
+			const unsignedHash = hasher.h64Raw(b)
 			const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
-			stmt.run(s, signedHash)
+			stmt.run(signedHash, s)
 		}
 	})()
 }

db/orm-utils.d.ts

@@ -54,7 +54,7 @@ export type Models = {
 	sim_member: {
 		mxid: string
 		room_id: string
-		profile_event_content_hash: any
+		hashed_profile_content: number
 	}
 
 	webhook: {
@@ -79,8 +79,9 @@ export type Models = {
 
 export type Prepared<Row> = {
 	pluck: () => Prepared<Row[keyof Row]>
+	safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
 	all: (..._: any[]) => Row[]
-	get: (..._: any[]) => Row?
+	get: (..._: any[]) => Row | null
 }
 
 export type AllKeys<U> = U extends any ? keyof U : never