Move everything to src folder... it had to happen

src/db/migrate.js

@@ -0,0 +1,42 @@
+// @ts-check
+
+const fs = require("fs")
+const {join} = require("path")
+
+async function migrate(db) {
+	let files = fs.readdirSync(join(__dirname, "migrations"))
+	files = files.sort()
+	db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run()
+	let progress = db.prepare("SELECT * FROM migration").pluck().get()
+	if (!progress) {
+		progress = ""
+		db.prepare("INSERT INTO migration VALUES ('')").run()
+	}
+
+	let migrationRan = false
+
+	for (const filename of files) {
+		if (progress >= filename) continue
+		console.log(`Applying database migration ${filename}`)
+		if (filename.endsWith(".sql")) {
+			const sql = fs.readFileSync(join(__dirname, "migrations", filename), "utf8")
+			db.exec(sql)
+		} else if (filename.endsWith(".js")) {
+			await require("./" + join("migrations", filename))(db)
+		} else {
+			continue
+		}
+
+		migrationRan = true
+		db.transaction(() => {
+			db.prepare("DELETE FROM migration").run()
+			db.prepare("INSERT INTO migration VALUES (?)").run(filename)
+		})()
+	}
+
+	if (migrationRan) {
+		console.log("Database migrations all done.")
+	}
+}
+
+module.exports.migrate = migrate

src/db/migrations/0001-schema.sql

@@ -0,0 +1,92 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "sim" (
+	"discord_id"	TEXT NOT NULL,
+	"sim_name"	TEXT NOT NULL UNIQUE,
+	"localpart"	TEXT NOT NULL,
+	"mxid"	TEXT NOT NULL,
+	PRIMARY KEY("discord_id")
+);
+
+CREATE TABLE IF NOT EXISTS "webhook" (
+	"channel_id"	TEXT NOT NULL,
+	"webhook_id"	TEXT NOT NULL,
+	"webhook_token"	TEXT NOT NULL,
+	PRIMARY KEY("channel_id")
+);
+
+CREATE TABLE IF NOT EXISTS "sim_member" (
+	"mxid"	TEXT NOT NULL,
+	"room_id"	TEXT NOT NULL,
+	"profile_event_content_hash"	BLOB,
+	PRIMARY KEY("room_id","mxid")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "member_cache" (
+	"room_id"	TEXT NOT NULL,
+	"mxid"	TEXT NOT NULL,
+	"displayname"	TEXT,
+	"avatar_url"	TEXT,
+	PRIMARY KEY("room_id","mxid")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "file" (
+	"discord_url"	TEXT NOT NULL,
+	"mxc_url"	TEXT NOT NULL,
+	PRIMARY KEY("discord_url")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "guild_space" (
+	"guild_id"	TEXT NOT NULL,
+	"space_id"	TEXT NOT NULL,
+	PRIMARY KEY("guild_id")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "channel_room" (
+	"channel_id"	TEXT NOT NULL,
+	"room_id"	TEXT NOT NULL,
+	"name"	TEXT NOT NULL,
+	"nick"	TEXT,
+	"thread_parent"	TEXT,
+	"custom_avatar"	TEXT,
+	PRIMARY KEY("channel_id","room_id")
+);
+
+CREATE TABLE IF NOT EXISTS "message_channel" (
+	"message_id"	TEXT NOT NULL,
+	"channel_id"	TEXT NOT NULL,
+	PRIMARY KEY("message_id")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "event_message" (
+	"event_id"	TEXT NOT NULL,
+	"message_id"	TEXT NOT NULL,
+	"event_type"	TEXT,
+	"event_subtype"	TEXT,
+	"part"	INTEGER NOT NULL,
+	"source"	INTEGER NOT NULL,
+	PRIMARY KEY("message_id","event_id")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "lottie" (
+	"id"	TEXT NOT NULL,
+	"mxc"	TEXT NOT NULL,
+	PRIMARY KEY("id")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "emoji" (
+	"id"	TEXT NOT NULL,
+	"name"	TEXT NOT NULL,
+	"animated"	INTEGER NOT NULL,
+	"mxc_url"	TEXT NOT NULL,
+	PRIMARY KEY("id")
+) WITHOUT ROWID;
+
+CREATE TABLE IF NOT EXISTS "reaction" (
+	"hashed_event_id"	INTEGER NOT NULL,
+	"message_id"	TEXT NOT NULL,
+	"encoded_emoji"	TEXT NOT NULL,
+	PRIMARY KEY ("hashed_event_id")
+) WITHOUT ROWID;
+
+COMMIT;

src/db/migrations/0002-optimise-profile-content.sql

@@ -0,0 +1,20 @@
+BEGIN TRANSACTION;
+
+-- Change hashed_profile_content column affinity to INTEGER
+
+CREATE TABLE "new_sim_member" (
+	"mxid"	TEXT NOT NULL,
+	"room_id"	TEXT NOT NULL,
+	"hashed_profile_content"	INTEGER,
+	PRIMARY KEY("room_id","mxid")
+) WITHOUT ROWID;
+
+INSERT INTO new_sim_member SELECT * FROM sim_member;
+
+DROP TABLE sim_member;
+
+ALTER TABLE new_sim_member RENAME TO sim_member;
+
+COMMIT;
+
+VACUUM;

src/db/migrations/0002-optimise-profile-content.up.js

@@ -0,0 +1,13 @@
+module.exports = async function(db) {
+	const hasher = await require("xxhash-wasm")()
+	const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all()
+	const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?")
+	db.transaction(() => {
+		for (let s of contents) {
+			let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s))
+			const unsignedHash = hasher.h64Raw(b)
+			const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
+			stmt.run(signedHash, s)
+		}
+	})()
+}

src/db/migrations/0003-distinguish-column-names.sql

@@ -0,0 +1,19 @@
+BEGIN TRANSACTION;
+
+-- Rename mxc to mxc_url for consistency
+
+ALTER TABLE lottie RENAME COLUMN mxc TO mxc_url;
+
+-- Rename id to sticker_id so joins make sense in the future
+
+ALTER TABLE lottie RENAME COLUMN id TO sticker_id;
+
+-- Rename discord_id to user_id so joins make sense in the future
+
+ALTER TABLE sim RENAME COLUMN discord_id TO user_id;
+
+-- Rename id to emoji_id so joins make sense in the future
+
+ALTER TABLE emoji RENAME COLUMN id TO emoji_id;
+
+COMMIT;

src/db/migrations/0004-auto-emoji-guild.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE auto_emoji (
+	name	TEXT NOT NULL,
+	emoji_id	TEXT NOT NULL,
+	guild_id	TEXT NOT NULL,
+	PRIMARY KEY (name)
+) WITHOUT ROWID;
+
+COMMIT;

src/db/migrations/0005-clear-member-cache.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+DELETE FROM member_cache;
+
+COMMIT;

src/db/migrations/0006-add-privacy-to-space.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE guild_space ADD COLUMN privacy_level INTEGER NOT NULL DEFAULT 0;
+
+COMMIT;

src/db/migrations/0007-split-part-and-reaction-part.sql

@@ -0,0 +1,24 @@
+BEGIN TRANSACTION;
+
+-- Add column reaction_part to event_message, copying the existing value from part
+
+CREATE TABLE "new_event_message" (
+	"event_id"	TEXT NOT NULL,
+	"event_type"	TEXT,
+	"event_subtype"	TEXT,
+	"message_id"	TEXT NOT NULL,
+	"part"	INTEGER NOT NULL,
+	"reaction_part" INTEGER NOT NULL,
+	"source"	INTEGER NOT NULL,
+	PRIMARY KEY("message_id","event_id")
+) WITHOUT ROWID;
+
+INSERT INTO new_event_message SELECT event_id, event_type, event_subtype, message_id, part, part, source FROM event_message;
+
+DROP TABLE event_message;
+
+ALTER TABLE new_event_message RENAME TO event_message;
+
+COMMIT;
+
+VACUUM;

src/db/migrations/0008-add-last-bridged-pin-timestamp.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE channel_room ADD COLUMN last_bridged_pin_timestamp INTEGER;
+
+COMMIT;

src/db/migrations/0009-add-speedbump-id.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE channel_room ADD COLUMN speedbump_id TEXT;
+ALTER TABLE channel_room ADD COLUMN speedbump_webhook_id TEXT;
+ALTER TABLE channel_room ADD COLUMN speedbump_checked INTEGER;
+
+COMMIT;

src/db/migrations/0010-add-sim-proxy.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS sim_proxy (
+	user_id TEXT NOT NULL,
+	proxy_owner_id TEXT NOT NULL,
+	displayname TEXT NOT NULL,
+	PRIMARY KEY(user_id)
+) WITHOUT ROWID;

src/db/migrations/0011-move-bridge-bot-to-real-id.up.js

@@ -0,0 +1,16 @@
+/*
+	a. If the bridge bot sim already has the correct ID:
+		- No rows updated.
+
+	b. If the bridge bot sim has the wrong ID but there's no duplicate:
+		- One row updated.
+
+	c. If the bridge bot sim has the wrong ID and there's a duplicate:
+		- One row updated (replaces an existing row).
+*/
+
+module.exports = async function(db) {
+	const config = require("../../config")
+	const id = Buffer.from(config.discordToken.split(".")[0], "base64").toString()
+	db.prepare("UPDATE OR REPLACE sim SET user_id = ? WHERE user_id = '0'").run(id)
+}

src/db/migrations/0012-add-member-power.sql

@@ -0,0 +1,14 @@
+BEGIN TRANSACTION;
+
+-- the power we want them to have
+CREATE TABLE IF NOT EXISTS member_power (
+	mxid			TEXT NOT NULL,
+	room_id		TEXT NOT NULL,
+	power_level	INTEGER NOT NULL,
+	PRIMARY KEY(mxid, room_id)
+) WITHOUT ROWID;
+
+-- the power they have
+ALTER TABLE member_cache ADD COLUMN power_level INTEGER NOT NULL DEFAULT 0;
+
+COMMIT;

src/db/orm-defs.d.ts

@@ -0,0 +1,117 @@
+export type Models = {
+	channel_room: {
+		channel_id: string
+		room_id: string
+		name: string
+		nick: string | null
+		thread_parent: string | null
+		custom_avatar: string | null
+		last_bridged_pin_timestamp: number | null
+		speedbump_id: string | null
+		speedbump_webhook_id: string | null
+		speedbump_checked: number | null
+	}
+
+	event_message: {
+		event_id: string
+		message_id: string
+		event_type: string | null
+		event_subtype: string | null
+		part: number
+		reaction_part: number
+		source: number
+	}
+
+	file: {
+		discord_url: string
+		mxc_url: string
+	}
+
+	guild_space: {
+		guild_id: string
+		space_id: string
+		privacy_level: number
+	}
+
+	lottie: {
+		sticker_id: string
+		mxc_url: string
+	}
+
+	member_cache: {
+		room_id: string
+		mxid: string
+		displayname: string | null
+		avatar_url: string | null,
+		power_level: number
+	}
+
+	member_power: {
+		mxid: string
+		room_id: string
+		power_level: number
+	}
+
+	message_channel: {
+		message_id: string
+		channel_id: string
+	}
+
+	sim: {
+		user_id: string
+		sim_name: string
+		localpart: string
+		mxid: string
+	}
+
+	sim_member: {
+		mxid: string
+		room_id: string
+		hashed_profile_content: number
+	}
+
+	sim_proxy: {
+		user_id: string
+		proxy_owner_id: string
+		displayname: string
+	}
+
+	webhook: {
+		channel_id: string
+		webhook_id: string
+		webhook_token: string
+	}
+
+	emoji: {
+		emoji_id: string
+		name: string
+		animated: number
+		mxc_url: string
+	}
+
+	reaction: {
+		hashed_event_id: number
+		message_id: string
+		encoded_emoji: string
+	}
+
+	auto_emoji: {
+		name: string
+		emoji_id: string
+		guild_id: string
+	}
+}
+
+export type Prepared<Row> = {
+	pluck: () => Prepared<Row[keyof Row]>
+	safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
+	raw: () => Prepared<Row[keyof Row][]>
+	all: (..._: any[]) => Row[]
+	get: (..._: any[]) => Row | null | undefined
+}
+
+export type AllKeys<U> = U extends any ? keyof U : never
+export type PickTypeOf<T, K extends AllKeys<T>> = T extends { [k in K]?: any } ? T[K] : never
+export type Merge<U> = {[x in AllKeys<U>]: PickTypeOf<U, x>}
+export type Nullable<T> = {[k in keyof T]: T[k] | null}
+export type Numberish<T> = {[k in keyof T]: T[k] extends number ? (number | bigint) : T[k]}

src/db/orm-defs.js

@@ -0,0 +1 @@
+module.exports = {}

src/db/orm.js

@@ -0,0 +1,184 @@
+// @ts-check
+
+const {db} = require("../passthrough")
+const U = require("./orm-defs")
+
+/**
+ * @template {keyof U.Models} Table
+ * @template {keyof U.Models[Table]} Col
+ * @param {Table} table
+ * @param {Col[] | Col} cols
+ * @param {Partial<U.Numberish<U.Models[Table]>>} where
+ * @param {string} [e]
+ */
+function select(table, cols, where = {}, e = "") {
+	if (!Array.isArray(cols)) cols = [cols]
+	const parameters = []
+	const wheres = Object.entries(where).map(([col, value]) => {
+		parameters.push(value)
+		return `"${col}" = ?`
+	})
+	const whereString = wheres.length ? " WHERE " + wheres.join(" AND ") : ""
+	/** @type {U.Prepared<Pick<U.Models[Table], Col>>} */
+	const prepared = db.prepare(`SELECT ${cols.map(k => `"${String(k)}"`).join(", ")} FROM ${table} ${whereString} ${e}`)
+	prepared.get = prepared.get.bind(prepared, ...parameters)
+	prepared.all = prepared.all.bind(prepared, ...parameters)
+	return prepared
+}
+
+/**
+ * @template {keyof U.Models} Table
+ * @template {keyof U.Merge<U.Models[Table]>} Col
+ */
+class From {
+	/**
+	 * @param {Table} table
+	 */
+	constructor(table) {
+		/** @private @type {Table[]} */
+		this.tables = [table]
+		/** @private */
+		this.directions = []
+		/** @private */
+		this.sql = ""
+		/** @private */
+		this.cols = []
+		/** @private */
+		this.makeColsSafe = true
+		/** @private */
+		this.using = []
+		/** @private */
+		this.isPluck = false
+		/** @private */
+		this.parameters = []
+	}
+
+	/**
+	 * @template {keyof U.Models} Table2
+	 * @param {Table2} table
+	 * @param {Col & (keyof U.Models[Table2])} col
+	 * @param {"inner" | "left"} [direction]
+	 */
+	join(table, col, direction = "inner") {
+		/** @type {From<Table | Table2, keyof U.Merge<U.Models[Table | Table2]>>} */
+		// @ts-ignore
+		const r = this
+		r.tables.push(table)
+		r.directions.push(direction.toUpperCase())
+		r.using.push(col)
+		return r
+	}
+
+	/**
+	 * @template {Col} Select
+	 * @param {Col[] | Select[]} cols
+	 */
+	select(...cols) {
+		/** @type {From<Table, Select>} */
+		const r = this
+		r.cols = cols
+		return r
+	}
+
+	selectUnsafe(...cols) {
+		this.cols = cols
+		this.makeColsSafe = false
+		return this
+	}
+
+	/**
+	 * @template {Col} Select
+	 * @param {Select} col
+	 */
+	pluck(col) {
+		/** @type {Pluck<Table, Select>} */
+		// @ts-ignore
+		const r = this
+		r.cols = [col]
+		r.isPluck = true
+		return r
+	}
+
+	/**
+	 * @param {string} sql
+	 */
+	and(sql) {
+		this.sql += " " + sql
+		return this
+	}
+
+	/**
+	 * @param {Partial<U.Numberish<U.Models[Table]>>} conditions
+	 */
+	where(conditions) {
+		const wheres = Object.entries(conditions).map(([col, value]) => {
+			this.parameters.push(value)
+			return `"${col}" = ?`
+		})
+		this.sql += " WHERE " + wheres.join(" AND ")
+		return this
+	}
+
+	prepare() {
+		if (this.makeColsSafe) this.cols = this.cols.map(k => `"${k}"`)
+		let sql = `SELECT ${this.cols.join(", ")} FROM ${this.tables[0]} `
+		for (let i = 1; i < this.tables.length; i++) {
+			const table = this.tables[i]
+			const col = this.using[i-1]
+			const direction = this.directions[i-1]
+			sql += `${direction} JOIN ${table} USING (${col}) `
+		}
+		sql += this.sql
+		/** @type {U.Prepared<Pick<U.Merge<U.Models[Table]>, Col>>} */
+		let prepared = db.prepare(sql)
+		if (this.isPluck) prepared = prepared.pluck()
+		return prepared
+	}
+
+	get(..._) {
+		const prepared = this.prepare()
+		return prepared.get(...this.parameters, ..._)
+	}
+
+	all(..._) {
+		const prepared = this.prepare()
+		return prepared.all(...this.parameters, ..._)
+	}
+}
+
+/* c8 ignore start - this code is only used for types and does not actually execute */
+/**
+ * @template {keyof U.Models} Table
+ * @template {keyof U.Merge<U.Models[Table]>} Col
+ */
+class Pluck extends From {
+	// @ts-ignore
+	prepare() {
+		/** @type {U.Prepared<U.Merge<U.Models[Table]>[Col]>} */
+		// @ts-ignore
+		const prepared = super.prepare()
+		return prepared
+	}
+
+	get(..._) {
+		const prepared = this.prepare()
+		return prepared.get(..._)
+	}
+
+	all(..._) {
+		const prepared = this.prepare()
+		return prepared.all(..._)
+	}
+}
+/* c8 ignore stop */
+
+/**
+ * @template {keyof U.Models} Table
+ * @param {Table} table
+ */
+function from(table) {
+	return new From(table)
+}
+
+module.exports.from = from
+module.exports.select = select

src/db/orm.test.js

@@ -0,0 +1,55 @@
+// @ts-check
+
+const {test} = require("supertape")
+const data = require("../test/data")
+
+const {db, select, from} = require("../passthrough")
+
+test("orm: select: get works", t => {
+	const row = select("guild_space", "guild_id", {}, "WHERE space_id = ?").get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
+	t.equal(row?.guild_id, data.guild.general.id)
+})
+
+test("orm: from: get works", t => {
+	const row = from("guild_space").select("guild_id").and("WHERE space_id = ?").get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
+	t.equal(row?.guild_id, data.guild.general.id)
+})
+
+test("orm: select: get pluck works", t => {
+	const guildID = select("guild_space", "guild_id", {}, "WHERE space_id = ?").pluck().get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
+	t.equal(guildID, data.guild.general.id)
+})
+
+test("orm: select: get, where and pluck works", t => {
+	const channelID = select("message_channel", "channel_id", {message_id: "1128118177155526666"}).pluck().get()
+	t.equal(channelID, "112760669178241024")
+})
+
+test("orm: select: all, where and pluck works on multiple columns", t => {
+	const names = select("member_cache", "displayname", {room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", mxid: "@cadence:cadence.moe"}).pluck().all()
+	t.deepEqual(names, ["cadence [they]"])
+})
+
+test("orm: from: get pluck works", t => {
+	const guildID = from("guild_space").pluck("guild_id").and("WHERE space_id = ?").get("!jjWAGMeQdNrVZSSfvz:cadence.moe")
+	t.equal(guildID, data.guild.general.id)
+})
+
+test("orm: from: join and pluck works", t => {
+	const mxid = from("sim").join("sim_member", "mxid").and("WHERE user_id = ? AND room_id = ?").pluck("mxid").get("771520384671416320", "!hYnGGlPHlbujVVfktC:cadence.moe")
+	t.equal(mxid, "@_ooye_bojack_horseman:cadence.moe")
+})
+
+test("orm: from: where and pluck works", t => {
+	const subtypes = from("event_message").where({message_id: "1141501302736695316"}).pluck("event_subtype").all()
+	t.deepEqual(subtypes.sort(), ["m.image", "m.text"])
+})
+
+test("orm: from: join direction works", t => {
+	const hasOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "_pk_zoego"}).get()
+	t.deepEqual(hasOwner, {user_id: "43d378d5-1183-47dc-ab3c-d14e21c3fe58", proxy_owner_id: "196188877885538304"})
+	const hasNoOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
+	t.deepEqual(hasNoOwner, {user_id: "820865262526005258", proxy_owner_id: null})
+	const hasNoOwnerInner = from("sim").join("sim_proxy", "user_id", "inner").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get()
+	t.deepEqual(hasNoOwnerInner, undefined)
+})