Start moving from SQL to New Funny ORM

d2m/converters/message-to-event.js

@@ -6,7 +6,7 @@ const pb = require("prettier-bytes")
 const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
-const { sync, db, discord } = passthrough
+const {sync, db, discord, select, from} = passthrough
 /** @type {import("../../matrix/file")} */
 const file = sync.require("../../matrix/file")
 /** @type {import("./lottie")} */
@@ -19,7 +19,7 @@ function getDiscordParseCallbacks(message, useHTML) {
 	return {
 		/** @param {{id: string, type: "discordUser"}} node */
 		user: node => {
-			const mxid = db.prepare("SELECT mxid FROM sim WHERE discord_id = ?").pluck().get(node.id)
+			const mxid = select("sim", "mxid", "WHERE discord_id = ?").pluck().get(node.id)
 			const username = message.mentions.find(ment => ment.id === node.id)?.username || node.id
 			if (mxid && useHTML) {
 				return `<a href="https://matrix.to/#/${mxid}">@${username}</a>`
@@ -29,7 +29,7 @@ function getDiscordParseCallbacks(message, useHTML) {
 		},
 		/** @param {{id: string, type: "discordChannel"}} node */
 		channel: node => {
-			const row = db.prepare("SELECT room_id, name, nick FROM channel_room WHERE channel_id = ?").get(node.id)
+			const row = select("channel_room", ["room_id", "name", "nick"], "WHERE channel_id = ?").get(node.id)
 			if (!row) {
 				return `<#${node.id}>` // fallback for when this channel is not bridged
 			} else if (useHTML) {
@@ -81,8 +81,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 		const ref = message.message_reference
 		assert(ref)
 		assert(ref.message_id)
-		const eventID = db.prepare("SELECT event_id FROM event_message WHERE message_id = ?").pluck().get(ref.message_id)
+		const eventID = select("event_message", "event_id", "WHERE message_id = ?").pluck().get(ref.message_id)
-		const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(ref.channel_id)
+		const roomID = select("channel_room", "room_id", "WHERE channel_id = ?").pluck().get(ref.channel_id)
 		if (!eventID || !roomID) return []
 		const event = await di.api.getEvent(roomID, eventID)
 		return [{
@@ -108,10 +108,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 				- So make sure we don't do anything in this case.
 	*/
 	const mentions = {}
-	let repliedToEventId = null
+	let repliedToEventRow = null
-	let repliedToEventRoomId = null
 	let repliedToEventSenderMxid = null
-	let repliedToEventOriginallyFromMatrix = false
 
 	function addMention(mxid) {
 		if (!mentions.user_ids) mentions.user_ids = []
@@ -121,16 +119,14 @@ async function messageToEvent(message, guild, options = {}, di) {
 	// Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions
 	// (Still need to do scenarios 1 and 2 part B, and scenario 3.)
 	if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) {
-		const row = db.prepare("SELECT event_id, room_id, source FROM event_message INNER JOIN message_channel USING (message_id) INNER JOIN channel_room USING (channel_id) WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
+		const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
 		if (row) {
-			repliedToEventId = row.event_id
+			repliedToEventRow = row
-			repliedToEventRoomId = row.room_id
-			repliedToEventOriginallyFromMatrix = row.source === 0 // source 0 = matrix
 		}
 	}
-	if (repliedToEventOriginallyFromMatrix) {
+	if (repliedToEventRow && repliedToEventRow.source === 0) { // reply was originally from Matrix
 		// Need to figure out who sent that event...
-		const event = await di.api.getEvent(repliedToEventRoomId, repliedToEventId)
+		const event = await di.api.getEvent(repliedToEventRow.room_id, repliedToEventRow.event_id)
 		repliedToEventSenderMxid = event.sender
 		// Need to add the sender to m.mentions
 		addMention(repliedToEventSenderMxid)
@@ -147,8 +143,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 	if (message.content) {
 		let content = message.content
 		content = content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => {
-			const eventID = db.prepare("SELECT event_id FROM event_message WHERE message_id = ?").pluck().get(messageID)
+			const eventID = select("event_message", "event_id", "WHERE message_id = ?").pluck().get(messageID)
-			const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(channelID)
+			const roomID = select("channel_room", "room_id", "WHERE channel_id = ?").pluck().get(channelID)
 			if (eventID && roomID) {
 				return `https://matrix.to/#/${roomID}/${eventID}`
 			} else {
@@ -171,7 +167,8 @@ async function messageToEvent(message, guild, options = {}, di) {
 		const matches = [...content.matchAll(/@ ?([a-z0-9._]+)\b/gi)]
 		if (matches.length && matches.some(m => m[1].match(/[a-z]/i))) {
 			const writtenMentionsText = matches.map(m => m[1].toLowerCase())
-			const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(message.channel_id)
+			const roomID = select("channel_room", "room_id", "WHERE channel_id = ?").pluck().get(message.channel_id)
+			assert(roomID)
 			const {joined} = await di.api.getJoinedMembers(roomID)
 			for (const [mxid, member] of Object.entries(joined)) {
 				if (!userRegex.some(rx => mxid.match(rx))) {
@@ -191,10 +188,10 @@ async function messageToEvent(message, guild, options = {}, di) {
 
 		// Fallback body/formatted_body for replies
 		// This branch is optional - do NOT change anything apart from the reply fallback, since it may not be run
-		if (repliedToEventId && options.includeReplyFallback !== false) {
+		if (repliedToEventRow && options.includeReplyFallback !== false) {
 			let repliedToDisplayName
 			let repliedToUserHtml
-			if (repliedToEventOriginallyFromMatrix && repliedToEventSenderMxid) {
+			if (repliedToEventRow?.source === 0 && repliedToEventSenderMxid) {
 				const match = repliedToEventSenderMxid.match(/^@([^:]*)/)
 				assert(match)
 				repliedToDisplayName = match[1] || "a Matrix user" // grab the localpart as the display name, whatever
@@ -214,7 +211,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 				discordOnly: true,
 				escapeHTML: false,
 			}, null, null)
-			html = `<mx-reply><blockquote><a href="https://matrix.to/#/${repliedToEventRoomId}/${repliedToEventId}">In reply to</a> ${repliedToUserHtml}`
+			html = `<mx-reply><blockquote><a href="https://matrix.to/#/${repliedToEventRow.room_id}/${repliedToEventRow.event_id}">In reply to</a> ${repliedToUserHtml}`
 				+ `<br>${repliedToHtml}</blockquote></mx-reply>`
 				+ html
 			body = (`${repliedToDisplayName}: ` // scenario 1 part B for mentions
@@ -383,11 +380,11 @@ async function messageToEvent(message, guild, options = {}, di) {
 	}
 
 	// Rich replies
-	if (repliedToEventId) {
+	if (repliedToEventRow) {
 		Object.assign(events[0], {
 			"m.relates_to": {
 				"m.in_reply_to": {
-					event_id: repliedToEventId
+					event_id: repliedToEventRow.event_id
 				}
 			}
 		})

db/orm-utils.d.ts

@@ -0,0 +1,70 @@
+export type Models = {
+	channel_room: {
+		channel_id: string
+		room_id: string
+		name: string
+		nick: string | null
+		thread_parent: string | null
+		custom_avatar: string | null
+	}
+
+	event_message: {
+		event_id: string
+		message_id: string
+		event_type: string | null
+		event_subtype: string | null
+		part: number
+		source: number
+	}
+
+	file: {
+		discord_url: string
+		mxc_url: string
+	}
+
+	guild_space: {
+		guild_id: string
+		space_id: string
+	}
+
+	member_cache: {
+		room_id: string
+		mxid: string
+		displayname: string | null
+		avatar_url: string | null
+	}
+
+	message_channel: {
+		message_id: string
+		channel_id: string
+	}
+
+	sim: {
+		discord_id: string
+		sim_name: string
+		localpart: string
+		mxid: string
+	}
+
+	sim_member: {
+		mxid: string
+		room_id: string
+		profile_event_content_hash: string
+	}
+
+	webhook: {
+		channel_id: string
+		webhook_id: string
+		webhook_token: string
+	}
+}
+
+export type Prepared<Row> = {
+	pluck: () => Prepared<Row[keyof Row]>
+	all: (..._: any[]) => Row[]
+	get: (..._: any[]) => Row?
+}
+
+export type AllKeys<U> = U extends any ? keyof U : never
+export type PickTypeOf<T, K extends AllKeys<T>> = T extends { [k in K]?: any } ? T[K] : never
+export type Merge<U> = {[x in AllKeys<U>]: PickTypeOf<U, x>}

db/orm-utils.js

@@ -0,0 +1 @@
+module.exports = {}

db/orm.js

@@ -0,0 +1,140 @@
+// @ts-check
+
+const {db} = require("../passthrough")
+const U = require("./orm-utils")
+
+/**
+ * @template {keyof U.Models} Table
+ * @template {keyof U.Models[Table]} Col
+ * @param {Table} table
+ * @param {Col[] | Col} cols
+ * @param {string} [e]
+ */
+function select(table, cols, e = "") {
+	if (!Array.isArray(cols)) cols = [cols]
+	/** @type {U.Prepared<Pick<U.Models[Table], Col>>} */
+	const prepared = db.prepare(`SELECT ${cols.map(k => `"${String(k)}"`).join(", ")} FROM ${table} ${e}`)
+	return prepared
+}
+
+/**
+ * @template {keyof U.Models} Table
+ * @template {keyof U.Merge<U.Models[Table]>} Col
+ */
+class From {
+	/**
+	 * @param {Table} table
+	 */
+	constructor(table) {
+		/** @type {Table[]} */
+		this.tables = [table]
+
+		this.sql = ""
+		this.cols = []
+		this.using = []
+	}
+
+	/**
+	 * @template {keyof U.Models} Table2
+	 * @param {Table2} table
+	 * @param {Col & (keyof U.Models[Table2])} col
+	 */
+	join(table, col) {
+		/** @type {From<Table | Table2, keyof U.Merge<U.Models[Table | Table2]>>} */
+		// @ts-ignore
+		const r = this
+		r.tables.push(table)
+		r.using.push(col)
+		return r
+	}
+
+	/**
+	 * @template {Col} Select
+	 * @param {Col[] | Select[]} cols
+	 */
+	select(...cols) {
+		/** @type {From<Table, Select>} */
+		const r = this
+		r.cols = cols
+		return r
+	}
+
+	/**
+	 * @template {Col} Select
+	 * @param {Select} col
+	 */
+	pluck(col) {
+		/** @type {Pluck<Table, Select>} */
+		// @ts-ignore
+		const r = this
+		r.constructor = Pluck
+		r.cols = [col]
+		return r
+	}
+
+	/**
+	 * @param {string} sql
+	 */
+	and(sql) {
+		this.sql = sql
+		return this
+	}
+
+	prepare() {
+		let sql = `SELECT ${this.cols.map(k => `"${k}"`).join(", ")} FROM ${this.tables[0]} `
+		for (let i = 1; i < this.tables.length; i++) {
+			const table = this.tables[i]
+			const col = this.using[i-1]
+			sql += `INNER JOIN ${table} USING (${col}) `
+		}
+		sql += this.sql
+		/** @type {U.Prepared<Pick<U.Merge<U.Models[Table]>, Col>>} */
+		const prepared = db.prepare(sql)
+		return prepared
+	}
+
+	get(..._) {
+		const prepared = this.prepare()
+		return prepared.get(..._)
+	}
+
+	all(..._) {
+		const prepared = this.prepare()
+		return prepared.all(..._)
+	}
+}
+
+/**
+ * @template {keyof U.Models} Table
+ * @template {keyof U.Merge<U.Models[Table]>} Col
+ */
+class Pluck extends From {
+	// @ts-ignore
+	prepare() {
+		/** @type {U.Prepared<U.Merge<U.Models[Table]>[Col]>} */
+		// @ts-ignore
+		const prepared = super.prepare()
+		return prepared
+	}
+
+	get(..._) {
+		const prepared = this.prepare()
+		return prepared.get(..._)
+	}
+
+	all(..._) {
+		const prepared = this.prepare()
+		return prepared.all(..._)
+	}
+}
+
+/**
+ * @template {keyof U.Models} Table
+ * @param {Table} table
+ */
+function from(table) {
+	return new From(table)
+}
+
+module.exports.from = from
+module.exports.select = select

m2d/converters/event-to-message.test.js

@@ -1227,6 +1227,64 @@ test("event2message: with layered rich replies, the preview should only be the r
 	)
 })
 
+test("event2message: raw mentioning discord users in plaintext body works", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			content: {
+				msgtype: "m.text",
+				body: "<@114147806469554185> what do you think?"
+			},
+			event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
+			origin_server_ts: 1688301929913,
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe",
+			sender: "@cadence:cadence.moe",
+			type: "m.room.message",
+			unsigned: {
+				age: 405299
+			}
+		}),
+		{
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "<@114147806469554185> what do you think?",
+				avatar_url: undefined
+			}]
+		}
+	)
+})
+
+test("event2message: raw mentioning discord users in formatted body works", async t => {
+	t.deepEqual(
+		await eventToMessage({
+			content: {
+				msgtype: "m.text",
+				body: "wrong body",
+				format: "org.matrix.custom.html",
+				formatted_body: `&lt;@114147806469554185&gt; what do you think?`
+			},
+			event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU",
+			origin_server_ts: 1688301929913,
+			room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe",
+			sender: "@cadence:cadence.moe",
+			type: "m.room.message",
+			unsigned: {
+				age: 405299
+			}
+		}),
+		{
+			messagesToDelete: [],
+			messagesToEdit: [],
+			messagesToSend: [{
+				username: "cadence [they]",
+				content: "<@114147806469554185> what do you think?",
+				avatar_url: undefined
+			}]
+		}
+	)
+})
+
 test("event2message: mentioning discord users works", async t => {
 	t.deepEqual(
 		await eventToMessage({

passthrough.js

@@ -8,6 +8,8 @@
  * @property {import("heatsync")} sync
  * @property {import("better-sqlite3/lib/database")} db
  * @property {import("matrix-appservice").AppService} as
+ * @property {import("./db/orm").from} from
+ * @property {import("./db/orm").select} select
  */
 /** @type {Passthrough} */
 // @ts-ignore

start.js

@@ -19,6 +19,10 @@ passthrough.discord = discord
 const as = require("./matrix/appservice")
 passthrough.as = as
 
+const orm = sync.require("./db/orm")
+passthrough.from = orm.from
+passthrough.select = orm.select
+
 sync.require("./m2d/event-dispatcher")
 
 discord.snow.requestHandler.on("requestError", data => {

test/test.js

@@ -18,6 +18,10 @@ const sync = new HeatSync({watchFS: false})
 
 Object.assign(passthrough, { config, sync, db })
 
+const orm = sync.require("../db/orm")
+passthrough.from = orm.from
+passthrough.select = orm.select
+
 const file = sync.require("../matrix/file")
 file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) }