Improved database schema

- Some queries are faster due to better index ordering - Database is smaller thanks to splitting message_channel table and adding WITHOUT ROWID where helpful
2023-08-28 17:32:55 +12:00 · 2023-08-28 17:32:55 +12:00 · fcbb045cbb
commit fcbb045cbb
parent 21156446ee
11 changed files with 114 additions and 93 deletions
--- a/d2m/actions/delete-message.js
+++ b/d2m/actions/delete-message.js
@ -19,7 +19,7 @@ async function deleteMessage(data) {
 	for (const eventID of eventsToRedact) {
 		// Unfortuately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs
 		await api.redactEvent(roomID, eventID)
-		db.prepare("DELETE from event_message WHERE event_id = ?").run(eventID)
+		db.prepare("DELETE FROM event_message WHERE event_id = ?").run(eventID)
 	}
 }

--- a/d2m/actions/edit-message.js
+++ b/d2m/actions/edit-message.js
@ -32,7 +32,7 @@ async function editMessage(message, guild) {
 	// Not redacting as the last action because the last action is likely to be shown in the room preview in clients, and we don't want it to look like somebody actually deleted a message.
 	for (const eventID of eventsToRedact) {
 		await api.redactEvent(roomID, eventID, senderMxid)
-		db.prepare("DELETE from event_message WHERE event_id = ?").run(eventID)
+		db.prepare("DELETE FROM event_message WHERE event_id = ?").run(eventID)
 		// TODO: If I just redacted part = 0, I should update one of the other events to make it the new part = 0, right?
 	}

@ -44,7 +44,7 @@ async function editMessage(message, guild) {
 		delete contentWithoutType.$type

 		const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid)
-		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, 1, 1)").run(eventID, eventType, content.msgtype || null, message.id, message.channel_id) // part 1 = supporting; source 1 = discord
+		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 1, 1)").run(eventID, eventType, content.msgtype || null, message.id) // part 1 = supporting; source 1 = discord
 	}
 }

--- a/d2m/actions/send-message.js
+++ b/d2m/actions/send-message.js
@ -32,6 +32,9 @@ async function sendMessage(message, guild) {
 	const events = await messageToEvent.messageToEvent(message, guild, {}, {api})
 	const eventIDs = []
 	let eventPart = 0 // 0 is primary, 1 is supporting
+	if (events.length) {
+		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
+	}
 	for (const event of events) {
 		const eventType = event.$type
 		/** @type {Pick<typeof event, Exclude<keyof event, "$type">> & { $type?: string }} */
@ -40,7 +43,7 @@ async function sendMessage(message, guild) {

 		const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined
 		const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp)
-		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, message.channel_id, eventPart) // source 1 = discord
+		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, eventPart) // source 1 = discord

 		eventPart = 1 // TODO: use more intelligent algorithm to determine whether primary or supporting
 		eventIDs.push(eventID)
--- a/d2m/converters/message-to-event.js
+++ b/d2m/converters/message-to-event.js
@ -79,9 +79,10 @@ async function messageToEvent(message, guild, options = {}, di) {
 		const ref = message.message_reference
 		assert(ref)
 		assert(ref.message_id)
-		const row = db.prepare("SELECT room_id, event_id FROM event_message INNER JOIN channel_room USING (channel_id) WHERE channel_id = ? AND message_id = ?").get(ref.channel_id, ref.message_id)
-		if (!row) return []
-		const event = await di.api.getEvent(row.room_id, row.event_id)
+		const eventID = db.prepare("SELECT event_id FROM event_message WHERE message_id = ?").pluck().get(ref.message_id)
+		const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(ref.channel_id)
+		if (!eventID || !roomID) return []
+		const event = await di.api.getEvent(roomID, eventID)
 		return [{
 			...event.content,
 			$type: event.type
@ -118,7 +119,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 	// Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions
 	// (Still need to do scenarios 1 and 2 part B, and scenario 3.)
 	if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) {
-		const row = db.prepare("SELECT event_id, room_id, source FROM event_message INNER JOIN channel_room USING (channel_id) WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
+		const row = db.prepare("SELECT event_id, room_id, source FROM event_message INNER JOIN message_channel USING (message_id) INNER JOIN channel_room USING (channel_id) WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
 		if (row) {
 			repliedToEventId = row.event_id
 			repliedToEventRoomId = row.room_id
@ -144,9 +145,10 @@ async function messageToEvent(message, guild, options = {}, di) {
 	if (message.content) {
 		let content = message.content
 		content = content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => {
-			const row = db.prepare("SELECT room_id, event_id FROM event_message INNER JOIN channel_room USING (channel_id) WHERE channel_id = ? AND message_id = ? AND part = 0").get(channelID, messageID)
-			if (row) {
-				return `https://matrix.to/#/${row.room_id}/${row.event_id}`
+			const eventID = db.prepare("SELECT event_id FROM event_message WHERE message_id = ?").pluck().get(messageID)
+			const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(channelID)
+			if (eventID && roomID) {
+				return `https://matrix.to/#/${roomID}/${eventID}`
 			} else {
 				return `${whole} [event not found]`
 			}
--- a/d2m/event-dispatcher.js
+++ b/d2m/event-dispatcher.js
@ -81,18 +81,18 @@ module.exports = {
 	async checkMissedMessages(client, guild) {
 		if (guild.unavailable) return
 		const bridgedChannels = db.prepare("SELECT channel_id FROM channel_room").pluck().all()
-		const prepared = db.prepare("SELECT message_id FROM event_message WHERE channel_id = ? AND message_id = ?").pluck()
+		const prepared = db.prepare("SELECT 1 FROM event_message WHERE message_id = ?").pluck()
 		for (const channel of guild.channels.concat(guild.threads)) {
 			if (!bridgedChannels.includes(channel.id)) continue
 			if (!channel.last_message_id) continue
-			const latestWasBridged = prepared.get(channel.id, channel.last_message_id)
+			const latestWasBridged = prepared.get(channel.last_message_id)
 			if (latestWasBridged) continue

 			/** More recent messages come first. */
 			console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
 			const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50})
 			let latestBridgedMessageIndex = messages.findIndex(m => {
-				return prepared.get(channel.id, m.id)
+				return prepared.get(m.id)
 			})
 			console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`)
 			if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date.
--- a/db/ooye-schema.sql
+++ b/db/ooye-schema.sql
@ -0,0 +1,61 @@
+BEGIN TRANSACTION;
+CREATE TABLE IF NOT EXISTS "sim" (
+	"discord_id"	TEXT NOT NULL UNIQUE,
+	"sim_name"	TEXT NOT NULL UNIQUE,
+	"localpart"	TEXT NOT NULL UNIQUE,
+	"mxid"	TEXT NOT NULL UNIQUE,
+	PRIMARY KEY("discord_id")
+);
+CREATE TABLE IF NOT EXISTS "webhook" (
+	"channel_id"	TEXT NOT NULL UNIQUE,
+	"webhook_id"	TEXT NOT NULL UNIQUE,
+	"webhook_token"	TEXT NOT NULL,
+	PRIMARY KEY("channel_id")
+);
+CREATE TABLE IF NOT EXISTS "sim_member" (
+	"mxid"	TEXT NOT NULL,
+	"room_id"	TEXT NOT NULL,
+	"profile_event_content_hash"	BLOB,
+	PRIMARY KEY("room_id","mxid")
+) WITHOUT ROWID;
+CREATE TABLE IF NOT EXISTS "member_cache" (
+	"room_id"	TEXT NOT NULL,
+	"mxid"	TEXT NOT NULL,
+	"displayname"	TEXT,
+	"avatar_url"	TEXT,
+	PRIMARY KEY("room_id","mxid")
+) WITHOUT ROWID;
+CREATE TABLE IF NOT EXISTS "file" (
+	"discord_url"	TEXT NOT NULL,
+	"mxc_url"	TEXT NOT NULL,
+	PRIMARY KEY("discord_url")
+) WITHOUT ROWID;
+CREATE TABLE IF NOT EXISTS "guild_space" (
+	"guild_id"	TEXT NOT NULL,
+	"space_id"	TEXT NOT NULL,
+	PRIMARY KEY("guild_id")
+) WITHOUT ROWID;
+CREATE TABLE IF NOT EXISTS "channel_room" (
+	"channel_id"	TEXT NOT NULL,
+	"room_id"	TEXT NOT NULL,
+	"name"	TEXT NOT NULL,
+	"nick"	TEXT,
+	"thread_parent"	TEXT,
+	"custom_avatar"	TEXT,
+	PRIMARY KEY("channel_id","room_id")
+);
+CREATE TABLE IF NOT EXISTS "message_channel" (
+	"message_id"	TEXT NOT NULL,
+	"channel_id"	TEXT NOT NULL,
+	PRIMARY KEY("message_id")
+) WITHOUT ROWID;
+CREATE TABLE IF NOT EXISTS "event_message" (
+	"event_id"	TEXT NOT NULL,
+	"message_id"	TEXT NOT NULL,
+	"event_type"	TEXT,
+	"event_subtype"	TEXT,
+	"part"	INTEGER NOT NULL,
+	"source"	INTEGER NOT NULL,
+	PRIMARY KEY("message_id","event_id")
+) WITHOUT ROWID;
+COMMIT;
--- a/db/ooye-test-data.sql
+++ b/db/ooye-test-data.sql
@ -1,63 +1,3 @@
-BEGIN TRANSACTION;
-CREATE TABLE IF NOT EXISTS "guild_space" (
-	"guild_id"	TEXT NOT NULL UNIQUE,
-	"space_id"	TEXT NOT NULL UNIQUE,
-	PRIMARY KEY("guild_id")
-);
-CREATE TABLE IF NOT EXISTS "file" (
-	"discord_url"	TEXT NOT NULL UNIQUE,
-	"mxc_url"	TEXT NOT NULL UNIQUE,
-	PRIMARY KEY("discord_url")
-);
-CREATE TABLE IF NOT EXISTS "sim" (
-	"discord_id"	TEXT NOT NULL UNIQUE,
-	"sim_name"	TEXT NOT NULL UNIQUE,
-	"localpart"	TEXT NOT NULL UNIQUE,
-	"mxid"	TEXT NOT NULL UNIQUE,
-	PRIMARY KEY("discord_id")
-);
-CREATE TABLE IF NOT EXISTS "sim_member" (
-	"mxid"	TEXT NOT NULL,
-	"room_id"	TEXT NOT NULL,
-	"profile_event_content_hash"	BLOB,
-	PRIMARY KEY("mxid","room_id")
-);
-CREATE TABLE IF NOT EXISTS "webhook" (
-	"channel_id"	TEXT NOT NULL UNIQUE,
-	"webhook_id"	TEXT NOT NULL UNIQUE,
-	"webhook_token"	TEXT NOT NULL,
-	PRIMARY KEY("channel_id")
-);
-CREATE TABLE IF NOT EXISTS "channel_room" (
-	"channel_id"	TEXT NOT NULL UNIQUE,
-	"room_id"	TEXT NOT NULL UNIQUE,
-	"name"	TEXT,
-	"nick"	TEXT,
-	"thread_parent"	TEXT,
-	"custom_avatar"	TEXT,
-	PRIMARY KEY("channel_id")
-);
-CREATE TABLE IF NOT EXISTS "event_message" (
-	"event_id"	TEXT NOT NULL,
-	"event_type"	TEXT,
-	"event_subtype"	TEXT,
-	"message_id"	TEXT NOT NULL,
-	"channel_id"	TEXT,
-	"part"	INTEGER NOT NULL,
-	"source"	INTEGER NOT NULL,
-	PRIMARY KEY("event_id","message_id")
-);
-CREATE TABLE IF NOT EXISTS "member_cache" (
-	"room_id"	TEXT NOT NULL,
-	"mxid"	TEXT NOT NULL,
-	"displayname"	TEXT,
-	"avatar_url"	TEXT,
-	PRIMARY KEY("room_id", "mxid")
-);
-COMMIT;
-
-
-
 BEGIN TRANSACTION;

 INSERT INTO guild_space (guild_id, space_id) VALUES
@ -80,22 +20,35 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES
 INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES
 ('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL);

-INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES
-('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', '112760669178241024', 0, 1),
-('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', '112760669178241024', 0, 0),
-('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', '497161350934560778', 0, 1),
-('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', '160197704226439168', 0, 1),
-('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', '112760669178241024', 0, 1),
-('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', '112760669178241024', 1, 1),
-('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', '112760669178241024', 0, 1),
-('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', '112760669178241024', 0, 1),
-('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', '112760669178241024', 1, 1),
-('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', '112760669178241024', 0, 1),
-('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', '1100319550446252084', 0, 1),
-('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', '122155380120748034', 0, 1),
-('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', '122155380120748034', 0, 0),
-('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', '122155380120748034', 0, 0),
-('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', '687028734322147344', 0, 1);
+INSERT INTO message_channel (message_id, channel_id) VALUES
+('1106366167788044450', '122155380120748034'),
+('1126786462646550579', '112760669178241024'),
+('1128084748338741392', '112760669178241024'),
+('1128084851279536279', '112760669178241024'),
+('1128118177155526666', '112760669178241024'),
+('1141206225632112650', '160197704226439168'),
+('1141501302736695316', '112760669178241024'),
+('1141501302736695317', '112760669178241024'),
+('1141619794500649020', '497161350934560778'),
+('1143121514925928541', '1100319550446252084'),
+('1144865310588014633', '687028734322147344');
+
+INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES
+('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 1),
+('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', 0, 0),
+('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', 0, 1),
+('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', 0, 1),
+('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1),
+('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1),
+('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', 0, 1),
+('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', 0, 1),
+('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', 1, 1),
+('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 1),
+('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 1),
+('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1),
+('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 0, 0),
+('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 0, 0),
+('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 1);

 INSERT INTO file (discord_url, mxc_url) VALUES
 ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),
--- a/m2d/actions/send-event.js
+++ b/m2d/actions/send-event.js
@ -39,6 +39,7 @@ async function sendEvent(event) {
 	//	TODO ...
 	for (const message of messagesToSend) {
 		const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID)
+		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, channelID)
 		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content.msgtype || null, messageResponse.id, channelID, eventPart) // source 0 = matrix

 		eventPart = 1 // TODO: use more intelligent algorithm to determine whether primary or supporting?
--- a/m2d/converters/event-to-message.js
+++ b/m2d/converters/event-to-message.js
@ -188,7 +188,7 @@ async function eventToMessage(event, guild, di) {
 			if (!repliedToEventId) return
 			const repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId)
 			if (!repliedToEvent) return
-			const row = db.prepare("SELECT channel_id, message_id FROM event_message WHERE event_id = ? ORDER BY part").get(repliedToEventId)
+			const row = db.prepare("SELECT channel_id, message_id FROM event_message INNER JOIN message_channel USING (message_id) WHERE event_id = ? ORDER BY part").get(repliedToEventId)
 			if (row) {
 				replyLine = `<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} `
 			} else {
--- a/scripts/save-event-types-to-db.js
+++ b/scripts/save-event-types-to-db.js
@ -13,7 +13,7 @@ Object.assign(passthrough, {sync, db})
 const api = require("../matrix/api")

 /** @type {{event_id: string, room_id: string, event_type: string}[]} */ // @ts-ignore
-const rows = db.prepare("SELECT event_id, room_id, event_type FROM event_message INNER JOIN channel_room USING (channel_id)").all()
+const rows = db.prepare("SELECT event_id, room_id, event_type FROM event_message INNER JOIN message_channel USING (message_id) INNER JOIN channel_room USING (channel_id)").all()

 const preparedUpdate = db.prepare("UPDATE event_message SET event_type = ?, event_subtype = ? WHERE event_id = ?")

--- a/test/test.js
+++ b/test/test.js
@ -8,7 +8,8 @@ const config = require("../config")
 const passthrough = require("../passthrough")
 const db = new sqlite(":memory:")

-db.exec(fs.readFileSync("db/data-for-test.sql", "utf8"))
+db.exec(fs.readFileSync("db/ooye-schema.sql", "utf8"))
+db.exec(fs.readFileSync("db/ooye-test-data.sql", "utf8"))

 const sync = new HeatSync({watchFS: false})