diff --git a/d2m/actions/delete-message.js b/d2m/actions/delete-message.js index c9b43ee..bf6f034 100644 --- a/d2m/actions/delete-message.js +++ b/d2m/actions/delete-message.js @@ -19,7 +19,7 @@ async function deleteMessage(data) { for (const eventID of eventsToRedact) { // Unfortuately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs await api.redactEvent(roomID, eventID) - db.prepare("DELETE from event_message WHERE event_id = ?").run(eventID) + db.prepare("DELETE FROM event_message WHERE event_id = ?").run(eventID) } } diff --git a/d2m/actions/edit-message.js b/d2m/actions/edit-message.js index d7cb81d..28312a1 100644 --- a/d2m/actions/edit-message.js +++ b/d2m/actions/edit-message.js @@ -32,7 +32,7 @@ async function editMessage(message, guild) { // Not redacting as the last action because the last action is likely to be shown in the room preview in clients, and we don't want it to look like somebody actually deleted a message. for (const eventID of eventsToRedact) { await api.redactEvent(roomID, eventID, senderMxid) - db.prepare("DELETE from event_message WHERE event_id = ?").run(eventID) + db.prepare("DELETE FROM event_message WHERE event_id = ?").run(eventID) // TODO: If I just redacted part = 0, I should update one of the other events to make it the new part = 0, right? } @@ -44,7 +44,7 @@ async function editMessage(message, guild) { delete contentWithoutType.$type const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, 1, 1)").run(eventID, eventType, content.msgtype || null, message.id, message.channel_id) // part 1 = supporting; source 1 = discord + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 1, 1)").run(eventID, eventType, content.msgtype || null, message.id) // part 1 = supporting; source 1 = discord } } diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js index 49cbafc..6483e19 100644 --- a/d2m/actions/send-message.js +++ b/d2m/actions/send-message.js @@ -32,6 +32,9 @@ async function sendMessage(message, guild) { const events = await messageToEvent.messageToEvent(message, guild, {}, {api}) const eventIDs = [] let eventPart = 0 // 0 is primary, 1 is supporting + if (events.length) { + db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id) + } for (const event of events) { const eventType = event.$type /** @type {Pick> & { $type?: string }} */ @@ -40,7 +43,7 @@ async function sendMessage(message, guild) { const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, message.channel_id, eventPart) // source 1 = discord + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, eventPart) // source 1 = discord eventPart = 1 // TODO: use more intelligent algorithm to determine whether primary or supporting eventIDs.push(eventID) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index c5da78d..d8b8520 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -79,9 +79,10 @@ async function messageToEvent(message, guild, options = {}, di) { const ref = message.message_reference assert(ref) assert(ref.message_id) - const row = db.prepare("SELECT room_id, event_id FROM event_message INNER JOIN channel_room USING (channel_id) WHERE channel_id = ? AND message_id = ?").get(ref.channel_id, ref.message_id) - if (!row) return [] - const event = await di.api.getEvent(row.room_id, row.event_id) + const eventID = db.prepare("SELECT event_id FROM event_message WHERE message_id = ?").pluck().get(ref.message_id) + const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(ref.channel_id) + if (!eventID || !roomID) return [] + const event = await di.api.getEvent(roomID, eventID) return [{ ...event.content, $type: event.type @@ -118,7 +119,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions // (Still need to do scenarios 1 and 2 part B, and scenario 3.) if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) { - const row = db.prepare("SELECT event_id, room_id, source FROM event_message INNER JOIN channel_room USING (channel_id) WHERE message_id = ? AND part = 0").get(message.message_reference.message_id) + const row = db.prepare("SELECT event_id, room_id, source FROM event_message INNER JOIN message_channel USING (message_id) INNER JOIN channel_room USING (channel_id) WHERE message_id = ? AND part = 0").get(message.message_reference.message_id) if (row) { repliedToEventId = row.event_id repliedToEventRoomId = row.room_id @@ -144,9 +145,10 @@ async function messageToEvent(message, guild, options = {}, di) { if (message.content) { let content = message.content content = content.replace(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/, (whole, guildID, channelID, messageID) => { - const row = db.prepare("SELECT room_id, event_id FROM event_message INNER JOIN channel_room USING (channel_id) WHERE channel_id = ? AND message_id = ? AND part = 0").get(channelID, messageID) - if (row) { - return `https://matrix.to/#/${row.room_id}/${row.event_id}` + const eventID = db.prepare("SELECT event_id FROM event_message WHERE message_id = ?").pluck().get(messageID) + const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(channelID) + if (eventID && roomID) { + return `https://matrix.to/#/${roomID}/${eventID}` } else { return `${whole} [event not found]` } diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 3793a60..bbfc5a0 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -81,18 +81,18 @@ module.exports = { async checkMissedMessages(client, guild) { if (guild.unavailable) return const bridgedChannels = db.prepare("SELECT channel_id FROM channel_room").pluck().all() - const prepared = db.prepare("SELECT message_id FROM event_message WHERE channel_id = ? AND message_id = ?").pluck() + const prepared = db.prepare("SELECT 1 FROM event_message WHERE message_id = ?").pluck() for (const channel of guild.channels.concat(guild.threads)) { if (!bridgedChannels.includes(channel.id)) continue if (!channel.last_message_id) continue - const latestWasBridged = prepared.get(channel.id, channel.last_message_id) + const latestWasBridged = prepared.get(channel.last_message_id) if (latestWasBridged) continue /** More recent messages come first. */ console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) const messages = await client.snow.channel.getChannelMessages(channel.id, {limit: 50}) let latestBridgedMessageIndex = messages.findIndex(m => { - return prepared.get(channel.id, m.id) + return prepared.get(m.id) }) console.log(`[check missed messages] got ${messages.length} messages; last message that IS bridged is at position ${latestBridgedMessageIndex} in the channel`) if (latestBridgedMessageIndex === -1) latestBridgedMessageIndex = 1 // rather than crawling the ENTIRE channel history, let's just bridge the most recent 1 message to make it up to date. diff --git a/db/ooye-schema.sql b/db/ooye-schema.sql new file mode 100644 index 0000000..1d7b6ed --- /dev/null +++ b/db/ooye-schema.sql @@ -0,0 +1,61 @@ +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS "sim" ( + "discord_id" TEXT NOT NULL UNIQUE, + "sim_name" TEXT NOT NULL UNIQUE, + "localpart" TEXT NOT NULL UNIQUE, + "mxid" TEXT NOT NULL UNIQUE, + PRIMARY KEY("discord_id") +); +CREATE TABLE IF NOT EXISTS "webhook" ( + "channel_id" TEXT NOT NULL UNIQUE, + "webhook_id" TEXT NOT NULL UNIQUE, + "webhook_token" TEXT NOT NULL, + PRIMARY KEY("channel_id") +); +CREATE TABLE IF NOT EXISTS "sim_member" ( + "mxid" TEXT NOT NULL, + "room_id" TEXT NOT NULL, + "profile_event_content_hash" BLOB, + PRIMARY KEY("room_id","mxid") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "member_cache" ( + "room_id" TEXT NOT NULL, + "mxid" TEXT NOT NULL, + "displayname" TEXT, + "avatar_url" TEXT, + PRIMARY KEY("room_id","mxid") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "file" ( + "discord_url" TEXT NOT NULL, + "mxc_url" TEXT NOT NULL, + PRIMARY KEY("discord_url") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "guild_space" ( + "guild_id" TEXT NOT NULL, + "space_id" TEXT NOT NULL, + PRIMARY KEY("guild_id") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "channel_room" ( + "channel_id" TEXT NOT NULL, + "room_id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "nick" TEXT, + "thread_parent" TEXT, + "custom_avatar" TEXT, + PRIMARY KEY("channel_id","room_id") +); +CREATE TABLE IF NOT EXISTS "message_channel" ( + "message_id" TEXT NOT NULL, + "channel_id" TEXT NOT NULL, + PRIMARY KEY("message_id") +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS "event_message" ( + "event_id" TEXT NOT NULL, + "message_id" TEXT NOT NULL, + "event_type" TEXT, + "event_subtype" TEXT, + "part" INTEGER NOT NULL, + "source" INTEGER NOT NULL, + PRIMARY KEY("message_id","event_id") +) WITHOUT ROWID; +COMMIT; diff --git a/db/data-for-test.sql b/db/ooye-test-data.sql similarity index 62% rename from db/data-for-test.sql rename to db/ooye-test-data.sql index 8c564df..1f4fb29 100644 --- a/db/data-for-test.sql +++ b/db/ooye-test-data.sql @@ -1,63 +1,3 @@ -BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "guild_space" ( - "guild_id" TEXT NOT NULL UNIQUE, - "space_id" TEXT NOT NULL UNIQUE, - PRIMARY KEY("guild_id") -); -CREATE TABLE IF NOT EXISTS "file" ( - "discord_url" TEXT NOT NULL UNIQUE, - "mxc_url" TEXT NOT NULL UNIQUE, - PRIMARY KEY("discord_url") -); -CREATE TABLE IF NOT EXISTS "sim" ( - "discord_id" TEXT NOT NULL UNIQUE, - "sim_name" TEXT NOT NULL UNIQUE, - "localpart" TEXT NOT NULL UNIQUE, - "mxid" TEXT NOT NULL UNIQUE, - PRIMARY KEY("discord_id") -); -CREATE TABLE IF NOT EXISTS "sim_member" ( - "mxid" TEXT NOT NULL, - "room_id" TEXT NOT NULL, - "profile_event_content_hash" BLOB, - PRIMARY KEY("mxid","room_id") -); -CREATE TABLE IF NOT EXISTS "webhook" ( - "channel_id" TEXT NOT NULL UNIQUE, - "webhook_id" TEXT NOT NULL UNIQUE, - "webhook_token" TEXT NOT NULL, - PRIMARY KEY("channel_id") -); -CREATE TABLE IF NOT EXISTS "channel_room" ( - "channel_id" TEXT NOT NULL UNIQUE, - "room_id" TEXT NOT NULL UNIQUE, - "name" TEXT, - "nick" TEXT, - "thread_parent" TEXT, - "custom_avatar" TEXT, - PRIMARY KEY("channel_id") -); -CREATE TABLE IF NOT EXISTS "event_message" ( - "event_id" TEXT NOT NULL, - "event_type" TEXT, - "event_subtype" TEXT, - "message_id" TEXT NOT NULL, - "channel_id" TEXT, - "part" INTEGER NOT NULL, - "source" INTEGER NOT NULL, - PRIMARY KEY("event_id","message_id") -); -CREATE TABLE IF NOT EXISTS "member_cache" ( - "room_id" TEXT NOT NULL, - "mxid" TEXT NOT NULL, - "displayname" TEXT, - "avatar_url" TEXT, - PRIMARY KEY("room_id", "mxid") -); -COMMIT; - - - BEGIN TRANSACTION; INSERT INTO guild_space (guild_id, space_id) VALUES @@ -80,22 +20,35 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES ('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL); -INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES -('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', '112760669178241024', 0, 1), -('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', '112760669178241024', 0, 0), -('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', '497161350934560778', 0, 1), -('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', '160197704226439168', 0, 1), -('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', '112760669178241024', 0, 1), -('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', '112760669178241024', 1, 1), -('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', '112760669178241024', 0, 1), -('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', '112760669178241024', 0, 1), -('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', '112760669178241024', 1, 1), -('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', '112760669178241024', 0, 1), -('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', '1100319550446252084', 0, 1), -('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', '122155380120748034', 0, 1), -('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', '122155380120748034', 0, 0), -('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', '122155380120748034', 0, 0), -('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', '687028734322147344', 0, 1); +INSERT INTO message_channel (message_id, channel_id) VALUES +('1106366167788044450', '122155380120748034'), +('1126786462646550579', '112760669178241024'), +('1128084748338741392', '112760669178241024'), +('1128084851279536279', '112760669178241024'), +('1128118177155526666', '112760669178241024'), +('1141206225632112650', '160197704226439168'), +('1141501302736695316', '112760669178241024'), +('1141501302736695317', '112760669178241024'), +('1141619794500649020', '497161350934560778'), +('1143121514925928541', '1100319550446252084'), +('1144865310588014633', '687028734322147344'); + +INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES +('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 1), +('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', 0, 0), +('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', 0, 1), +('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', 0, 1), +('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1), +('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1), +('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', 0, 1), +('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', 0, 1), +('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', 1, 1), +('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 1), +('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 1), +('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1), +('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 0, 0), +('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 0, 0), +('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 1); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js index 3aa6346..0208c57 100644 --- a/m2d/actions/send-event.js +++ b/m2d/actions/send-event.js @@ -39,6 +39,7 @@ async function sendEvent(event) { // TODO ... for (const message of messagesToSend) { const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID) + db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, channelID) db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content.msgtype || null, messageResponse.id, channelID, eventPart) // source 0 = matrix eventPart = 1 // TODO: use more intelligent algorithm to determine whether primary or supporting? diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 70a0a69..3e22093 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -188,7 +188,7 @@ async function eventToMessage(event, guild, di) { if (!repliedToEventId) return const repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId) if (!repliedToEvent) return - const row = db.prepare("SELECT channel_id, message_id FROM event_message WHERE event_id = ? ORDER BY part").get(repliedToEventId) + const row = db.prepare("SELECT channel_id, message_id FROM event_message INNER JOIN message_channel USING (message_id) WHERE event_id = ? ORDER BY part").get(repliedToEventId) if (row) { replyLine = `<:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} ` } else { diff --git a/scripts/save-event-types-to-db.js b/scripts/save-event-types-to-db.js index 547e85c..290aa4a 100644 --- a/scripts/save-event-types-to-db.js +++ b/scripts/save-event-types-to-db.js @@ -13,7 +13,7 @@ Object.assign(passthrough, {sync, db}) const api = require("../matrix/api") /** @type {{event_id: string, room_id: string, event_type: string}[]} */ // @ts-ignore -const rows = db.prepare("SELECT event_id, room_id, event_type FROM event_message INNER JOIN channel_room USING (channel_id)").all() +const rows = db.prepare("SELECT event_id, room_id, event_type FROM event_message INNER JOIN message_channel USING (message_id) INNER JOIN channel_room USING (channel_id)").all() const preparedUpdate = db.prepare("UPDATE event_message SET event_type = ?, event_subtype = ? WHERE event_id = ?") diff --git a/test/test.js b/test/test.js index e19f8ff..ce2a2a0 100644 --- a/test/test.js +++ b/test/test.js @@ -8,7 +8,8 @@ const config = require("../config") const passthrough = require("../passthrough") const db = new sqlite(":memory:") -db.exec(fs.readFileSync("db/data-for-test.sql", "utf8")) +db.exec(fs.readFileSync("db/ooye-schema.sql", "utf8")) +db.exec(fs.readFileSync("db/ooye-test-data.sql", "utf8")) const sync = new HeatSync({watchFS: false})