From 0fc8e68f15c35d490ec88013f8af75afaf5f33f2 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 19 Aug 2023 18:39:23 +1200 Subject: [PATCH] support threads --- d2m/actions/create-room.js | 68 +++++++++++++++++++++++----------- d2m/actions/create-space.js | 12 +++--- d2m/discord-packets.js | 6 +++ d2m/event-dispatcher.js | 61 ++++++++++++++++-------------- db/data-for-test.sql | 8 ++-- m2d/actions/channel-webhook.js | 5 ++- m2d/actions/send-event.js | 10 ++++- notes.md | 7 ++++ test/data.js | 58 +++++++++++++++++++++++++++++ 9 files changed, 172 insertions(+), 63 deletions(-) diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js index 0fd06467..6d64d585 100644 --- a/d2m/actions/create-room.js +++ b/d2m/actions/create-room.js @@ -21,8 +21,8 @@ async function roomToKState(roomID) { } /** - * @params {string} roomID - * @params {any} kstate + * @param {string} roomID + * @param {any} kstate */ function applyKStateDiffToRoom(roomID, kstate) { const events = ks.kstateToState(kstate) @@ -51,7 +51,7 @@ function convertNameAndTopic(channel, guild, customName) { } /** - * @param {DiscordTypes.APIGuildTextChannel} channel + * @param {DiscordTypes.APIGuildTextChannel | DiscordTypes.APIThreadChannel} channel * @param {DiscordTypes.APIGuild} guild */ async function channelToKState(channel, guild) { @@ -98,21 +98,27 @@ async function channelToKState(channel, guild) { * @returns {Promise} room ID */ async function createRoom(channel, guild, spaceID, kstate) { + const [convertedName, convertedTopic] = convertNameAndTopic(channel, guild, null) const roomID = await api.createRoom({ - name: channel.name, - topic: channel.topic || undefined, + name: convertedName, + topic: convertedTopic, preset: "private_chat", visibility: "private", invite: ["@cadence:cadence.moe"], // TODO initial_state: ks.kstateToState(kstate) }) - db.prepare("INSERT INTO channel_room (channel_id, room_id) VALUES (?, ?)").run(channel.id, roomID) + let threadParent = null + if (channel.type === DiscordTypes.ChannelType.PublicThread) { + /** @type {DiscordTypes.APIThreadChannel} */ // @ts-ignore + const thread = channel + threadParent = thread.parent_id + } + + db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent) // Put the newly created child into the space - await api.sendState(spaceID, "m.space.child", roomID, { // TODO: should I deduplicate with the equivalent code from syncRoom? - via: ["cadence.moe"] // TODO: use the proper server - }) + _syncSpaceMember(channel, spaceID, roomID) return roomID } @@ -156,14 +162,15 @@ async function _syncRoom(channelID, shouldActuallySync) { assert.ok(channel) const guild = channelToGuild(channel) - /** @type {string?} */ - const existing = db.prepare("SELECT room_id from channel_room WHERE channel_id = ?").pluck().get(channel.id) + /** @type {{room_id: string, thread_parent: string?}} */ + const existing = db.prepare("SELECT room_id, thread_parent from channel_room WHERE channel_id = ?").get(channelID) + if (!existing) { const {spaceID, channelKState} = await channelToKState(channel, guild) return createRoom(channel, guild, spaceID, channelKState) } else { if (!shouldActuallySync) { - return existing // only need to ensure room exists, and it does. return the room ID + return existing.room_id // only need to ensure room exists, and it does. return the room ID } console.log(`[room sync] to matrix: ${channel.name}`) @@ -171,24 +178,41 @@ async function _syncRoom(channelID, shouldActuallySync) { const {spaceID, channelKState} = await channelToKState(channel, guild) // sync channel state to room - const roomKState = await roomToKState(existing) + const roomKState = await roomToKState(existing.room_id) const roomDiff = ks.diffKState(roomKState, channelKState) - const roomApply = applyKStateDiffToRoom(existing, roomDiff) + const roomApply = applyKStateDiffToRoom(existing.room_id, roomDiff) // sync room as space member - const spaceKState = await roomToKState(spaceID) - const spaceDiff = ks.diffKState(spaceKState, { - [`m.space.child/${existing}`]: { - via: ["cadence.moe"] // TODO: use the proper server - } - }) - const spaceApply = applyKStateDiffToRoom(spaceID, spaceDiff) + const spaceApply = _syncSpaceMember(channel, spaceID, existing.room_id) await Promise.all([roomApply, spaceApply]) - return existing + return existing.room_id } } +/** + * @param {DiscordTypes.APIGuildTextChannel} channel + * @param {string} spaceID + * @param {string} roomID + * @returns {Promise} + */ +async function _syncSpaceMember(channel, spaceID, roomID) { + const spaceKState = await roomToKState(spaceID) + let spaceEventContent = {} + if ( + channel.type !== DiscordTypes.ChannelType.PrivateThread // private threads do not belong in the space (don't offer people something they can't join) + || channel["thread_metadata"]?.archived // archived threads do not belong in the space (don't offer people conversations that are no longer relevant) + ) { + spaceEventContent = { + via: ["cadence.moe"] // TODO: use the proper server + } + } + const spaceDiff = ks.diffKState(spaceKState, { + [`m.space.child/${roomID}`]: spaceEventContent + }) + return applyKStateDiffToRoom(spaceID, spaceDiff) +} + function ensureRoom(channelID) { return _syncRoom(channelID, false) } diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index e3b6da7c..02c2dcfd 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -1,5 +1,6 @@ // @ts-check +const assert = require("assert") const passthrough = require("../../passthrough") const { sync, db } = passthrough /** @type {import("../../matrix/api")} */ @@ -9,13 +10,14 @@ const api = sync.require("../../matrix/api") * @param {import("discord-api-types/v10").RESTGetAPIGuildResult} guild */ async function createSpace(guild) { + assert(guild.name) const roomID = await api.createRoom({ name: guild.name, - preset: "private_chat", + preset: "private_chat", // cannot join space unless invited visibility: "private", power_level_content_override: { - events_default: 100, - invite: 50 + events_default: 100, // space can only be managed by bridge + invite: 0 // any existing member can invite others }, invite: ["@cadence:cadence.moe"], // TODO topic: guild.description || undefined, @@ -27,13 +29,13 @@ async function createSpace(guild) { type: "m.room.guest_access", state_key: "", content: { - guest_access: "can_join" + guest_access: "can_join" // guests can join space if other conditions are met } }, { type: "m.room.history_visibility", content: { - history_visibility: "invited" + history_visibility: "invited" // any events sent after user was invited are visible } } ] diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index c0ba1a6b..79138b2f 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -35,6 +35,12 @@ const utils = { arr.push(channel.id) client.channels.set(channel.id, channel) } + for (const thread of message.d.threads || []) { + // @ts-ignore + thread.guild_id = message.d.id + arr.push(thread.id) + client.channels.set(thread.id, thread) + } } else if (message.t === "GUILD_DELETE") { diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 8e645913..93871998 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -15,6 +15,10 @@ const api = sync.require("../matrix/api") let lastReportedEvent = 0 +function isGuildAllowed(guildID) { + return ["112760669178241024", "497159726455455754", "1100319549670301727"].includes(guildID) +} + // Grab Discord events we care about for the bridge, check them, and pass them on module.exports = { @@ -29,31 +33,34 @@ module.exports = { console.error(`while handling this ${gatewayMessage.t} gateway event:`) console.dir(gatewayMessage.d, {depth: null}) - if (Date.now() - lastReportedEvent > 5000) { - lastReportedEvent = Date.now() - const channelID = gatewayMessage.d.channel_id - if (channelID) { - const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(channelID) - let stackLines = e.stack.split("\n") - let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/")) - if (cloudstormLine !== -1) { - stackLines = stackLines.slice(0, cloudstormLine - 2) - } - api.sendEvent(roomID, "m.room.message", { - msgtype: "m.text", - body: "\u26a0 Bridged event from Discord not delivered. See formatted content for full details.", - format: "org.matrix.custom.html", - formatted_body: "\u26a0 Bridged event from Discord not delivered" - + `
Gateway event: ${gatewayMessage.t}` - + `
${stackLines.join("\n")}
` - + `
Original payload` - + `
${util.inspect(gatewayMessage.d, false, 4, false)}
`, - "m.mentions": { - user_ids: ["@cadence:cadence.moe"] - } - }) - } + if (Date.now() - lastReportedEvent < 5000) return + lastReportedEvent = Date.now() + + const channelID = gatewayMessage.d.channel_id + if (!channelID) return + const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").pluck().get(channelID) + if (!roomID) return + + let stackLines = e.stack.split("\n") + let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/")) + if (cloudstormLine !== -1) { + stackLines = stackLines.slice(0, cloudstormLine - 2) } + api.sendEvent(roomID, "m.room.message", { + msgtype: "m.text", + body: "\u26a0 Bridged event from Discord not delivered. See formatted content for full details.", + format: "org.matrix.custom.html", + formatted_body: "\u26a0 Bridged event from Discord not delivered" + + `
Gateway event: ${gatewayMessage.t}` + + `
${e.toString()}` + + `
Error trace` + + `
${stackLines.join("\n")}
` + + `
Original payload` + + `
${util.inspect(gatewayMessage.d, false, 4, false)}
`, + "m.mentions": { + user_ids: ["@cadence:cadence.moe"] + } + }) }, /** @@ -72,7 +79,7 @@ module.exports = { const channel = client.channels.get(message.channel_id) if (!channel.guild_id) return // Nothing we can do in direct messages. const guild = client.guilds.get(channel.guild_id) - if (message.guild_id !== "112760669178241024" && message.guild_id !== "497159726455455754") return // TODO: activate on other servers (requires the space creation flow to be done first) + if (!isGuildAllowed(guild.id)) return await sendMessage.sendMessage(message, guild) }, @@ -97,7 +104,7 @@ module.exports = { const channel = client.channels.get(message.channel_id) if (!channel.guild_id) return // Nothing we can do in direct messages. const guild = client.guilds.get(channel.guild_id) - if (message.guild_id !== "112760669178241024" && message.guild_id !== "497159726455455754") return // TODO: activate on other servers (requires the space creation flow to be done first) + if (!isGuildAllowed(guild.id)) return await editMessage.editMessage(message, guild) } }, @@ -109,7 +116,6 @@ module.exports = { async onReactionAdd(client, data) { if (data.user_id === client.user.id) return // m2d reactions are added by the discord bot user - do not reflect them back to matrix. if (data.emoji.id !== null) return // TODO: image emoji reactions - console.log(data) await addReaction.addReaction(data) }, @@ -118,7 +124,6 @@ module.exports = { * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data */ async onMessageDelete(client, data) { - console.log(data) await deleteMessage.deleteMessage(data) } } diff --git a/db/data-for-test.sql b/db/data-for-test.sql index fa045629..aa82c912 100644 --- a/db/data-for-test.sql +++ b/db/data-for-test.sql @@ -54,10 +54,10 @@ BEGIN TRANSACTION; INSERT INTO guild_space (guild_id, space_id) VALUES ('112760669178241024', '!jjWAGMeQdNrVZSSfvz:cadence.moe'); -INSERT INTO channel_room (channel_id, room_id, name, nick) VALUES -('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main'), -('497161350934560778', '!edUxjVdzgUvXDUIQCK:cadence.moe', 'amanda-spam', NULL), -('160197704226439168', '!uCtjHhfGlYbVnPVlkG:cadence.moe', 'the-stanley-parable-channel', 'bots'); +INSERT INTO channel_room (channel_id, room_id, name, nick, is_thread) VALUES +('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL, 0), +('497161350934560778', '!edUxjVdzgUvXDUIQCK:cadence.moe', 'amanda-spam', NULL, 0), +('160197704226439168', '!uCtjHhfGlYbVnPVlkG:cadence.moe', 'the-stanley-parable-channel', 'bots', 0); INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'), diff --git a/m2d/actions/channel-webhook.js b/m2d/actions/channel-webhook.js index b62057b4..f5fd9a9c 100644 --- a/m2d/actions/channel-webhook.js +++ b/m2d/actions/channel-webhook.js @@ -50,10 +50,11 @@ async function withWebhook(channelID, callback) { /** * @param {string} channelID * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]}} data + * @param {string} [threadID] */ -async function sendMessageWithWebhook(channelID, data) { +async function sendMessageWithWebhook(channelID, data, threadID) { const result = await withWebhook(channelID, async webhook => { - return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, disableEveryone: true}) + return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID, disableEveryone: true}) }) return result } diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js index 3f49fa4c..88ba0fd6 100644 --- a/m2d/actions/send-event.js +++ b/m2d/actions/send-event.js @@ -13,7 +13,13 @@ const eventToMessage = sync.require("../converters/event-to-message") /** @param {import("../../types").Event.Outer} event */ async function sendEvent(event) { // TODO: we just assume the bridge has already been created - const channelID = db.prepare("SELECT channel_id FROM channel_room WHERE room_id = ?").pluck().get(event.room_id) + const row = db.prepare("SELECT channel_id, thread_parent FROM channel_room WHERE room_id = ?").get(event.room_id) + let channelID = row.channel_id + let threadID = undefined + if (row.thread_parent) { + threadID = channelID + channelID = row.thread_parent // it's the thread's parent... get with the times... + } // no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it @@ -24,7 +30,7 @@ async function sendEvent(event) { const messageResponses = [] let eventPart = 0 // 0 is primary, 1 is supporting for (const message of messages) { - const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message) + const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID) db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, channel_id, part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content.msgtype || null, messageResponse.id, channelID, eventPart) // source 0 = matrix eventPart = 1 // TODO: use more intelligent algorithm to determine whether primary or supporting? diff --git a/notes.md b/notes.md index ec2b9bb4..1dcbcd7f 100644 --- a/notes.md +++ b/notes.md @@ -9,6 +9,13 @@ A database will be used to store the discord id to matrix event id mapping. Tabl There needs to be a way to easily manually trigger something later. For example, it should be easy to manually retry sending a message, or check all members for changes, etc. +## Current manual process for setting up a server + +1. Call createSpace.createSpace(discord.guilds.get(GUILD_ID)) +2. Call createRoom.createAllForGuild(GUILD_ID) +3. Edit source code of event-dispatcher.js isGuildAllowed() and add the guild ID to the list +4. If developing, make sure SSH port forward is activated, then wait for events to sync over! + ## Transforming content 1. Upload attachments to mxc if they are small enough. diff --git a/test/data.js b/test/data.js index a1d3ece2..b579a242 100644 --- a/test/data.js +++ b/test/data.js @@ -816,6 +816,64 @@ module.exports = { format_type: 1, name: "pomu puff" }] + }, + message_in_thread: { + type: 0, + tts: false, + timestamp: "2023-08-19T01:55:02.063000+00:00", + referenced_message: null, + position: 942, + pinned: false, + nonce: "1142275498206822400", + mentions: [], + mention_roles: [], + mention_everyone: false, + member: { + roles: [ + "112767366235959296", "118924814567211009", + "204427286542417920", "199995902742626304", + "222168467627835392", "238028326281805825", + "259806643414499328", "265239342648131584", + "271173313575780353", "287733611912757249", + "225744901915148298", "305775031223320577", + "318243902521868288", "348651574924541953", + "349185088157777920", "378402925128712193", + "392141548932038658", "393912152173576203", + "482860581670486028", "495384759074160642", + "638988388740890635", "373336013109461013", + "530220455085473813", "454567553738473472", + "790724320824655873", "1123518980456452097", + "1040735082610167858", "695946570482450442", + "1123460940935991296", "849737964090556488" + ], + premium_since: null, + pending: false, + nick: null, + mute: false, + joined_at: "2015-11-11T09:55:40.321000+00:00", + flags: 0, + deaf: false, + communication_disabled_until: null, + avatar: null + }, + id: "1142275501721911467", + flags: 0, + embeds: [], + edited_timestamp: null, + content: "don't mind me, posting something for cadence", + components: [], + channel_id: "910283343378120754", + author: { + username: "kumaccino", + public_flags: 128, + id: "113340068197859328", + global_name: "kumaccino", + discriminator: "0", + avatar_decoration_data: null, + avatar: "b48302623a12bc7c59a71328f72ccb39" + }, + attachments: [], + guild_id: "112760669178241024" } }, message_update: {