From 8ad0117fd24ba98fe390ca1d64ed79646cc8e983 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Feb 2025 15:04:34 +1300 Subject: [PATCH 1/4] d->m: Presence --- readme.md | 1 + src/d2m/actions/set-presence.js | 45 +++++++++++++++++++++++++++++++++ src/d2m/discord-client.js | 2 +- src/d2m/discord-packets.js | 3 +++ src/d2m/event-dispatcher.js | 11 ++++++++ src/matrix/api.js | 9 +++++++ 6 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/d2m/actions/set-presence.js diff --git a/readme.md b/readme.md index 1f01d01..8ab6bb4 100644 --- a/readme.md +++ b/readme.md @@ -36,6 +36,7 @@ Most features you'd expect in both directions, plus a little extra spice: * Attachments * Spoiler attachments * Embeds +* Presence * Guild-Space details syncing * Channel-Room details syncing * Custom emoji list syncing diff --git a/src/d2m/actions/set-presence.js b/src/d2m/actions/set-presence.js new file mode 100644 index 0000000..e62456a --- /dev/null +++ b/src/d2m/actions/set-presence.js @@ -0,0 +1,45 @@ +// @ts-check + +const passthrough = require("../../passthrough") +const {sync, select} = passthrough +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") + +// Adding a debounce to all updates because events are issued multiple times, once for each guild. +// Sometimes a status update is even issued twice in a row for the same user+guild, weird! +const presenceDelay = 1500 +/** @type {Map} user ID -> cancelable timeout */ +const presenceDelayMap = new Map() + +/** + * @param {string} userID Discord user ID + * @param {string} status status field from Discord's PRESENCE_UPDATE event + */ +function setPresence(userID, status) { + // cancel existing timer if one is already set + if (presenceDelayMap.has(userID)) { + clearTimeout(presenceDelayMap.get(userID)) + } + // new timer, which will run if nothing else comes in soon + presenceDelayMap.set(userID, setTimeout(setPresenceCallback, presenceDelay, userID, status).unref()) +} + +/** + * @param {string} user_id Discord user ID + * @param {string} status status field from Discord's PRESENCE_UPDATE event + */ +function setPresenceCallback(user_id, status) { + presenceDelayMap.delete(user_id) + const mxid = select("sim", "mxid", {user_id}).pluck().get() + if (!mxid) return + const presence = + ( status === "online" ? "online" + : status === "offline" ? "offline" + : "unavailable") // idle, dnd, and anything else they dream up in the future + api.setPresence(presence, mxid).catch(e => { + console.error("d->m: Skipping presence update failure:") + console.error(e) + }) +} + +module.exports.setPresence = setPresence diff --git a/src/d2m/discord-client.js b/src/d2m/discord-client.js index 37b3eac..9745ec2 100644 --- a/src/d2m/discord-client.js +++ b/src/d2m/discord-client.js @@ -28,7 +28,7 @@ class DiscordClient { intents: [ "DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING", "GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS", - "MESSAGE_CONTENT" + "MESSAGE_CONTENT", "GUILD_PRESENCES" ], ws: { compress: false, diff --git a/src/d2m/discord-packets.js b/src/d2m/discord-packets.js index 2e97671..20ed07f 100644 --- a/src/d2m/discord-packets.js +++ b/src/d2m/discord-packets.js @@ -196,6 +196,9 @@ const utils = { } else if (message.t === "INTERACTION_CREATE") { await interactions.dispatchInteraction(message.d) + + } else if (message.t === "PRESENCE_UPDATE") { + eventDispatcher.onPresenceUpdate(message.d.user.id, message.d.status) } } catch (e) { diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index a80c451..c2e6b03 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -33,6 +33,8 @@ const mxUtils = require("../m2d/converters/utils") const speedbump = sync.require("./actions/speedbump") /** @type {import("./actions/retrigger")} */ const retrigger = sync.require("./actions/retrigger") +/** @type {import("./actions/set-presence")} */ +const setPresence = sync.require("./actions/set-presence") /** @type {any} */ // @ts-ignore bad types from semaphore const Semaphore = require("@chriscdn/promise-semaphore") @@ -369,5 +371,14 @@ module.exports = { */ async onExpressionsUpdate(client, data) { await createSpace.syncSpaceExpressions(data, false) + }, + + /** + * @param {string} userID + * @param {string} [status] + */ + async onPresenceUpdate(userID, status) { + if (!status) return + setPresence.setPresence(userID, status) } } diff --git a/src/matrix/api.js b/src/matrix/api.js index d1d9516..40fe6fb 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -408,6 +408,14 @@ async function setAccountData(type, content, mxid) { await mreq.mreq("PUT", `/client/v3/user/${mxid}/account_data/${type}`, content) } +/** + * @param {"online" | "offline" | "unavailable"} presence + * @param {string} mxid + */ +async function setPresence(presence, mxid) { + await mreq.mreq("PUT", path(`/client/v3/presence/${mxid}/status`, mxid), {presence}) +} + module.exports.path = path module.exports.register = register module.exports.createRoom = createRoom @@ -440,3 +448,4 @@ module.exports.ackEvent = ackEvent module.exports.getAlias = getAlias module.exports.getAccountData = getAccountData module.exports.setAccountData = setAccountData +module.exports.setPresence = setPresence From 69e3d64905c1a0f19b54e2577c166df6b8c678b7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Feb 2025 16:44:22 +1300 Subject: [PATCH 2/4] Handle replies to state events with no body --- src/m2d/converters/event-to-message.js | 39 ++++++++--------- src/m2d/converters/event-to-message.test.js | 46 +++++++++++++++++++++ 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 9a17817..24e59c9 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -476,7 +476,7 @@ async function eventToMessage(event, guild, di) { // Try to extract an accurate display name and avatar URL from the member event const member = await getMemberFromCacheOrHomeserver(event.room_id, event.sender, di?.api) if (member.displayname) displayName = member.displayname - if (member.avatar_url) avatarURL = mxUtils.getPublicUrlForMxc(member.avatar_url) || undefined + if (member.avatar_url) avatarURL = mxUtils.getPublicUrlForMxc(member.avatar_url) // If the display name is too long to be put into the webhook (80 characters is the maximum), // put the excess characters into displayNameRunoff, later to be put at the top of the message let [displayNameShortened, displayNameRunoff] = splitDisplayName(displayName) @@ -512,8 +512,7 @@ async function eventToMessage(event, guild, di) { // Is it editing a reply? We need special handling if it is. // Get the original event, then check if it was a reply const originalEvent = await di.api.getEvent(event.room_id, originalEventId) - if (!originalEvent) return - const repliedToEventId = originalEvent.content["m.relates_to"]?.["m.in_reply_to"]?.event_id + const repliedToEventId = originalEvent?.content?.["m.relates_to"]?.["m.in_reply_to"]?.event_id if (!repliedToEventId) return // After all that, it's an edit of a reply. @@ -576,34 +575,27 @@ async function eventToMessage(event, guild, di) { if (row) { replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} ` } - const sender = repliedToEvent.sender - const authorID = getUserOrProxyOwnerID(sender) - if (authorID) { - replyLine += `<@${authorID}>` - } else { - let senderName = select("member_cache", "displayname", {mxid: sender}).pluck().get() - if (!senderName) { - const match = sender.match(/@([^:]*)/) - assert(match) - senderName = match[1] - } - replyLine += `**Ⓜ${senderName}**` - } // If the event has been edited, the homeserver will include the relation in `unsigned`. if (repliedToEvent.unsigned?.["m.relations"]?.["m.replace"]?.content?.["m.new_content"]) { repliedToEvent = repliedToEvent.unsigned["m.relations"]["m.replace"] // Note: this changes which event_id is in repliedToEvent. repliedToEvent.content = repliedToEvent.content["m.new_content"] } - let contentPreview + /** @type {string} */ + let repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body const fileReplyContentAlternative = attachmentEmojis.get(repliedToEvent.content.msgtype) + let contentPreview if (fileReplyContentAlternative) { contentPreview = " " + fileReplyContentAlternative } else if (repliedToEvent.unsigned?.redacted_because) { contentPreview = " (in reply to a deleted message)" + } else if (typeof repliedToContent !== "string") { + // in reply to a weird metadata event like m.room.name, m.room.member... + // I'm not implementing text fallbacks for arbitrary room events. this should cover most cases + // this has never ever happened in the wild anyway + repliedToEvent.sender = "" + contentPreview = " (channel details edited)" } else { // Generate a reply preview for a standard message - /** @type {string} */ - let repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body repliedToContent = repliedToContent.replace(/.*<\/mx-reply>/s, "") // Remove everything before replies, so just use the actual message body repliedToContent = repliedToContent.replace(/^\s*
.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards repliedToContent = repliedToContent.replace(/(?:\n|
)+/g, " ") // Should all be on one line @@ -624,6 +616,15 @@ async function eventToMessage(event, guild, di) { contentPreview = "" } } + const sender = repliedToEvent.sender + const authorID = getUserOrProxyOwnerID(sender) + if (authorID) { + replyLine += `<@${authorID}>` + } else { + let senderName = select("member_cache", "displayname", {mxid: sender}).pluck().get() + if (!senderName) senderName = sender.match(/@([^:]*)/)?.[1] + if (senderName) replyLine += `**Ⓜ${senderName}**` + } replyLine = `-# > ${replyLine}${contentPreview}\n` })() diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index cc3d19a..d3871b3 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -2625,6 +2625,52 @@ test("event2message: rich reply to a deleted event", async t => { ) }) +test("event2message: rich reply to a state event with no body", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@ampflower:matrix.org", + content: { + msgtype: "m.text", + body: "> <@ampflower:matrix.org> changed the room topic\n\nnice room topic", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @ampflower:matrix.org changed the room topic
nice room topic", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$f-noT-d-Eo_Xgpc05Ww89ErUXku4NwKWYGHLzWKo1kU" + } + } + }, + event_id: "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + room_id: "!TqlyQmifxGUggEmdBN:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$f-noT-d-Eo_Xgpc05Ww89ErUXku4NwKWYGHLzWKo1kU", { + type: "m.room.topic", + sender: "@ampflower:matrix.org", + content: { + topic: "you're cute" + }, + user_id: "@ampflower:matrix.org" + }) + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "Ampflower 🌺", + content: "-# > <:L1:1144820033948762203><:L2:1144820084079087647> (channel details edited)\nnice room topic", + avatar_url: "https://bridge.example.org/download/matrix/cadence.moe/PRfhXYBTOalvgQYtmCLeUXko", + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + test("event2message: raw mentioning discord users in plaintext body works", async t => { t.deepEqual( await eventToMessage({ From 0f435e930e189cf28ed973f58a8fc252bbf77275 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Feb 2025 16:54:30 +1300 Subject: [PATCH 3/4] Per-guild presence sync settings On by default for existing and small guilds. Off for new large guilds. Can be toggled either way. --- src/d2m/actions/create-space.js | 4 +++- src/d2m/actions/set-presence.js | 14 +++++++++++++- src/d2m/discord-packets.js | 2 +- src/d2m/event-dispatcher.js | 9 +++++---- .../0020-add-presence-to-guild-space.sql | 5 +++++ src/db/orm-defs.d.ts | 1 + src/web/pug/guild.pug | 11 +++++++++++ src/web/routes/guild-settings.js | 17 +++++++++++++++++ 8 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 src/db/migrations/0020-add-presence-to-guild-space.sql diff --git a/src/d2m/actions/create-space.js b/src/d2m/actions/create-space.js index ce86789..85f9621 100644 --- a/src/d2m/actions/create-space.js +++ b/src/d2m/actions/create-space.js @@ -31,6 +31,8 @@ async function createSpace(guild, kstate) { const topic = kstate["m.room.topic/"]?.topic || undefined assert(name) + const memberCount = guild["member_count"] ?? guild.approximate_member_count ?? 0 + const enablePresenceByDefault = +(memberCount < 150) // could increase this later on if it doesn't cause any problems const globalAdmins = select("member_power", "mxid", {room_id: "*"}).pluck().all() const roomID = await createRoom.postApplyPowerLevels(kstate, async kstate => { @@ -50,7 +52,7 @@ async function createSpace(guild, kstate) { initial_state: await ks.kstateToState(kstate) }) }) - db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, roomID) + db.prepare("INSERT INTO guild_space (guild_id, space_id, presence) VALUES (?, ?, ?)").run(guild.id, roomID, enablePresenceByDefault) return roomID } diff --git a/src/d2m/actions/set-presence.js b/src/d2m/actions/set-presence.js index e62456a..6ff912d 100644 --- a/src/d2m/actions/set-presence.js +++ b/src/d2m/actions/set-presence.js @@ -11,11 +11,22 @@ const presenceDelay = 1500 /** @type {Map} user ID -> cancelable timeout */ const presenceDelayMap = new Map() +// Access the list of enabled guilds as needed rather than like multiple times per second when a user changes presence +/** @type {Set} */ +let presenceEnabledGuilds +function checkPresenceEnabledGuilds() { + presenceEnabledGuilds = new Set(select("guild_space", "guild_id", {presence: 1}).pluck().all()) +} +checkPresenceEnabledGuilds() + /** * @param {string} userID Discord user ID + * @param {string} guildID Discord guild ID that this presence applies to (really, the same presence applies to every single guild, but is delivered separately) * @param {string} status status field from Discord's PRESENCE_UPDATE event */ -function setPresence(userID, status) { +function setPresence(userID, guildID, status) { + // check if we care about this guild + if (!presenceEnabledGuilds.has(guildID)) return // cancel existing timer if one is already set if (presenceDelayMap.has(userID)) { clearTimeout(presenceDelayMap.get(userID)) @@ -43,3 +54,4 @@ function setPresenceCallback(user_id, status) { } module.exports.setPresence = setPresence +module.exports.checkPresenceEnabledGuilds = checkPresenceEnabledGuilds diff --git a/src/d2m/discord-packets.js b/src/d2m/discord-packets.js index 20ed07f..b9aba41 100644 --- a/src/d2m/discord-packets.js +++ b/src/d2m/discord-packets.js @@ -198,7 +198,7 @@ const utils = { await interactions.dispatchInteraction(message.d) } else if (message.t === "PRESENCE_UPDATE") { - eventDispatcher.onPresenceUpdate(message.d.user.id, message.d.status) + eventDispatcher.onPresenceUpdate(client, message.d) } } catch (e) { diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index c2e6b03..889e047 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -374,11 +374,12 @@ module.exports = { }, /** - * @param {string} userID - * @param {string} [status] + * @param {import("./discord-client")} client + * @param {DiscordTypes.GatewayPresenceUpdateDispatchData} data */ - async onPresenceUpdate(userID, status) { + async onPresenceUpdate(client, data) { + const status = data.status if (!status) return - setPresence.setPresence(userID, status) + setPresence.setPresence(data.user.id, data.guild_id, status) } } diff --git a/src/db/migrations/0020-add-presence-to-guild-space.sql b/src/db/migrations/0020-add-presence-to-guild-space.sql new file mode 100644 index 0000000..ea4c908 --- /dev/null +++ b/src/db/migrations/0020-add-presence-to-guild-space.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +ALTER TABLE guild_space ADD COLUMN presence INTEGER NOT NULL DEFAULT 1; + +COMMIT; diff --git a/src/db/orm-defs.d.ts b/src/db/orm-defs.d.ts index bab3c80..ac026f1 100644 --- a/src/db/orm-defs.d.ts +++ b/src/db/orm-defs.d.ts @@ -33,6 +33,7 @@ export type Models = { guild_id: string space_id: string privacy_level: number + presence: 0 | 1 } guild_active: { diff --git a/src/web/pug/guild.pug b/src/web/pug/guild.pug index 5a0d2fc..29ab301 100644 --- a/src/web/pug/guild.pug +++ b/src/web/pug/guild.pug @@ -102,6 +102,17 @@ block body #autocreate-loading if space_id + h3.mt32.fs-category Presence + .s-card + form.d-flex.ai-center.g8 + label.s-label.fl-grow1(for="presence") + | Show online statuses on Matrix + p.s-description This might cause lag on really big Discord servers. + - let value = !!select("guild_space", "presence", {guild_id}).pluck().get() + input(type="hidden" name="guild_id" value=guild_id) + input.s-toggle-switch.order-last#autocreate(name="presence" type="checkbox" hx-post="/api/presence" hx-indicator="#presence-loading" hx-disabled-elt="this" checked=value autocomplete="off") + #presence-loading + h3.mt32.fs-category Privacy level .s-card form(hx-post="/api/privacy-level" hx-trigger="change" hx-indicator="#privacy-level-loading" hx-disabled-elt="input") diff --git a/src/web/routes/guild-settings.js b/src/web/routes/guild-settings.js index bdc7148..7f2cb1d 100644 --- a/src/web/routes/guild-settings.js +++ b/src/web/routes/guild-settings.js @@ -8,6 +8,8 @@ const {as, db, sync, select} = require("../../passthrough") /** @type {import("../auth")} */ const auth = sync.require("../auth") +/** @type {import("../../d2m/actions/set-presence")} */ +const setPresence = sync.require("../../d2m/actions/set-presence") /** * @param {H3Event} event @@ -25,6 +27,10 @@ const schema = { guild_id: z.string(), autocreate: z.string().optional() }), + presence: z.object({ + guild_id: z.string(), + presence: z.string().optional() + }), privacyLevel: z.object({ guild_id: z.string(), level: z.enum(levels) @@ -51,6 +57,17 @@ as.router.post("/api/autocreate", defineEventHandler(async event => { return null // 204 })) +as.router.post("/api/presence", defineEventHandler(async event => { + const parsedBody = await readValidatedBody(event, schema.presence.parse) + const managed = await auth.getManagedGuilds(event) + if (!managed.has(parsedBody.guild_id)) throw createError({status: 403, message: "Forbidden", data: "Can't change settings for a guild you don't have Manage Server permissions in"}) + + db.prepare("UPDATE guild_space SET presence = ? WHERE guild_id = ?").run(+!!parsedBody.presence, parsedBody.guild_id) + setPresence.checkPresenceEnabledGuilds() + + return null // 204 +})) + as.router.post("/api/privacy-level", defineEventHandler(async event => { const parsedBody = await readValidatedBody(event, schema.privacyLevel.parse) const managed = await auth.getManagedGuilds(event) From fc6cb8e0d5c72692c8096df245eab54e40ec4c89 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Feb 2025 16:54:40 +1300 Subject: [PATCH 4/4] Web UI improvements --- src/m2d/converters/utils.js | 4 ++-- src/web/pug/guild_access_denied.pug | 13 +++++++------ src/web/pug/includes/template.pug | 4 ++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/m2d/converters/utils.js b/src/m2d/converters/utils.js index 17cb0fd..41cb0af 100644 --- a/src/m2d/converters/utils.js +++ b/src/m2d/converters/utils.js @@ -217,12 +217,12 @@ async function getViaServersQuery(roomID, api) { * @see https://matrix.org/blog/2024/06/20/matrix-v1.11-release/ implementation details * @see https://www.sqlite.org/fileformat2.html#record_format SQLite integer field size * @param {string} mxc - * @returns {string?} + * @returns {string | undefined} */ function getPublicUrlForMxc(mxc) { assert(hasher, "xxhash is not ready yet") const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) - if (!mediaParts) return null + if (!mediaParts) return undefined const serverAndMediaID = `${mediaParts[1]}/${mediaParts[2]}` const unsignedHash = hasher.h64(serverAndMediaID) diff --git a/src/web/pug/guild_access_denied.pug b/src/web/pug/guild_access_denied.pug index 1476697..42fea7b 100644 --- a/src/web/pug/guild_access_denied.pug +++ b/src/web/pug/guild_access_denied.pug @@ -5,12 +5,13 @@ block body .s-empty-state.wmx4.p48 != icons.Spots.SpotEmptyXL p You need to log in to manage your servers. - a.s-btn.s-btn__icon.s-btn__featured.s-btn__filled(href=rel("/oauth")) - != icons.Icons.IconDiscord - = ` Log in with Discord` - a.s-btn.s-btn__icon.s-btn__matrix.s-btn__filled(href=rel("/log-in-with-matrix")) - != icons.Icons.IconChatBubble - = ` Log in with Matrix` + .d-flex.jc-center.g8 + a.s-btn.s-btn__icon.s-btn__featured.s-btn__filled(href=rel("/oauth")) + != icons.Icons.IconDiscord + = ` Log in with Discord` + a.s-btn.s-btn__icon.s-btn__matrix.s-btn__filled(href=rel("/log-in-with-matrix")) + != icons.Icons.IconSpeechBubble + = ` Log in with Matrix` else if !guild_id .s-empty-state.wmx4.p48 diff --git a/src/web/pug/includes/template.pug b/src/web/pug/includes/template.pug index 9c51d6c..ac15376 100644 --- a/src/web/pug/includes/template.pug +++ b/src/web/pug/includes/template.pug @@ -81,6 +81,10 @@ html(lang="en") else if managed.size button.s-topbar--item.s-btn.s-btn__muted.s-btn__dropdown.pr24.s-user-card.bar0.fc-black(popovertarget="guilds") | Your servers + else + .d-flex.ai-center + .s-badge.s-badge__bot.py6.px16.bar-md + | No servers available #guilds(popover data-popper-placement="bottom" style="display: revert; width: revert;").s-popover.overflow-visible .s-popover--arrow.s-popover--arrow__tc .s-popover--content.overflow-y-auto.overflow-x-hidden