From 8987107685f8748288be0e247163c5722195d2c3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 18 Jan 2024 00:30:55 +1300 Subject: [PATCH] Backfill missed pins and pins from the past --- d2m/actions/update-pins.js | 33 +++++++--- d2m/converters/pins-to-list.test.js | 2 +- d2m/discord-packets.js | 8 +++ d2m/event-dispatcher.js | 45 ++++++++++++- .../0008-add-last-bridged-pin-timestamp.sql | 5 ++ db/orm-defs.d.ts | 1 + discord/utils.js | 2 +- discord/utils.test.js | 63 ++++++++++++++++++- stdin.js | 1 + 9 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 db/migrations/0008-add-last-bridged-pin-timestamp.sql diff --git a/d2m/actions/update-pins.js b/d2m/actions/update-pins.js index 40cc358..5d98501 100644 --- a/d2m/actions/update-pins.js +++ b/d2m/actions/update-pins.js @@ -1,22 +1,37 @@ // @ts-check const passthrough = require("../../passthrough") -const {discord, sync} = passthrough +const {discord, sync, db} = passthrough /** @type {import("../converters/pins-to-list")} */ const pinsToList = sync.require("../converters/pins-to-list") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** - * @param {string} channelID - * @param {string} roomID + * @template {string | null | undefined} T + * @param {T} timestamp + * @returns {T extends string ? number : null} */ -async function updatePins(channelID, roomID) { - const pins = await discord.snow.channel.getChannelPinnedMessages(channelID) - const eventIDs = pinsToList.pinsToList(pins) - await api.sendState(roomID, "m.room.pinned_events", "", { - pinned: eventIDs - }) +function convertTimestamp(timestamp) { + // @ts-ignore + return typeof timestamp === "string" ? Math.floor(new Date(timestamp).getTime() / 1000) : null } +/** + * @param {string} channelID + * @param {string} roomID + * @param {number?} convertedTimestamp + */ +async function updatePins(channelID, roomID, convertedTimestamp) { + const pins = await discord.snow.channel.getChannelPinnedMessages(channelID) + const eventIDs = pinsToList.pinsToList(pins) + if (pins.length === eventIDs.length || eventIDs.length) { + await api.sendState(roomID, "m.room.pinned_events", "", { + pinned: eventIDs + }) + } + db.prepare("UPDATE channel_room SET last_bridged_pin_timestamp = ? WHERE channel_id = ?").run(convertedTimestamp || 0, channelID) +} + +module.exports.convertTimestamp = convertTimestamp module.exports.updatePins = updatePins diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js index 92e5678..8a6daea 100644 --- a/d2m/converters/pins-to-list.test.js +++ b/d2m/converters/pins-to-list.test.js @@ -6,7 +6,7 @@ test("pins2list: converts known IDs, ignores unknown IDs", t => { const result = pinsToList(data.pins.faked) t.deepEqual(result, [ "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4", - "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuAno", + "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA", "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg" ]) }) diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index 1129b71..6ba839a 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -43,6 +43,7 @@ const utils = { } if (listen === "full") { eventDispatcher.checkMissedExpressions(message.d) + eventDispatcher.checkMissedPins(client, message.d) eventDispatcher.checkMissedMessages(client, message.d) } @@ -94,6 +95,13 @@ const utils = { client.channels.set(message.d.id, message.d) + } else if (message.t === "CHANNEL_PINS_UPDATE") { + const channel = client.channels.get(message.d.channel_id) + if (channel) { + channel["last_pin_timestamp"] = message.d.last_pin_timestamp + } + + } else if (message.t === "GUILD_DELETE") { client.guilds.delete(message.d.id) const channels = client.guildChannelMap.get(message.d.id) diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 7974be6..6c872fb 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -25,9 +25,15 @@ const createSpace = sync.require("./actions/create-space") const updatePins = sync.require("./actions/update-pins") /** @type {import("../matrix/api")}) */ const api = sync.require("../matrix/api") +/** @type {import("../discord/utils")} */ +const utils = sync.require("../discord/utils") /** @type {import("../discord/discord-command-handler")}) */ const discordCommandHandler = sync.require("../discord/discord-command-handler") +/** @type {any} */ // @ts-ignore bad types from semaphore +const Semaphore = require("@chriscdn/promise-semaphore") +const checkMissedPinsSema = new Semaphore() + let lastReportedEvent = 0 // Grab Discord events we care about for the bridge, check them, and pass them on @@ -103,6 +109,14 @@ module.exports = { const latestWasBridged = prepared.get(channel.last_message_id) if (latestWasBridged) continue + // Permissions check + const member = guild.members.find(m => m.user?.id === client.user.id) + if (!member) return + if (!("permission_overwrites" in channel)) continue + const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY + if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel + /** More recent messages come first. */ // console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) let messages @@ -132,6 +146,34 @@ module.exports = { } }, + /** + * When logging back in, check if the pins on Matrix-side are up to date. If they aren't, update all pins. + * Rather than query every room on Matrix-side, we cache the latest pinned message in the database and compare against that. + * @param {import("./discord-client")} client + * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild + */ + async checkMissedPins(client, guild) { + if (guild.unavailable) return + const member = guild.members.find(m => m.user?.id === client.user.id) + if (!member) return + for (const channel of guild.channels) { + if (!("last_pin_timestamp" in channel) || !channel.last_pin_timestamp) continue // Only care about channels that have pins + if (!("permission_overwrites" in channel)) continue + const lastPin = updatePins.convertTimestamp(channel.last_pin_timestamp) + + // Permissions check + const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY + if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel + + const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get() + if (!row) continue // Only care about already bridged channels + if (row.last_bridged_pin_timestamp == null || lastPin > row.last_bridged_pin_timestamp) { + checkMissedPinsSema.request(() => updatePins.updatePins(channel.id, row.room_id, lastPin)) + } + } + }, + /** * When logging back in, check if we missed any changes to emojis or stickers. Apply the changes if so. * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild @@ -183,7 +225,8 @@ module.exports = { async onChannelPinsUpdate(client, data) { const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() if (!roomID) return // No target room to update pins in - await updatePins.updatePins(data.channel_id, roomID) + const convertedTimestamp = updatePins.convertTimestamp(data.last_pin_timestamp) + await updatePins.updatePins(data.channel_id, roomID, convertedTimestamp) }, /** diff --git a/db/migrations/0008-add-last-bridged-pin-timestamp.sql b/db/migrations/0008-add-last-bridged-pin-timestamp.sql new file mode 100644 index 0000000..dbcb81d --- /dev/null +++ b/db/migrations/0008-add-last-bridged-pin-timestamp.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +ALTER TABLE channel_room ADD COLUMN last_bridged_pin_timestamp INTEGER; + +COMMIT; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index ec8d498..64e5c77 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -6,6 +6,7 @@ export type Models = { nick: string | null thread_parent: string | null custom_avatar: string | null + last_bridged_pin_timestamp: number | null } event_message: { diff --git a/discord/utils.js b/discord/utils.js index ceb4468..6788bcf 100644 --- a/discord/utils.js +++ b/discord/utils.js @@ -34,7 +34,7 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) { // Role deny overwrite => userRoles.includes(overwrite.id) && (allowed &= ~BigInt(overwrite.deny)), // Role allow - overwrite => userRoles.includes(overwrite.id) && (allowed |= ~BigInt(overwrite.allow)), + overwrite => userRoles.includes(overwrite.id) && (allowed |= BigInt(overwrite.allow)), // User deny overwrite => overwrite.id === userID && (allowed &= ~BigInt(overwrite.deny)), // User allow diff --git a/discord/utils.test.js b/discord/utils.test.js index fd064ef..1f3783f 100644 --- a/discord/utils.test.js +++ b/discord/utils.test.js @@ -18,6 +18,67 @@ test("discord utils: converts snowflake to timestamp", t => { t.equal(utils.snowflakeToTimestampExact("86913608335773696"), 1440792219004) }) -test("discerd utils: converts timestamp to snowflake", t => { +test("discord utils: converts timestamp to snowflake", t => { t.match(utils.timestampToSnowflakeInexact(1440792219004), /^869136083357.....$/) }) + +test("getPermissions: channel overwrite to allow role works", t => { + const guildRoles = [ + { + version: 1695412489043, + unicode_emoji: null, + tags: {}, + position: 0, + permissions: "559623605571137", + name: "@everyone", + mentionable: false, + managed: false, + id: "1154868424724463687", + icon: null, + hoist: false, + flags: 0, + color: 0 + }, + { + version: 1695412604262, + unicode_emoji: null, + tags: { bot_id: "466378653216014359" }, + position: 1, + permissions: "536995904", + name: "PluralKit", + mentionable: false, + managed: true, + id: "1154868908336099444", + icon: null, + hoist: false, + flags: 0, + color: 0 + }, + { + version: 1698778936921, + unicode_emoji: null, + tags: {}, + position: 1, + permissions: "536870912", + name: "web hookers", + mentionable: false, + managed: false, + id: "1168988246680801360", + icon: null, + hoist: false, + flags: 0, + color: 0 + } + ] + const userRoles = [ "1168988246680801360" ] + const userID = "684280192553844747" + const overwrites = [ + { type: 0, id: "1154868908336099444", deny: "0", allow: "1024" }, + { type: 0, id: "1154868424724463687", deny: "1024", allow: "0" }, + { type: 0, id: "1168988246680801360", deny: "0", allow: "1024" }, + { type: 1, id: "353373325575323648", deny: "0", allow: "1024" } + ] + const permissions = utils.getPermissions(userRoles, guildRoles, userID, overwrites) + const want = BigInt(1 << 10 | 1 << 16) + t.equal((permissions & want), want) +}) diff --git a/stdin.js b/stdin.js index 587b176..da69d7c 100644 --- a/stdin.js +++ b/stdin.js @@ -16,6 +16,7 @@ const api = sync.require("./matrix/api") const file = sync.require("./matrix/file") const sendEvent = sync.require("./m2d/actions/send-event") const eventDispatcher = sync.require("./d2m/event-dispatcher") +const updatePins = sync.require("./d2m/actions/update-pins") const ks = sync.require("./matrix/kstate") const guildID = "112760669178241024"