diff --git a/.gitignore b/.gitignore index c38dd88..9c175d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,7 @@ -# Secrets +node_modules config.js registration.yaml -ooye.db* -events.db* -backfill.db* -custom-webroot - -# Automatically generated -node_modules coverage +db/ooye.db* test/res/* !test/res/lottie* -icon.svg -*~ -.#* -\#*# -launch.json diff --git a/addbot.js b/addbot.js old mode 100755 new mode 100644 index f0e850c..667fbab --- a/addbot.js +++ b/addbot.js @@ -1,27 +1,15 @@ -#!/usr/bin/env node // @ts-check -const DiscordTypes = require("discord-api-types/v10") - -const {reg} = require("./src/matrix/read-registration") -const token = reg.ooye.discord_token -const id = Buffer.from(token.split(".")[0], "base64").toString() -const permissions = -( DiscordTypes.PermissionFlagsBits.ManageWebhooks -| DiscordTypes.PermissionFlagsBits.ManageGuildExpressions -| DiscordTypes.PermissionFlagsBits.ManageMessages -| DiscordTypes.PermissionFlagsBits.PinMessages -| DiscordTypes.PermissionFlagsBits.UseExternalEmojis) +const config = require("./config") function addbot() { - return `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=${id}&scope=bot&permissions=${permissions} ` + const token = config.discordToken + const id = Buffer.from(token.split(".")[0], "base64") + return `Open this link to add the bot to a Discord server:\nhttps://discord.com/oauth2/authorize?client_id=${id}&scope=bot&permissions=1610883072 ` } -/* c8 ignore next 3 */ if (process.argv.find(a => a.endsWith("addbot") || a.endsWith("addbot.js"))) { console.log(addbot()) } -module.exports.id = id module.exports.addbot = addbot -module.exports.permissions = permissions diff --git a/addbot.sh b/addbot.sh index d40d0da..6c3ff4b 100755 --- a/addbot.sh +++ b/addbot.sh @@ -1,3 +1,3 @@ #!/usr/bin/env sh echo "Open this link to add the bot to a Discord server:" -echo "https://discord.com/oauth2/authorize?client_id=$(grep discord_token registration.yaml | sed -E 's!.*: ["'\'']([A-Za-z0-9+=/_-]*).*!\1!g' | base64 -d)&scope=bot&permissions=1610883072" +echo "https://discord.com/oauth2/authorize?client_id=$(grep discordToken config.js | sed -E 's!.*: ["'\'']([A-Za-z0-9+=/_-]*).*!\1!g' | base64 -d)&scope=bot&permissions=1610883072" diff --git a/config.example.js b/config.example.js new file mode 100644 index 0000000..0d1a29e --- /dev/null +++ b/config.example.js @@ -0,0 +1,3 @@ +module.exports = { + discordToken: "yes" +} diff --git a/src/d2m/actions/add-reaction.js b/d2m/actions/add-reaction.js similarity index 91% rename from src/d2m/actions/add-reaction.js rename to d2m/actions/add-reaction.js index 476f8dd..b131f13 100644 --- a/src/d2m/actions/add-reaction.js +++ b/d2m/actions/add-reaction.js @@ -21,11 +21,11 @@ async function addReaction(data) { const user = data.member?.user assert.ok(user && user.username) - const parentID = select("event_message", "event_id", {message_id: data.message_id}, "ORDER BY reaction_part").pluck().get() + const parentID = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get() if (!parentID) return // Nothing can be done if the parent message was never bridged. assert.equal(typeof parentID, "string") - const key = await emojiToKey.emojiToKey(data.emoji, data.message_id) + const key = await emojiToKey.emojiToKey(data.emoji) const shortcode = key.startsWith("mxc://") ? `:${data.emoji.name}:` : undefined const roomID = await createRoom.ensureRoom(data.channel_id) diff --git a/src/d2m/actions/announce-thread.js b/d2m/actions/announce-thread.js similarity index 96% rename from src/d2m/actions/announce-thread.js rename to d2m/actions/announce-thread.js index c8cbf9d..324c7a5 100644 --- a/src/d2m/actions/announce-thread.js +++ b/d2m/actions/announce-thread.js @@ -1,6 +1,6 @@ // @ts-check -const assert = require("assert").strict +const assert = require("assert") const passthrough = require("../../passthrough") const {discord, sync, db, select} = passthrough diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js new file mode 100644 index 0000000..6994b57 --- /dev/null +++ b/d2m/actions/create-room.js @@ -0,0 +1,414 @@ +// @ts-check + +const assert = require("assert").strict +const DiscordTypes = require("discord-api-types/v10") +const reg = require("../../matrix/read-registration") + +const passthrough = require("../../passthrough") +const {discord, sync, db, select} = passthrough +/** @type {import("../../matrix/file")} */ +const file = sync.require("../../matrix/file") +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/kstate")} */ +const ks = sync.require("../../matrix/kstate") +/** @type {import("./create-space")}) */ +const createSpace = sync.require("./create-space") // watch out for the require loop + +/** + * There are 3 levels of room privacy: + * 0: Room is invite-only. + * 1: Anybody can use a link to join. + * 2: Room is published in room directory. + */ +const PRIVACY_ENUMS = { + PRESET: ["private_chat", "public_chat", "public_chat"], + VISIBILITY: ["private", "private", "public"], + SPACE_HISTORY_VISIBILITY: ["invited", "world_readable", "world_readable"], // copying from element client + ROOM_HISTORY_VISIBILITY: ["shared", "shared", "world_readable"], // any events sent after are visible, but for world_readable anybody can read without even joining + GUEST_ACCESS: ["can_join", "forbidden", "forbidden"], // whether guests can join space if other conditions are met + SPACE_JOIN_RULES: ["invite", "public", "public"], + ROOM_JOIN_RULES: ["restricted", "public", "public"] +} + +const DEFAULT_PRIVACY_LEVEL = 0 + +/** @type {Map>} channel ID -> Promise */ +const inflightRoomCreate = new Map() + +/** + * Async because it gets all room state from the homeserver. + * @param {string} roomID + */ +async function roomToKState(roomID) { + const root = await api.getAllState(roomID) + return ks.stateToKState(root) +} + +/** + * @param {string} roomID + * @param {any} kstate + */ +function applyKStateDiffToRoom(roomID, kstate) { + const events = ks.kstateToState(kstate) + return Promise.all(events.map(({type, state_key, content}) => + api.sendState(roomID, type, state_key, content) + )) +} + +/** + * @param {{id: string, name: string, topic?: string?, type: number}} channel + * @param {{id: string}} guild + * @param {string | null | undefined} customName + */ +function convertNameAndTopic(channel, guild, customName) { + let channelPrefix = + ( channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] " + : channel.type === DiscordTypes.ChannelType.PrivateThread ? "[🔒⛓️] " + : channel.type === DiscordTypes.ChannelType.GuildVoice ? "[🔊] " + : "") + const chosenName = customName || (channelPrefix + channel.name); + const maybeTopicWithPipe = channel.topic ? ` | ${channel.topic}` : ''; + const maybeTopicWithNewlines = channel.topic ? `${channel.topic}\n\n` : ''; + const channelIDPart = `Channel ID: ${channel.id}`; + const guildIDPart = `Guild ID: ${guild.id}`; + + const convertedTopic = customName + ? `#${channel.name}${maybeTopicWithPipe}\n\n${channelIDPart}\n${guildIDPart}` + : `${maybeTopicWithNewlines}${channelIDPart}\n${guildIDPart}`; + + return [chosenName, convertedTopic]; +} + +/** + * Async because it may create the guild and/or upload the guild icon to mxc. + * @param {DiscordTypes.APIGuildTextChannel | DiscordTypes.APIThreadChannel} channel + * @param {DiscordTypes.APIGuild} guild + */ +async function channelToKState(channel, guild) { + const spaceID = await createSpace.ensureSpace(guild) + assert(typeof spaceID === "string") + const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get() + assert(typeof privacyLevel === "number") + + const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get() + const customName = row?.nick + const customAvatar = row?.custom_avatar + const [convertedName, convertedTopic] = convertNameAndTopic(channel, guild, customName) + + const avatarEventContent = {} + if (customAvatar) { + avatarEventContent.url = customAvatar + } else if (guild.icon) { + avatarEventContent.discord_path = file.guildIcon(guild) + avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API + } + + let history_visibility = PRIVACY_ENUMS.ROOM_HISTORY_VISIBILITY[privacyLevel] + if (channel["thread_metadata"]) history_visibility = "world_readable" + + /** @type {{join_rule: string, allow?: any}} */ + let join_rules = { + join_rule: "restricted", + allow: [{ + type: "m.room_membership", + room_id: spaceID + }] + } + if (PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel] !== "restricted") { + join_rules = {join_rule: PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel]} + } + + const channelKState = { + "m.room.name/": {name: convertedName}, + "m.room.topic/": {topic: convertedTopic}, + "m.room.avatar/": avatarEventContent, + "m.room.guest_access/": {guest_access: PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]}, + "m.room.history_visibility/": {history_visibility}, + [`m.space.parent/${spaceID}`]: { + via: [reg.ooye.server_name], + canonical: true + }, + /** @type {{join_rule: string, [x: string]: any}} */ + "m.room.join_rules/": join_rules, + "m.room.power_levels/": { + events: { + "m.room.avatar": 0 + }, + users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {}) + }, + "chat.schildi.hide_ui/read_receipts": { + hidden: true + }, + [`uk.half-shot.bridge/moe.cadence.ooye://discord/${guild.id}/${channel.id}`]: { + bridgebot: `@${reg.sender_localpart}:${reg.ooye.server_name}`, + protocol: { + id: "discord", + displayname: "Discord" + }, + network: { + id: guild.id, + displayname: guild.name, + avatar_url: await file.uploadDiscordFileToMxc(file.guildIcon(guild)) + }, + channel: { + id: channel.id, + displayname: channel.name, + external_url: `https://discord.com/channels/${guild.id}/${channel.id}` + } + } + } + + return {spaceID, privacyLevel, channelKState} +} + +/** + * Create a bridge room, store the relationship in the database, and add it to the guild's space. + * @param {DiscordTypes.APIGuildTextChannel} channel + * @param guild + * @param {string} spaceID + * @param {any} kstate + * @param {number} privacyLevel + * @returns {Promise} room ID + */ +async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { + let threadParent = null + if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id + + // Name and topic can be done earlier in room creation rather than in initial_state + // https://spec.matrix.org/latest/client-server-api/#creation + const name = kstate["m.room.name/"].name + delete kstate["m.room.name/"] + assert(name) + const topic = kstate["m.room.topic/"].topic + delete kstate["m.room.topic/"] + assert(topic) + + const roomID = await postApplyPowerLevels(kstate, async kstate => { + const roomID = await api.createRoom({ + name, + topic, + preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway + visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel], + invite: [], + initial_state: ks.kstateToState(kstate) + }) + + db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent) + + return roomID + }) + + // Put the newly created child into the space, no need to await this + _syncSpaceMember(channel, spaceID, roomID) + + return roomID +} + +/** + * Handling power levels separately. The spec doesn't specify what happens, Dendrite differs, + * and Synapse does an absolutely insane *shallow merge* of what I provide on top of what it creates. + * We don't want the `events` key to be overridden completely. + * https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/room.py#L1170-L1210 + * https://github.com/matrix-org/matrix-spec/issues/492 + * @param {any} kstate + * @param {(_: any) => Promise} callback must return room ID + * @returns {Promise} room ID + */ +async function postApplyPowerLevels(kstate, callback) { + const powerLevelContent = kstate["m.room.power_levels/"] + const kstateWithoutPowerLevels = {...kstate} + delete kstateWithoutPowerLevels["m.room.power_levels/"] + + /** @type {string} */ + const roomID = await callback(kstateWithoutPowerLevels) + + // Now *really* apply the power level overrides on top of what Synapse *really* set + if (powerLevelContent) { + const newRoomKState = await roomToKState(roomID) + const newRoomPowerLevelsDiff = ks.diffKState(newRoomKState, {"m.room.power_levels/": powerLevelContent}) + await applyKStateDiffToRoom(roomID, newRoomPowerLevelsDiff) + } + + return roomID +} + +/** + * @param {DiscordTypes.APIGuildChannel} channel + */ +function channelToGuild(channel) { + const guildID = channel.guild_id + assert(guildID) + const guild = discord.guilds.get(guildID) + assert(guild) + return guild +} + +/* + Ensure flow: + 1. Get IDs + 2. Does room exist? If so great! + (it doesn't, so it needs to be created) + 3. Get kstate for channel + 4. Create room, return new ID + + Ensure + sync flow: + 1. Get IDs + 2. Does room exist? + 2.5: If room does exist AND wasn't asked to sync: return here + 3. Get kstate for channel + 4. Create room with kstate if room doesn't exist + 5. Get and update room state with kstate if room does exist +*/ + +/** + * @param {string} channelID + * @param {boolean} shouldActuallySync false if just need to ensure room exists (which is a quick database check), true if also want to sync room data when it does exist (slow) + * @returns {Promise} room ID + */ +async function _syncRoom(channelID, shouldActuallySync) { + /** @ts-ignore @type {DiscordTypes.APIGuildChannel} */ + const channel = discord.channels.get(channelID) + assert.ok(channel) + const guild = channelToGuild(channel) + + if (inflightRoomCreate.has(channelID)) { + await inflightRoomCreate.get(channelID) // just waiting, and then doing a new db query afterwards, is the simplest way of doing it + } + + const existing = select("channel_room", ["room_id", "thread_parent"], {channel_id: channelID}).get() + + if (!existing) { + const creation = (async () => { + const {spaceID, privacyLevel, channelKState} = await channelToKState(channel, guild) + const roomID = await createRoom(channel, guild, spaceID, channelKState, privacyLevel) + inflightRoomCreate.delete(channelID) // OK to release inflight waiters now. they will read the correct `existing` row + return roomID + })() + inflightRoomCreate.set(channelID, creation) + return creation // Naturally, the newly created room is already up to date, so we can always skip syncing here. + } + + const roomID = existing.room_id + + if (!shouldActuallySync) { + return existing.room_id // only need to ensure room exists, and it does. return the room ID + } + + console.log(`[room sync] to matrix: ${channel.name}`) + + const {spaceID, channelKState} = await channelToKState(channel, guild) // calling this in both branches because we don't want to calculate this if not syncing + + // sync channel state to room + const roomKState = await roomToKState(roomID) + if (+roomKState["m.room.create/"].room_version <= 8) { + // join_rule `restricted` is not available in room version < 8 and not working properly in version == 8 + // read more: https://spec.matrix.org/v1.8/rooms/v9/ + // we have to use `public` instead, otherwise the room will be unjoinable. + channelKState["m.room.join_rules/"] = {join_rule: "public"} + } + const roomDiff = ks.diffKState(roomKState, channelKState) + const roomApply = applyKStateDiffToRoom(roomID, roomDiff) + db.prepare("UPDATE channel_room SET name = ? WHERE room_id = ?").run(channel.name, roomID) + + // sync room as space member + const spaceApply = _syncSpaceMember(channel, spaceID, roomID) + await Promise.all([roomApply, spaceApply]) + + return roomID +} + +/** Ensures the room exists. If it doesn't, creates the room with an accurate initial state. */ +function ensureRoom(channelID) { + return _syncRoom(channelID, false) +} + +/** Actually syncs. Gets all room state from the homeserver in order to diff, and uploads the icon to mxc if it has changed. */ +function syncRoom(channelID) { + return _syncRoom(channelID, true) +} + +async function _unbridgeRoom(channelID) { + /** @ts-ignore @type {DiscordTypes.APIGuildChannel} */ + const channel = discord.channels.get(channelID) + assert.ok(channel) + return unbridgeDeletedChannel(channel.id, channel.guild_id) +} + +async function unbridgeDeletedChannel(channelID, guildID) { + const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() + assert.ok(roomID) + const spaceID = select("guild_space", "space_id", {guild_id: guildID}).pluck().get() + assert.ok(spaceID) + + // remove room from being a space member + await api.sendState(roomID, "m.space.parent", spaceID, {}) + await api.sendState(spaceID, "m.space.child", roomID, {}) + + // remove declaration that the room is bridged + await api.sendState(roomID, "uk.half-shot.bridge", `moe.cadence.ooye://discord/${guildID}/${channelID}`, {}) + + // send a notification in the room + await api.sendEvent(roomID, "m.room.message", { + msgtype: "m.notice", + body: "⚠️ This room was removed from the bridge." + }) + + // leave room + await api.leaveRoom(roomID) + + // delete room from database + const {changes} = db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channelID) + assert.equal(changes, 1) +} + +/** + * Async because it gets all space state from the homeserver, then if necessary sends one state event back. + * @param {DiscordTypes.APIGuildTextChannel} channel + * @param {string} spaceID + * @param {string} roomID + * @returns {Promise} + */ +async function _syncSpaceMember(channel, spaceID, roomID) { + const spaceKState = await roomToKState(spaceID) + let spaceEventContent = {} + if ( + channel.type !== DiscordTypes.ChannelType.PrivateThread // private threads do not belong in the space (don't offer people something they can't join) + && !channel["thread_metadata"]?.archived // archived threads do not belong in the space (don't offer people conversations that are no longer relevant) + ) { + spaceEventContent = { + via: [reg.ooye.server_name] + } + } + const spaceDiff = ks.diffKState(spaceKState, { + [`m.space.child/${roomID}`]: spaceEventContent + }) + return applyKStateDiffToRoom(spaceID, spaceDiff) +} + +async function createAllForGuild(guildID) { + const channelIDs = discord.guildChannelMap.get(guildID) + assert.ok(channelIDs) + for (const channelID of channelIDs) { + const allowedTypes = [DiscordTypes.ChannelType.GuildText, DiscordTypes.ChannelType.PublicThread] + // @ts-ignore + if (allowedTypes.includes(discord.channels.get(channelID)?.type)) { + const roomID = await syncRoom(channelID) + console.log(`synced ${channelID} <-> ${roomID}`) + } + } +} + +module.exports.DEFAULT_PRIVACY_LEVEL = DEFAULT_PRIVACY_LEVEL +module.exports.PRIVACY_ENUMS = PRIVACY_ENUMS +module.exports.createRoom = createRoom +module.exports.ensureRoom = ensureRoom +module.exports.syncRoom = syncRoom +module.exports.createAllForGuild = createAllForGuild +module.exports.channelToKState = channelToKState +module.exports.roomToKState = roomToKState +module.exports.applyKStateDiffToRoom = applyKStateDiffToRoom +module.exports.postApplyPowerLevels = postApplyPowerLevels +module.exports._convertNameAndTopic = convertNameAndTopic +module.exports._unbridgeRoom = _unbridgeRoom +module.exports.unbridgeDeletedChannel = unbridgeDeletedChannel diff --git a/d2m/actions/create-room.test.js b/d2m/actions/create-room.test.js new file mode 100644 index 0000000..93f9203 --- /dev/null +++ b/d2m/actions/create-room.test.js @@ -0,0 +1,89 @@ +// @ts-check + +const {channelToKState, _convertNameAndTopic} = require("./create-room") +const {kstateStripConditionals} = require("../../matrix/kstate") +const {test} = require("supertape") +const testData = require("../../test/data") + +const passthrough = require("../../passthrough") +const {db} = passthrough + +test("channel2room: discoverable privacy room", async t => { + db.prepare("UPDATE guild_space SET privacy_level = 2").run() + t.deepEqual( + kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)), + Object.assign({}, testData.room.general, { + "m.room.guest_access/": {guest_access: "forbidden"}, + "m.room.join_rules/": {join_rule: "public"}, + "m.room.history_visibility/": {history_visibility: "world_readable"} + }) + ) +}) + +test("channel2room: linkable privacy room", async t => { + db.prepare("UPDATE guild_space SET privacy_level = 1").run() + t.deepEqual( + kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)), + Object.assign({}, testData.room.general, { + "m.room.guest_access/": {guest_access: "forbidden"}, + "m.room.join_rules/": {join_rule: "public"} + }) + ) +}) + +test("channel2room: invite-only privacy room", async t => { + db.prepare("UPDATE guild_space SET privacy_level = 0").run() + t.deepEqual( + kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)), + testData.room.general + ) +}) + +test("convertNameAndTopic: custom name and topic", t => { + t.deepEqual( + _convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, "hauntings"), + ["hauntings", "#the-twilight-zone | Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"] + ) +}) + +test("convertNameAndTopic: custom name, no topic", t => { + t.deepEqual( + _convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, "hauntings"), + ["hauntings", "#the-twilight-zone\n\nChannel ID: 123\nGuild ID: 456"] + ) +}) + +test("convertNameAndTopic: original name and topic", t => { + t.deepEqual( + _convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 0}, {id: "456"}, null), + ["the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"] + ) +}) + +test("convertNameAndTopic: original name, no topic", t => { + t.deepEqual( + _convertNameAndTopic({id: "123", name: "the-twilight-zone", type: 0}, {id: "456"}, null), + ["the-twilight-zone", "Channel ID: 123\nGuild ID: 456"] + ) +}) + +test("convertNameAndTopic: public thread icon", t => { + t.deepEqual( + _convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 11}, {id: "456"}, null), + ["[⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"] + ) +}) + +test("convertNameAndTopic: private thread icon", t => { + t.deepEqual( + _convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 12}, {id: "456"}, null), + ["[🔒⛓️] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"] + ) +}) + +test("convertNameAndTopic: voice channel icon", t => { + t.deepEqual( + _convertNameAndTopic({id: "123", name: "the-twilight-zone", topic: "Spooky stuff here. :ghost:", type: 2}, {id: "456"}, null), + ["[🔊] the-twilight-zone", "Spooky stuff here. :ghost:\n\nChannel ID: 123\nGuild ID: 456"] + ) +}) diff --git a/src/d2m/actions/create-space.js b/d2m/actions/create-space.js similarity index 69% rename from src/d2m/actions/create-space.js rename to d2m/actions/create-space.js index 7a751e2..7d50199 100644 --- a/src/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -1,10 +1,9 @@ // @ts-check const assert = require("assert").strict -const {isDeepStrictEqual} = require("util") const DiscordTypes = require("discord-api-types/v10") -const Ty = require("../../types") -const {reg} = require("../../matrix/read-registration") +const deepEqual = require("deep-equal") +const reg = require("../../matrix/read-registration") const passthrough = require("../../passthrough") const {discord, sync, db, select} = passthrough @@ -31,12 +30,8 @@ async function createSpace(guild, kstate) { const topic = kstate["m.room.topic/"]?.topic || undefined assert(name) - const memberCount = guild["member_count"] ?? guild.approximate_member_count ?? 0 - const enablePresenceByDefault = +(memberCount < 50) // scary! all active users in a presence-enabled guild will be pinging the server every <30 seconds to stay online - const globalAdmins = select("member_power", "mxid", {room_id: "*"}).pluck().all() - - const roomCreate = await createRoom.postApplyPowerLevels(kstate, async kstate => { - const roomID = await api.createRoom({ + const roomID = await createRoom.postApplyPowerLevels(kstate, async kstate => { + return api.createRoom({ name, preset: createRoom.PRIVACY_ENUMS.PRESET[createRoom.DEFAULT_PRIVACY_LEVEL], // New spaces will have to use the default privacy level; we obviously can't look up the existing entry visibility: createRoom.PRIVACY_ENUMS.VISIBILITY[createRoom.DEFAULT_PRIVACY_LEVEL], @@ -44,17 +39,15 @@ async function createSpace(guild, kstate) { events_default: 100, // space can only be managed by bridge invite: 0 // any existing member can invite others }, - invite: globalAdmins, + invite: reg.ooye.invite, topic, - initial_state: await ks.kstateToState(kstate), - creation_content: ks.kstateToCreationContent(kstate) + creation_content: { + type: "m.space" + }, + initial_state: ks.kstateToState(kstate) }) - const roomCreate = await api.getStateEventOuter(roomID, "m.room.create", "") - return roomCreate }) - const roomID = roomCreate.room_id - - db.prepare("INSERT INTO guild_space (guild_id, space_id, presence) VALUES (?, ?, ?)").run(guild.id, roomID, enablePresenceByDefault) + db.prepare("INSERT INTO guild_space (guild_id, space_id) VALUES (?, ?)").run(guild.id, roomID) return roomID } @@ -63,24 +56,19 @@ async function createSpace(guild, kstate) { * @param {number} privacyLevel */ async function guildToKState(guild, privacyLevel) { - assert.equal(typeof privacyLevel, "number") - const globalAdmins = select("member_power", ["mxid", "power_level"], {room_id: "*"}).all() - const additionalCreators = select("member_power", "mxid", {room_id: "*"}, "AND power_level > 100").pluck().all() + const avatarEventContent = {} + if (guild.icon) { + avatarEventContent.discord_path = file.guildIcon(guild) + avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API + } const guildKState = { - "m.room.create/": { - type: "m.space", - additional_creators: additionalCreators - }, "m.room.name/": {name: guild.name}, - "m.room.avatar/": { - $if: guild.icon, - url: {$url: file.guildIcon(guild)} - }, + "m.room.avatar/": avatarEventContent, "m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]}, "m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]}, "m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]}, - "m.room.power_levels/": {users: globalAdmins.reduce((a, c) => (a[c.mxid] = c.power_level, a), {})} // used in guild initial creation postApplyPowerLevels + "m.room.power_levels/": {users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})} } return guildKState @@ -102,9 +90,6 @@ async function _syncSpace(guild, shouldActuallySync) { const row = select("guild_space", ["space_id", "privacy_level"], {guild_id: guild.id}).get() if (!row) { - const autocreate = select("guild_active", "autocreate", {guild_id: guild.id}).pluck().get() - assert.equal(autocreate, 1, `refusing to implicitly create a space for guild ${guild.id}. set the guild_active data first before calling ensureSpace/syncSpace.`) - const creation = (async () => { const guildKState = await guildToKState(guild, createRoom.DEFAULT_PRIVACY_LEVEL) // New spaces will have to use the default privacy level; we obviously can't look up the existing entry const spaceID = await createSpace(guild, guildKState) @@ -124,13 +109,11 @@ async function _syncSpace(guild, shouldActuallySync) { console.log(`[space sync] to matrix: ${guild.name}`) const guildKState = await guildToKState(guild, privacy_level) // calling this in both branches because we don't want to calculate this if not syncing - ks.kstateStripConditionals(guildKState) // pre-upload icons before diffing - await ks.kstateUploadMxc(guildKState) // sync guild state to space - const spaceKState = await ks.roomToKState(spaceID) + const spaceKState = await createRoom.roomToKState(spaceID) const spaceDiff = ks.diffKState(spaceKState, guildKState) - await ks.applyKStateDiffToRoom(spaceID, spaceDiff) + await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff) // guild icon was changed, so room avatars need to be updated as well as the space ones // doing it this way rather than calling syncRoom for great efficiency gains @@ -139,10 +122,15 @@ async function _syncSpace(guild, shouldActuallySync) { // don't try to update rooms with custom avatars though const roomsWithCustomAvatars = select("channel_room", "room_id", {}, "WHERE custom_avatar IS NOT NULL").pluck().all() - for await (const room of api.generateFullHierarchy(spaceID)) { - if (room.avatar_url === newAvatarState.url) continue - if (roomsWithCustomAvatars.includes(room.room_id)) continue - await api.sendState(room.room_id, "m.room.avatar", "", newAvatarState) + const childRooms = ks.kstateToState(spaceKState).filter(({type, state_key, content}) => { + return type === "m.space.child" && "via" in content && !roomsWithCustomAvatars.includes(state_key) + }).map(({state_key}) => state_key) + + for (const roomID of childRooms) { + const avatarEventContent = await api.getStateEvent(roomID, "m.room.avatar", "") + if (avatarEventContent.url !== newAvatarState.url) { + await api.sendState(roomID, "m.room.avatar", "", newAvatarState) + } } } @@ -187,23 +175,23 @@ async function syncSpaceFully(guildID) { console.log(`[space sync] to matrix: ${guild.name}`) const guildKState = await guildToKState(guild, privacy_level) - ks.kstateStripConditionals(guildKState) // pre-upload icons before diffing - await ks.kstateUploadMxc(guildKState) // sync guild state to space - const spaceKState = await ks.roomToKState(spaceID) + const spaceKState = await createRoom.roomToKState(spaceID) const spaceDiff = ks.diffKState(spaceKState, guildKState) - await ks.applyKStateDiffToRoom(spaceID, spaceDiff) + await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff) - const childRooms = await api.getFullHierarchy(spaceID) + const childRooms = ks.kstateToState(spaceKState).filter(({type, content}) => { + return type === "m.space.child" && "via" in content + }).map(({state_key}) => state_key) - for (const {room_id} of childRooms) { - const channelID = select("channel_room", "channel_id", {room_id}).pluck().get() + for (const roomID of childRooms) { + const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() if (!channelID) continue if (discord.channels.has(channelID)) { await createRoom.syncRoom(channelID) } else { - await createRoom.unbridgeChannel({id: channelID}, guildID) + await createRoom.unbridgeDeletedChannel(channelID, guildID) } } @@ -229,24 +217,22 @@ async function syncSpaceExpressions(data, checkBeforeSync) { */ async function update(spaceID, key, eventKey, fn) { if (!(key in data) || !data[key].length) return - const guild = discord.guilds.get(data.guild_id) - assert(guild) - const content = await fn(data[key], guild) + const content = await fn(data[key]) if (checkBeforeSync) { let existing try { existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey) } catch (e) { // State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake. - existing = fn([], guild) + existing = fn([]) } - if (isDeepStrictEqual(existing, content)) return + if (deepEqual(existing, content, {strict: true})) return } - await api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content) + api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content) } - await update(spaceID, "emojis", "moe.cadence.ooye.pack.emojis", expression.emojisToState) - await update(spaceID, "stickers", "moe.cadence.ooye.pack.stickers", expression.stickersToState) + update(spaceID, "emojis", "moe.cadence.ooye.pack.emojis", expression.emojisToState) + update(spaceID, "stickers", "moe.cadence.ooye.pack.stickers", expression.stickersToState) } module.exports.createSpace = createSpace diff --git a/src/d2m/actions/delete-message.js b/d2m/actions/delete-message.js similarity index 54% rename from src/d2m/actions/delete-message.js rename to d2m/actions/delete-message.js index 39b9fc8..440e123 100644 --- a/src/d2m/actions/delete-message.js +++ b/d2m/actions/delete-message.js @@ -14,13 +14,12 @@ async function deleteMessage(data) { const row = select("channel_room", ["room_id", "speedbump_checked", "thread_parent"], {channel_id: data.channel_id}).get() if (!row) return - // Assume we can redact from tombstoned rooms. - const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") - .select("event_id", "room_id").where({message_id: data.id}).all() - db.prepare("DELETE FROM message_room WHERE message_id = ?").run(data.id) - for (const {event_id, room_id} of eventsToRedact) { + const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all() + db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id) + db.prepare("DELETE FROM event_message WHERE message_id = ?").run(data.id) + for (const eventID of eventsToRedact) { // Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs - await api.redactEvent(room_id, event_id) + await api.redactEvent(row.room_id, eventID) } await speedbump.updateCache(row.thread_parent || data.channel_id, row.speedbump_checked) @@ -30,17 +29,16 @@ async function deleteMessage(data) { * @param {import("discord-api-types/v10").GatewayMessageDeleteBulkDispatchData} data */ async function deleteMessageBulk(data) { - const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get() - if (!row) return + const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() + if (!roomID) return const sids = JSON.stringify(data.ids) - // Assume we can redact from tombstoned rooms. - const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") - .select("event_id", "room_id").and("WHERE message_id IN (SELECT value FROM json_each(?))").all(sids) - db.prepare("DELETE FROM message_room WHERE message_id IN (SELECT value FROM json_each(?))").run(sids) - for (const {event_id, room_id} of eventsToRedact) { + const eventsToRedact = from("event_message").pluck("event_id").and("WHERE message_id IN (SELECT value FROM json_each(?)").all(sids) + db.prepare("DELETE FROM message_channel WHERE message_id IN (SELECT value FROM json_each(?)").run(sids) + db.prepare("DELETE FROM event_message WHERE message_id IN (SELECT value FROM json_each(?)").run(sids) + for (const eventID of eventsToRedact) { // Awaiting will make it go slower, but since this could be a long-running operation either way, we want to leave rate limit capacity for other operations - await api.redactEvent(room_id, event_id) + await api.redactEvent(roomID, eventID) } } diff --git a/src/d2m/actions/edit-message.js b/d2m/actions/edit-message.js similarity index 64% rename from src/d2m/actions/edit-message.js rename to d2m/actions/edit-message.js index f86a9c8..d52fcbd 100644 --- a/src/d2m/actions/edit-message.js +++ b/d2m/actions/edit-message.js @@ -3,15 +3,13 @@ const assert = require("assert").strict const passthrough = require("../../passthrough") -const {sync, db, select, from} = passthrough +const {sync, db, select} = passthrough /** @type {import("../converters/edit-to-changes")} */ const editToChanges = sync.require("../converters/edit-to-changes") /** @type {import("./register-pk-user")} */ const registerPkUser = sync.require("./register-pk-user") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") -/** @type {import("../../matrix/mreq")} */ -const mreq = sync.require("../../matrix/mreq") /** * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message @@ -19,18 +17,14 @@ const mreq = sync.require("../../matrix/mreq") * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel */ async function editMessage(message, guild, row) { - const historicalRoomOfMessage = from("message_room").join("historical_channel_room", "historical_room_index").where({message_id: message.id}).select("room_id").get() - const currentRoom = from("channel_room").join("historical_channel_room", "room_id").where({channel_id: message.channel_id}).select("room_id", "historical_room_index").get() - if (!currentRoom) return - - if (historicalRoomOfMessage && historicalRoomOfMessage.room_id !== currentRoom.room_id) return // tombstoned rooms should not have new events (including edits) sent to them - let {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api) if (row && row.speedbump_webhook_id === message.webhook_id) { // Handle the PluralKit public instance if (row.speedbump_id === "466378653216014359") { - senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, true) + const root = await registerPkUser.fetchMessage(message.id) + assert(root.member) + senderMxid = await registerPkUser.ensureSimJoined(root, roomID) } } @@ -59,7 +53,7 @@ async function editMessage(message, guild, row) { const sendNewEventParts = new Set() for (const promotion of promotions) { if ("eventID" in promotion) { - db.prepare(`UPDATE event_message SET ${promotion.column} = ? WHERE event_id = ?`).run(promotion.value ?? 0, promotion.eventID) + db.prepare(`UPDATE event_message SET ${promotion.column} = 0 WHERE event_id = ?`).run(promotion.eventID) } else if ("nextEvent" in promotion) { sendNewEventParts.add(promotion.column) } @@ -67,7 +61,7 @@ async function editMessage(message, guild, row) { // 4. Send all the things. if (eventsToSend.length) { - db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(message.id, currentRoom.historical_room_index) + db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id) } for (const content of eventsToSend) { const eventType = content.$type @@ -78,17 +72,8 @@ async function editMessage(message, guild, row) { const part = sendNewEventParts.has("part") && eventsToSend[0] === content ? 0 : 1 const reactionPart = sendNewEventParts.has("reaction_part") && eventsToSend[eventsToSend.length - 1] === content ? 0 : 1 - - try { - const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord - } catch (e) { - if (e instanceof mreq.MatrixServerError && e.errcode === "M_FORBIDDEN") { - // sending user doesn't have permission to update message, e.g. because Discord generated an embed in a read-only room - } else { - throw e - } - } + const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid) + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord } } diff --git a/src/d2m/actions/expression.js b/d2m/actions/expression.js similarity index 78% rename from src/d2m/actions/expression.js rename to d2m/actions/expression.js index c7ab27a..b7b5d5a 100644 --- a/src/d2m/actions/expression.js +++ b/d2m/actions/expression.js @@ -9,12 +9,11 @@ const file = sync.require("../../matrix/file") /** * @param {DiscordTypes.APIEmoji[]} emojis - * @param {DiscordTypes.APIGuild} guild */ -async function emojisToState(emojis, guild) { +async function emojisToState(emojis) { const result = { pack: { - display_name: `${guild.name} (Discord Emojis)`, + display_name: "Discord Emojis", usage: ["emoticon"] // we'll see... }, images: { @@ -25,13 +24,13 @@ async function emojisToState(emojis, guild) { file.uploadDiscordFileToMxc(file.emoji(emoji.id, emoji.animated)).then(url => { result.images[emoji.name] = { info: { - mimetype: "image/webp" + mimetype: emoji.animated ? "image/gif" : "image/png" }, url } db.prepare("INSERT OR IGNORE INTO emoji (emoji_id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(emoji.id, emoji.name, +!!emoji.animated, url) }).catch(e => { - if (e.data?.errcode === "M_TOO_LARGE") { // Very unlikely to happen. Only possible for 3x-series emojis uploaded shortly after animated emojis were introduced, when there was no 256 KB size limit. + if (e.data.errcode === "M_TOO_LARGE") { // Very unlikely to happen. Only possible for 3x-series emojis uploaded shortly after animated emojis were introduced, when there was no 256 KB size limit. return } console.error(`Trying to handle emoji ${emoji.name} (${emoji.id}), but...`) @@ -43,12 +42,11 @@ async function emojisToState(emojis, guild) { /** * @param {DiscordTypes.APISticker[]} stickers - * @param {DiscordTypes.APIGuild} guild */ -async function stickersToState(stickers, guild) { +async function stickersToState(stickers) { const result = { pack: { - display_name: `${guild.name} (Discord Stickers)`, + display_name: "Discord Stickers", usage: ["sticker"] // we'll see... }, images: { @@ -68,7 +66,7 @@ async function stickersToState(stickers, guild) { while (shortcodes.includes(shortcode)) shortcode = shortcode + "~" shortcodes.push(shortcode) - result.images[shortcode] = { + result.images[shortcodes] = { info: { mimetype: file.stickerFormat.get(sticker.format_type)?.mime || "image/png" }, diff --git a/src/d2m/actions/lottie.js b/d2m/actions/lottie.js similarity index 97% rename from src/d2m/actions/lottie.js rename to d2m/actions/lottie.js index 0185980..4635fed 100644 --- a/src/d2m/actions/lottie.js +++ b/d2m/actions/lottie.js @@ -33,7 +33,7 @@ async function convert(stickerItem) { if (res.status !== 200) throw new Error("Sticker data file not found.") const text = await res.text() - // Convert to PNG (stream.Readable) + // Convert to PNG (readable stream) const readablePng = await convertLottie.convert(text) // Upload to MXC diff --git a/src/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js similarity index 58% rename from src/d2m/actions/register-pk-user.js rename to d2m/actions/register-pk-user.js index 6ecd077..ca47b7c 100644 --- a/src/d2m/actions/register-pk-user.js +++ b/d2m/actions/register-pk-user.js @@ -1,11 +1,12 @@ // @ts-check const assert = require("assert") -const {reg} = require("../../matrix/read-registration") +const reg = require("../../matrix/read-registration") const Ty = require("../../types") +const fetch = require("node-fetch").default const passthrough = require("../../passthrough") -const {sync, db, select, from} = passthrough +const {discord, sync, db, select} = passthrough /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** @type {import("../../matrix/file")} */ @@ -13,20 +14,12 @@ const file = sync.require("../../matrix/file") /** @type {import("./register-user")} */ const registerUser = sync.require("./register-user") -/** @returns {Promise} */ -async function fetchMessage(messageID) { - try { - var res = await fetch(`https://api.pluralkit.me/v2/messages/${messageID}`) - } catch (networkError) { - // Network issue, raise a more readable message - throw new Error(`Failed to connect to PK API: ${networkError.toString()}`) - } - if (!res.ok) throw new Error(`PK API returned an error: ${await res.text()}`) - /** @type {any} */ - const root = await res.json() - if (!root.member) throw new Error(`PK API didn't return member data: ${JSON.stringify(root)}`) - return root -} +/** + * @typedef WebhookAuthor Discord API message->author. A webhook as an author. + * @prop {string} username + * @prop {string?} avatar + * @prop {string} id + */ /** * A sim is an account that is being simulated by the bridge to copy events from the other side. @@ -40,7 +33,7 @@ async function createSim(pkMessage) { const mxid = `@${localpart}:${reg.ooye.server_name}` // Save chosen name in the database forever - db.prepare("INSERT INTO sim (user_id, username, sim_name, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, simName, mxid) + db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, localpart, mxid) // Register matrix user with that name try { @@ -87,17 +80,24 @@ async function ensureSimJoined(pkMessage, roomID) { // Ensure joined const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() if (!existing) { - await api.inviteToRoom(roomID, mxid) - await api.joinRoom(roomID, mxid) + try { + await api.inviteToRoom(roomID, mxid) + await api.joinRoom(roomID, mxid) + } catch (e) { + if (e.message.includes("is already in the room.")) { + // Sweet! + } else { + throw e + } + } db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid) } return mxid } /** - * Generate profile data based on webhook displayname and configured avatar. * @param {Ty.PkMessage} pkMessage - * @param {Ty.WebhookAuthor} author + * @param {WebhookAuthor} author */ async function memberToStateContent(pkMessage, author) { // We prefer to use the member's avatar URL data since the image upload can be cached across channels, @@ -116,40 +116,38 @@ async function memberToStateContent(pkMessage, author) { /** * Sync profile data for a sim user. This function follows the following process: - * 1. Look up data about proxy user from API - * 2. If this fails, try to use previously cached data (won't sync) - * 3. Create and join the sim to the room if needed - * 4. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before - * 5. Compare against the previously known state content, which is helpfully stored in the database - * 6. If the state content has changed, send it to Matrix and update it in the database for next time - * @param {string} messageID to call API with - * @param {Ty.WebhookAuthor} author for profile data - * @param {string} roomID room to join member to - * @param {boolean} shouldActuallySync whether to actually sync updated user data or just ensure it's joined + * 1. Join the sim to the room if needed + * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before + * 3. Compare against the previously known state content, which is helpfully stored in the database + * 4. If the state content has changed, send it to Matrix and update it in the database for next time + * @param {WebhookAuthor} author + * @param {Ty.PkMessage} pkMessage + * @param {string} roomID * @returns {Promise} mxid of the updated sim */ -async function syncUser(messageID, author, roomID, shouldActuallySync) { - try { - // API lookup - var pkMessage = await fetchMessage(messageID) - db.prepare("REPLACE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username) - } catch (e) { - // Fall back to offline cache - const senderMxid = from("sim_proxy").join("sim", "user_id").join("sim_member", "mxid").where({displayname: author.username, room_id: roomID}).pluck("mxid").get() - if (!senderMxid) throw e - return senderMxid - } - - // Create and join the sim to the room if needed +async function syncUser(author, pkMessage, roomID) { const mxid = await ensureSimJoined(pkMessage, roomID) - - if (shouldActuallySync) { - // Build current profile data and sync if the hash has changed - const content = await memberToStateContent(pkMessage, author) - await registerUser._sendSyncUser(roomID, mxid, content, null) + // Update the sim_proxy table, so mentions can look up the original sender later + db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username) + // Sync the member state + const content = await memberToStateContent(pkMessage, author) + const currentHash = registerUser._hashProfileContent(content) + const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() + // only do the actual sync if the hash has changed since we last looked + if (existingHash !== currentHash) { + await api.sendState(roomID, "m.room.member", mxid, content, mxid) + db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) } - return mxid } +/** @returns {Promise} */ +function fetchMessage(messageID) { + return fetch(`https://api.pluralkit.me/v2/messages/${messageID}`).then(res => res.json()) +} + +module.exports._memberToStateContent = memberToStateContent +module.exports.ensureSim = ensureSim +module.exports.ensureSimJoined = ensureSimJoined module.exports.syncUser = syncUser +module.exports.fetchMessage = fetchMessage diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js new file mode 100644 index 0000000..8244fe2 --- /dev/null +++ b/d2m/actions/register-user.js @@ -0,0 +1,186 @@ +// @ts-check + +const assert = require("assert") +const reg = require("../../matrix/read-registration") + +const passthrough = require("../../passthrough") +const {discord, sync, db, select} = passthrough +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/file")} */ +const file = sync.require("../../matrix/file") +/** @type {import("../converters/user-to-mxid")} */ +const userToMxid = sync.require("../converters/user-to-mxid") +/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore +let hasher = null +// @ts-ignore +require("xxhash-wasm")().then(h => hasher = h) + +/** + * A sim is an account that is being simulated by the bridge to copy events from the other side. + * @param {import("discord-api-types/v10").APIUser} user + * @returns mxid + */ +async function createSim(user) { + // Choose sim name + const simName = userToMxid.userToSimName(user) + const localpart = reg.ooye.namespace_prefix + simName + const mxid = `@${localpart}:${reg.ooye.server_name}` + + // Save chosen name in the database forever + // Making this database change right away so that in a concurrent registration, the 2nd registration will already have generated a different localpart because it can see this row when it generates + db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(user.id, simName, localpart, mxid) + + // Register matrix user with that name + try { + await api.register(localpart) + } catch (e) { + // If user creation fails, manually undo the database change. Still isn't perfect, but should help. + // (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.) + db.prepare("DELETE FROM sim WHERE user_id = ?").run(user.id) + throw e + } + return mxid +} + +/** + * Ensure a sim is registered for the user. + * If there is already a sim, use that one. If there isn't one yet, register a new sim. + * @param {import("discord-api-types/v10").APIUser} user + * @returns {Promise} mxid + */ +async function ensureSim(user) { + let mxid = null + const existing = select("sim", "mxid", {user_id: user.id}).pluck().get() + if (existing) { + mxid = existing + } else { + mxid = await createSim(user) + } + return mxid +} + +/** + * Ensure a sim is registered for the user and is joined to the room. + * @param {import("discord-api-types/v10").APIUser} user + * @param {string} roomID + * @returns {Promise} mxid + */ +async function ensureSimJoined(user, roomID) { + // Ensure room ID is really an ID, not an alias + assert.ok(roomID[0] === "!") + + // Ensure user + const mxid = await ensureSim(user) + + // Ensure joined + const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() + if (!existing) { + try { + await api.inviteToRoom(roomID, mxid) + await api.joinRoom(roomID, mxid) + } catch (e) { + if (e.message.includes("is already in the room.")) { + // Sweet! + } else { + throw e + } + } + db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid) + } + return mxid +} + +/** + * @param {import("discord-api-types/v10").APIUser} user + * @param {Omit} member + */ +async function memberToStateContent(user, member, guildID) { + let displayname = user.username + if (user.global_name) displayname = user.global_name + if (member.nick) displayname = member.nick + + const content = { + displayname, + membership: "join", + "moe.cadence.ooye.member": { + }, + "uk.half-shot.discord.member": { + bot: !!user.bot, + displayColor: user.accent_color, + id: user.id, + username: user.discriminator.length === 4 ? `${user.username}#${user.discriminator}` : `@${user.username}` + } + } + + if (member.avatar || user.avatar) { + // const avatarPath = file.userAvatar(user) // the user avatar only + const avatarPath = file.memberAvatar(guildID, user, member) // the member avatar or the user avatar + content["moe.cadence.ooye.member"].avatar = avatarPath + content.avatar_url = await file.uploadDiscordFileToMxc(avatarPath) + } + + return content +} + +function _hashProfileContent(content) { + const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`) + const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range + return signedHash +} + +/** + * Sync profile data for a sim user. This function follows the following process: + * 1. Join the sim to the room if needed + * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before + * 3. Compare against the previously known state content, which is helpfully stored in the database + * 4. If the state content has changed, send it to Matrix and update it in the database for next time + * @param {import("discord-api-types/v10").APIUser} user + * @param {Omit} member + * @returns {Promise} mxid of the updated sim + */ +async function syncUser(user, member, guildID, roomID) { + const mxid = await ensureSimJoined(user, roomID) + const content = await memberToStateContent(user, member, guildID) + const currentHash = _hashProfileContent(content) + const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() + // only do the actual sync if the hash has changed since we last looked + if (existingHash !== currentHash) { + await api.sendState(roomID, "m.room.member", mxid, content, mxid) + db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) + } + return mxid +} + +async function syncAllUsersInRoom(roomID) { + const mxids = select("sim_member", "mxid", {room_id: roomID}).pluck().all() + + const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() + assert.ok(typeof channelID === "string") + + /** @ts-ignore @type {import("discord-api-types/v10").APIGuildChannel} */ + const channel = discord.channels.get(channelID) + const guildID = channel.guild_id + assert.ok(typeof guildID === "string") + + for (const mxid of mxids) { + const userID = select("sim", "user_id", {mxid}).pluck().get() + assert.ok(typeof userID === "string") + + /** @ts-ignore @type {Required} */ + const member = await discord.snow.guild.getGuildMember(guildID, userID) + /** @ts-ignore @type {Required} user */ + const user = member.user + assert.ok(user) + + console.log(`[user sync] to matrix: ${user.username} in ${channel.name}`) + await syncUser(user, member, guildID, roomID) + } +} + +module.exports._memberToStateContent = memberToStateContent +module.exports._hashProfileContent = _hashProfileContent +module.exports.ensureSim = ensureSim +module.exports.ensureSimJoined = ensureSimJoined +module.exports.syncUser = syncUser +module.exports.syncAllUsersInRoom = syncAllUsersInRoom diff --git a/d2m/actions/register-user.test.js b/d2m/actions/register-user.test.js new file mode 100644 index 0000000..96c73aa --- /dev/null +++ b/d2m/actions/register-user.test.js @@ -0,0 +1,63 @@ +const {_memberToStateContent} = require("./register-user") +const {test} = require("supertape") +const testData = require("../../test/data") + +test("member2state: without member nick or avatar", async t => { + t.deepEqual( + await _memberToStateContent(testData.member.kumaccino.user, testData.member.kumaccino, testData.guild.general.id), + { + avatar_url: "mxc://cadence.moe/UpAeIqeclhKfeiZNdIWNcXXL", + displayname: "kumaccino", + membership: "join", + "moe.cadence.ooye.member": { + avatar: "/avatars/113340068197859328/b48302623a12bc7c59a71328f72ccb39.png?size=1024" + }, + "uk.half-shot.discord.member": { + bot: false, + displayColor: 10206929, + id: "113340068197859328", + username: "@kumaccino" + } + } + ) +}) + +test("member2state: with global name, without member nick or avatar", async t => { + t.deepEqual( + await _memberToStateContent(testData.member.papiophidian.user, testData.member.papiophidian, testData.guild.general.id), + { + avatar_url: "mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX", + displayname: "PapiOphidian", + membership: "join", + "moe.cadence.ooye.member": { + avatar: "/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024" + }, + "uk.half-shot.discord.member": { + bot: false, + displayColor: 1579292, + id: "320067006521147393", + username: "@papiophidian" + } + } + ) +}) + +test("member2state: with member nick and avatar", async t => { + t.deepEqual( + await _memberToStateContent(testData.member.sheep.user, testData.member.sheep, testData.guild.general.id), + { + avatar_url: "mxc://cadence.moe/rfemHmAtcprjLEiPiEuzPhpl", + displayname: "The Expert's Submarine", + membership: "join", + "moe.cadence.ooye.member": { + avatar: "/guilds/112760669178241024/users/134826546694193153/avatars/38dd359aa12bcd52dd3164126c587f8c.png?size=1024" + }, + "uk.half-shot.discord.member": { + bot: false, + displayColor: null, + id: "134826546694193153", + username: "@aprilsong" + } + } + ) +}) diff --git a/src/d2m/actions/remove-reaction.js b/d2m/actions/remove-reaction.js similarity index 70% rename from src/d2m/actions/remove-reaction.js rename to d2m/actions/remove-reaction.js index af7fd6a..95fc0aa 100644 --- a/src/d2m/actions/remove-reaction.js +++ b/d2m/actions/remove-reaction.js @@ -4,7 +4,7 @@ const Ty = require("../../types") const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") -const {discord, sync, db, from, select} = passthrough +const {discord, sync, db, select} = passthrough /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** @type {import("../converters/emoji-to-key")} */ @@ -18,15 +18,21 @@ const converter = sync.require("../converters/remove-reaction") * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData | DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData | DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data */ async function removeSomeReactions(data) { - const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get() - if (!row) return + const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() + if (!roomID) return + const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get() + if (!eventIDForMessage) return - const eventReactedTo = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") - .where({message_id: data.message_id}).and("ORDER BY reaction_part").select("event_id", "room_id").get() - if (!eventReactedTo) return - - // Due to server restrictions, all relations (i.e. reactions) have to be in the same room as the original event. - const reactions = await api.getFullRelations(eventReactedTo.room_id, eventReactedTo.event_id, "m.annotation") + /** @type {Ty.Event.Outer[]} */ + let reactions = [] + /** @type {string | undefined} */ + let nextBatch = undefined + do { + /** @type {Ty.Pagination>} */ + const res = await api.getRelations(roomID, eventIDForMessage, {from: nextBatch}, "m.annotation") + reactions = reactions.concat(res.chunk) + nextBatch = res.next_batch + } while (nextBatch) // Run the proper strategy and any strategy-specific database changes const removals = await @@ -36,7 +42,7 @@ async function removeSomeReactions(data) { // Redact the events and delete individual stored events in the database for (const removal of removals) { - await api.redactEvent(eventReactedTo.room_id, removal.eventID, removal.mxid) + await api.redactEvent(roomID, removal.eventID, removal.mxid) if (removal.hash) db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(removal.hash) } } @@ -46,7 +52,7 @@ async function removeSomeReactions(data) { * @param {Ty.Event.Outer[]} reactions */ async function removeReaction(data, reactions) { - const key = await emojiToKey.emojiToKey(data.emoji, data.message_id) + const key = await emojiToKey.emojiToKey(data.emoji) return converter.removeReaction(data, reactions, key) } @@ -55,8 +61,8 @@ async function removeReaction(data, reactions) { * @param {Ty.Event.Outer[]} reactions */ async function removeEmojiReaction(data, reactions) { - const key = await emojiToKey.emojiToKey(data.emoji, data.message_id) - const discordPreferredEncoding = await emoji.encodeEmoji(key, undefined) + const key = await emojiToKey.emojiToKey(data.emoji) + const discordPreferredEncoding = emoji.encodeEmoji(key, undefined) db.prepare("DELETE FROM reaction WHERE message_id = ? AND encoded_emoji = ?").run(data.message_id, discordPreferredEncoding) return converter.removeEmojiReaction(data, reactions, key) diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js new file mode 100644 index 0000000..8c26f07 --- /dev/null +++ b/d2m/actions/send-message.js @@ -0,0 +1,83 @@ +// @ts-check + +const assert = require("assert") + +const passthrough = require("../../passthrough") +const { discord, sync, db } = passthrough +/** @type {import("../converters/message-to-event")} */ +const messageToEvent = sync.require("../converters/message-to-event") +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("./register-user")} */ +const registerUser = sync.require("./register-user") +/** @type {import("./register-pk-user")} */ +const registerPkUser = sync.require("./register-pk-user") +/** @type {import("../actions/create-room")} */ +const createRoom = sync.require("../actions/create-room") +/** @type {import("../../discord/utils")} */ +const dUtils = sync.require("../../discord/utils") + +/** + * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message + * @param {import("discord-api-types/v10").APIGuild} guild + * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel + */ +async function sendMessage(message, guild, row) { + const roomID = await createRoom.ensureRoom(message.channel_id) + + let senderMxid = null + if (!dUtils.isWebhookMessage(message)) { + if (message.member) { // available on a gateway message create event + senderMxid = await registerUser.syncUser(message.author, message.member, message.guild_id, roomID) + } else { // well, good enough... + senderMxid = await registerUser.ensureSimJoined(message.author, roomID) + } + } else if (row && row.speedbump_webhook_id === message.webhook_id) { + // Handle the PluralKit public instance + if (row.speedbump_id === "466378653216014359") { + const root = await registerPkUser.fetchMessage(message.id) + // Member is null if member was deleted. We just got this message, so member surely exists. + if (!root.member) { + const e = new Error("PK API did not return a member") + message["__pk_response__"] = root + console.error(root) + throw e + } + senderMxid = await registerPkUser.syncUser(message.author, root, roomID) + } + } + + const events = await messageToEvent.messageToEvent(message, guild, {}, {api}) + const eventIDs = [] + if (events.length) { + db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id) + if (senderMxid) api.sendTyping(roomID, false, senderMxid) + } + for (const event of events) { + const part = event === events[0] ? 0 : 1 + const reactionPart = event === events[events.length - 1] ? 0 : 1 + + const eventType = event.$type + if ("$sender" in event) senderMxid = event.$sender + /** @type {Pick> & { $type?: string, $sender?: string }} */ + const eventWithoutType = {...event} + delete eventWithoutType.$type + delete eventWithoutType.$sender + + const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined + const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp) + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, part, reactionPart) // source 1 = discord + + // The primary event is part = 0 and has the most important and distinct information. It is used to provide reply previews, be pinned, and possibly future uses. + // The first event is chosen to be the primary part because it is usually the message text content and is more likely to be distinct. + // For example, "Reply to 'this meme made me think of you'" is more useful than "Replied to image". + + // The last event gets reaction_part = 0. Reactions are managed there because reactions are supposed to appear at the bottom. + + eventIDs.push(eventID) + } + + return eventIDs +} + +module.exports.sendMessage = sendMessage diff --git a/src/d2m/actions/speedbump.js b/d2m/actions/speedbump.js similarity index 76% rename from src/d2m/actions/speedbump.js rename to d2m/actions/speedbump.js index 218f046..7c3109b 100644 --- a/src/d2m/actions/speedbump.js +++ b/d2m/actions/speedbump.js @@ -4,14 +4,6 @@ const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") const {discord, select, db} = passthrough -const DEBUG_SPEEDBUMP = false - -function debugSpeedbump(message) { - if (DEBUG_SPEEDBUMP) { - console.log(message) - } -} - const SPEEDBUMP_SPEED = 4000 // 4 seconds delay const SPEEDBUMP_UPDATE_FREQUENCY = 2 * 60 * 60 // 2 hours @@ -35,8 +27,8 @@ async function updateCache(channelID, lastChecked) { db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_webhook_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(foundApplication, foundWebhook, now, channelID) } -/** @type {Map} messageID -> number of gateway events currently bumping */ -const bumping = new Map() +/** @type {Set} set of messageID */ +const bumping = new Set() /** * Slow down a message. After it passes the speedbump, return whether it's okay or if it's been deleted. @@ -44,26 +36,9 @@ const bumping = new Map() * @returns whether it was deleted */ async function doSpeedbump(messageID) { - let value = (bumping.get(messageID) ?? 0) + 1 - bumping.set(messageID, value) - debugSpeedbump(`[speedbump] WAIT ${messageID}++ = ${value}`) - + bumping.add(messageID) await new Promise(resolve => setTimeout(resolve, SPEEDBUMP_SPEED)) - - if (!bumping.has(messageID)) { - debugSpeedbump(`[speedbump] DELETED ${messageID}`) - return true - } - value = (bumping.get(messageID) ?? 0) - 1 - if (value <= 0) { - debugSpeedbump(`[speedbump] OK ${messageID}-- = ${value}`) - bumping.delete(messageID) - return false - } else { - debugSpeedbump(`[speedbump] MULTI ${messageID}-- = ${value}`) - bumping.set(messageID, value) - return true - } + return !bumping.delete(messageID) } /** diff --git a/src/d2m/actions/update-pins.js b/d2m/actions/update-pins.js similarity index 59% rename from src/d2m/actions/update-pins.js rename to d2m/actions/update-pins.js index 56c9642..5d98501 100644 --- a/src/d2m/actions/update-pins.js +++ b/d2m/actions/update-pins.js @@ -6,8 +6,6 @@ const {discord, sync, db} = passthrough const pinsToList = sync.require("../converters/pins-to-list") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") -/** @type {import("../../matrix/kstate")} */ -const ks = sync.require("../../matrix/kstate") /** * @template {string | null | undefined} T @@ -25,21 +23,13 @@ function convertTimestamp(timestamp) { * @param {number?} convertedTimestamp */ async function updatePins(channelID, roomID, convertedTimestamp) { - try { - var discordPins = await discord.snow.channel.getChannelPinnedMessages(channelID) - } catch (e) { - if (e.message === `{"message": "Missing Access", "code": 50001}`) { - return // Discord sends channel pins update events even for channels that the bot can't view/get pins in, just ignore it - } - throw e + const pins = await discord.snow.channel.getChannelPinnedMessages(channelID) + const eventIDs = pinsToList.pinsToList(pins) + if (pins.length === eventIDs.length || eventIDs.length) { + await api.sendState(roomID, "m.room.pinned_events", "", { + pinned: eventIDs + }) } - - const kstate = await ks.roomToKState(roomID, [["m.room.pinned_events", ""]]) - const pinned = pinsToList.pinsToList(discordPins, kstate) - - const diff = ks.diffKState(kstate, {"m.room.pinned_events/": {pinned}}) - await ks.applyKStateDiffToRoom(roomID, diff) - db.prepare("UPDATE channel_room SET last_bridged_pin_timestamp = ? WHERE channel_id = ?").run(convertedTimestamp || 0, channelID) } diff --git a/d2m/converters/edit-to-changes.js b/d2m/converters/edit-to-changes.js new file mode 100644 index 0000000..08f787c --- /dev/null +++ b/d2m/converters/edit-to-changes.js @@ -0,0 +1,157 @@ +// @ts-check + +const assert = require("assert").strict + +const passthrough = require("../../passthrough") +const {discord, sync, db, select, from} = passthrough +/** @type {import("./message-to-event")} */ +const messageToEvent = sync.require("../converters/message-to-event") +/** @type {import("../actions/register-user")} */ +const registerUser = sync.require("../actions/register-user") +/** @type {import("../actions/create-room")} */ +const createRoom = sync.require("../actions/create-room") + +/** + * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message + * IMPORTANT: This may not have all the normal fields! The API documentation doesn't provide possible types, just says it's all optional! + * Since I don't have a spec, I will have to capture some real traffic and add it as test cases... I hope they don't change anything later... + * @param {import("discord-api-types/v10").APIGuild} guild + * @param {import("../../matrix/api")} api simple-as-nails dependency injection for the matrix API + */ +async function editToChanges(message, guild, api) { + // Figure out what events we will be replacing + + const roomID = select("channel_room", "room_id", {channel_id: message.channel_id}).pluck().get() + assert(roomID) + /** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */ + const senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null + + const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all() + + // Figure out what we will be replacing them with + + const newFallbackContent = await messageToEvent.messageToEvent(message, guild, {includeEditFallbackStar: true}, {api}) + const newInnerContent = await messageToEvent.messageToEvent(message, guild, {includeReplyFallback: false}, {api}) + assert.ok(newFallbackContent.length === newInnerContent.length) + + // Match the new events to the old events + + /* + Rules: + + The events must have the same type. + + The events must have the same subtype. + Events will therefore be divided into four categories: + */ + /** 1. Events that are matched, and should be edited by sending another m.replace event */ + let eventsToReplace = [] + /** 2. Events that are present in the old version only, and should be blanked or redacted */ + let eventsToRedact = [] + /** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */ + let eventsToSend = [] + // 4. Events that are matched and have definitely not changed, so they don't need to be edited or replaced at all. This is represented as nothing. + + function shift() { + newFallbackContent.shift() + newInnerContent.shift() + } + + // For each old event... + outer: while (newFallbackContent.length) { + const newe = newFallbackContent[0] + // Find a new event to pair it with... + for (let i = 0; i < oldEventRows.length; i++) { + const olde = oldEventRows[i] + if (olde.event_type === newe.$type && olde.event_subtype === (newe.msgtype || null)) { // The spec does allow subtypes to change, so I can change this condition later if I want to + // Found one! + // Set up the pairing + eventsToReplace.push({ + old: olde, + newFallbackContent: newFallbackContent[0], + newInnerContent: newInnerContent[0] + }) + // These events have been handled now, so remove them from the source arrays + shift() + oldEventRows.splice(i, 1) + // Go all the way back to the start of the next iteration of the outer loop + continue outer + } + } + // If we got this far, we could not pair it to an existing event, so it'll have to be a new one + eventsToSend.push(newInnerContent[0]) + shift() + } + // Anything remaining in oldEventRows is present in the old version only and should be redacted. + eventsToRedact = oldEventRows + + // We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times. + /** @type {({column: string, eventID: string} | {column: string, nextEvent: true})[]} */ + const promotions = [] + for (const column of ["part", "reaction_part"]) { + // If no events with part = 0 exist (or will exist), we need to do some management. + if (!eventsToReplace.some(e => e.old[column] === 0)) { + if (eventsToReplace.length) { + // We can choose an existing event to promote. Bigger order is better. + const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text") + eventsToReplace.sort((a, b) => order(b) - order(a)) + if (column === "part") { + promotions.push({column, eventID: eventsToReplace[0].old.event_id}) // part should be the first one + } else { + promotions.push({column, eventID: eventsToReplace[eventsToReplace.length - 1].old.event_id}) // reaction_part should be the last one + } + } else { + // No existing events to promote, but new events are being sent. Whatever gets sent will be the next part = 0. + promotions.push({column, nextEvent: true}) + } + } + } + + // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! + // (Example: a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.) + // So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed. + eventsToReplace = eventsToReplace.filter(ev => { + // Discord does not allow files, images, attachments, or videos to be edited. + if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") { + return false + } + // Discord does not allow stickers to be edited. + if (ev.old.event_type === "m.sticker") { + return false + } + // Anything else is fair game. + return true + }) + + // Removing unnecessary properties before returning + eventsToRedact = eventsToRedact.map(e => e.event_id) + eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)})) + + return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions} +} + +/** + * @template T + * @param {string} oldID + * @param {T} newFallbackContent + * @param {T} newInnerContent + * @returns {import("../../types").Event.ReplacementContent} content + */ +function makeReplacementEventContent(oldID, newFallbackContent, newInnerContent) { + const content = { + ...newFallbackContent, + "m.mentions": {}, + "m.new_content": { + ...newInnerContent + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: oldID + } + } + delete content["m.new_content"]["$type"] + // Client-Server API spec 11.37.3: Any m.relates_to property within m.new_content is ignored. + delete content["m.new_content"]["m.relates_to"] + return content +} + +module.exports.editToChanges = editToChanges +module.exports.makeReplacementEventContent = makeReplacementEventContent diff --git a/d2m/converters/edit-to-changes.test.js b/d2m/converters/edit-to-changes.test.js new file mode 100644 index 0000000..04f5568 --- /dev/null +++ b/d2m/converters/edit-to-changes.test.js @@ -0,0 +1,237 @@ +const {test} = require("supertape") +const {editToChanges} = require("./edit-to-changes") +const data = require("../../test/data") +const Ty = require("../../types") + +test("edit2changes: edit by webhook", async t => { + const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edit_by_webhook, data.guild.general, {}) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(eventsToReplace, [{ + oldID: "$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10", + newContent: { + $type: "m.room.message", + msgtype: "m.text", + body: "* test 2", + "m.mentions": {}, + "m.new_content": { + // *** Replaced With: *** + msgtype: "m.text", + body: "test 2", + "m.mentions": {} + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10" + } + } + }]) + t.equal(senderMxid, null) + t.deepEqual(promotions, []) +}) + +test("edit2changes: bot response", async t => { + const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.bot_response, data.guild.general, { + async getJoinedMembers(roomID) { + t.equal(roomID, "!hYnGGlPHlbujVVfktC:cadence.moe") + return new Promise(resolve => { + setTimeout(() => { + resolve({ + joined: { + "@cadence:cadence.moe": { + displayname: "cadence [they]", + avatar_url: "whatever" + }, + "@_ooye_botrac4r:cadence.moe": { + displayname: "botrac4r", + avatar_url: "whatever" + } + } + }) + }) + }) + } + }) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(eventsToReplace, [{ + oldID: "$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY", + newContent: { + $type: "m.room.message", + msgtype: "m.text", + body: "* :ae_botrac4r: @cadence asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.", + format: "org.matrix.custom.html", + formatted_body: '* :ae_botrac4r: @cadence asked ­, I respond: Stop drinking paint. (No)

Hit :bn_re: to reroll.', + "m.mentions": { + // Client-Server API spec 11.37.7: Copy Discord's behaviour by not re-notifying anyone that an *edit occurred* + }, + // *** Replaced With: *** + "m.new_content": { + msgtype: "m.text", + body: ":ae_botrac4r: @cadence asked ``­``, I respond: Stop drinking paint. (No)\n\nHit :bn_re: to reroll.", + format: "org.matrix.custom.html", + formatted_body: ':ae_botrac4r: @cadence asked ­, I respond: Stop drinking paint. (No)

Hit :bn_re: to reroll.', + "m.mentions": { + // Client-Server API spec 11.37.7: This should contain the mentions for the final version of the event + "user_ids": ["@cadence:cadence.moe"] + } + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY" + } + } + }]) + t.equal(senderMxid, "@_ooye_bojack_horseman:cadence.moe") + t.deepEqual(promotions, []) +}) + +test("edit2changes: remove caption from image", async t => { + const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.removed_caption_from_image, data.guild.general, {}) + t.deepEqual(eventsToRedact, ["$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA"]) + t.deepEqual(eventsToSend, []) + t.deepEqual(eventsToReplace, []) + t.deepEqual(promotions, [{column: "part", eventID: "$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI"}]) +}) + +test("edit2changes: change file type", async t => { + const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.changed_file_type, data.guild.general, {}) + t.deepEqual(eventsToRedact, ["$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ"]) + t.deepEqual(eventsToSend, [{ + $type: "m.room.message", + body: "📝 Uploaded file: https://cdn.discordapp.com/attachments/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt (20 MB)", + format: "org.matrix.custom.html", + formatted_body: "📝 Uploaded file: gaze_into_my_dark_mind.txt (20 MB)", + "m.mentions": {}, + msgtype: "m.text" + }]) + t.deepEqual(eventsToReplace, []) + t.deepEqual(promotions, [{column: "part", nextEvent: true}, {column: "reaction_part", nextEvent: true}]) +}) + +test("edit2changes: add caption back to that image", async t => { + const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.added_caption_to_image, data.guild.general, {}) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToSend, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "some text", + "m.mentions": {} + }]) + t.deepEqual(eventsToReplace, []) + t.deepEqual(promotions, []) +}) + +test("edit2changes: stickers and attachments are not changed, only the content can be edited", async t => { + const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments, data.guild.general, {}) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(eventsToReplace, [{ + oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4", + newContent: { + $type: "m.room.message", + msgtype: "m.text", + body: "* only the content can be edited", + "m.mentions": {}, + // *** Replaced With: *** + "m.new_content": { + msgtype: "m.text", + body: "only the content can be edited", + "m.mentions": {} + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4" + } + } + }]) +}) + +test("edit2changes: edit of reply to skull webp attachment with content", async t => { + const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edit_of_reply_to_skull_webp_attachment_with_content, data.guild.general, {}) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(eventsToReplace, [{ + oldID: "$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M", + newContent: { + $type: "m.room.message", + msgtype: "m.text", + body: "> Extremity: Image\n\n* Edit", + format: "org.matrix.custom.html", + formatted_body: + '
In reply to Extremity' + + '
Image
' + + '* Edit', + "m.mentions": {}, + "m.new_content": { + msgtype: "m.text", + body: "Edit", + "m.mentions": {} + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M" + } + } + }]) +}) + +test("edit2changes: edits the text event when multiple rows have part = 0 (should never happen in real life, but make sure the safety net works)", async t => { + const {eventsToRedact, eventsToReplace, eventsToSend} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_0, data.guild.general, {}) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(eventsToReplace, [{ + oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999", + newContent: { + $type: "m.room.message", + msgtype: "m.text", + body: "* only the content can be edited", + "m.mentions": {}, + // *** Replaced With: *** + "m.new_content": { + msgtype: "m.text", + body: "only the content can be edited", + "m.mentions": {} + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd999" + } + } + }]) +}) + +test("edit2changes: promotes the text event when multiple rows have part = 1 (should never happen in real life, but make sure the safety net works)", async t => { + const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edited_content_with_sticker_and_attachments_but_all_parts_equal_1, data.guild.general, {}) + t.deepEqual(eventsToRedact, []) + t.deepEqual(eventsToSend, []) + t.deepEqual(eventsToReplace, [{ + oldID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111", + newContent: { + $type: "m.room.message", + msgtype: "m.text", + body: "* only the content can be edited", + "m.mentions": {}, + // *** Replaced With: *** + "m.new_content": { + msgtype: "m.text", + body: "only the content can be edited", + "m.mentions": {} + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111" + } + } + }]) + t.deepEqual(promotions, [ + { + column: "part", + eventID: "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qd111" + }, + { + column: "reaction_part", + eventID: "$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd111" + } + ]) +}) diff --git a/src/d2m/converters/emoji-to-key.js b/d2m/converters/emoji-to-key.js similarity index 71% rename from src/d2m/converters/emoji-to-key.js rename to d2m/converters/emoji-to-key.js index 54bda18..267664c 100644 --- a/src/d2m/converters/emoji-to-key.js +++ b/d2m/converters/emoji-to-key.js @@ -8,10 +8,9 @@ const file = sync.require("../../matrix/file") /** * @param {import("discord-api-types/v10").APIEmoji} emoji - * @param {string} message_id * @returns {Promise} */ -async function emojiToKey(emoji, message_id) { +async function emojiToKey(emoji) { let key if (emoji.id) { // Custom emoji @@ -31,10 +30,7 @@ async function emojiToKey(emoji, message_id) { // Default emoji const name = emoji.name assert(name) - // If the reaction was used on Matrix already, it might be using a different arrangement of Variation Selector 16 characters. - // We'll use the same arrangement that was originally used, otherwise a duplicate of the emoji will appear as a separate reaction. - const originalEncoding = select("reaction", "original_encoding", {message_id, encoded_emoji: encodeURIComponent(name)}).pluck().get() - key = originalEncoding || name + key = name } return key } diff --git a/src/d2m/converters/emoji-to-key.test.js b/d2m/converters/emoji-to-key.test.js similarity index 94% rename from src/d2m/converters/emoji-to-key.test.js rename to d2m/converters/emoji-to-key.test.js index 544eada..5af046c 100644 --- a/src/d2m/converters/emoji-to-key.test.js +++ b/d2m/converters/emoji-to-key.test.js @@ -2,7 +2,7 @@ const {test} = require("supertape") const {emojiToKey} = require("./emoji-to-key") -const data = require("../../../test/data") +const data = require("../../test/data") const Ty = require("../../types") test("emoji2key: unicode emoji works", async t => { diff --git a/src/d2m/converters/lottie.js b/d2m/converters/lottie.js similarity index 93% rename from src/d2m/converters/lottie.js rename to d2m/converters/lottie.js index 969d345..6d13f70 100644 --- a/src/d2m/converters/lottie.js +++ b/d2m/converters/lottie.js @@ -2,7 +2,7 @@ const assert = require("assert") const stream = require("stream") -const {PNG} = require("@cloudrac3r/pngjs") +const {PNG} = require("pngjs") const SIZE = 160 // Discord's display size on 1x displays is 160 @@ -21,7 +21,7 @@ const Rlottie = (async () => { /** * @param {string} text - * @returns {Promise} + * @returns {Promise} */ async function convert(text) { const r = await Rlottie @@ -41,7 +41,6 @@ async function convert(text) { png.data = Buffer.from(rendered) // png.pack() is a bad stream and will throw away any data it sends if it's not connected to a destination straight away. // We use Duplex.from to convert it into a good stream. - // @ts-ignore return stream.Duplex.from(png.pack()) } diff --git a/src/d2m/converters/lottie.test.js b/d2m/converters/lottie.test.js similarity index 100% rename from src/d2m/converters/lottie.test.js rename to d2m/converters/lottie.test.js diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js new file mode 100644 index 0000000..dfc5679 --- /dev/null +++ b/d2m/converters/message-to-event.embeds.test.js @@ -0,0 +1,143 @@ +const {test} = require("supertape") +const {messageToEvent} = require("./message-to-event") +const data = require("../../test/data") +const Ty = require("../../types") + +test("message2event embeds: nothing but a field", async t => { + const events = await messageToEvent(data.message_with_embeds.nothing_but_a_field, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.notice", + body: "| ### Amanda 🎵#2192 :online:" + + "\n| willow tree, branch 0" + + "\n| **❯ Uptime:**\n| 3m 55s\n| **❯ Memory:**\n| 64.45MB", + format: "org.matrix.custom.html", + formatted_body: '

Amanda 🎵#2192 \":online:\"' + + '
willow tree, branch 0' + + '
❯ Uptime:
3m 55s' + + '
❯ Memory:
64.45MB

' + }]) +}) + +test("message2event embeds: reply with just an embed", async t => { + const events = await messageToEvent(data.message_with_embeds.reply_with_only_embed, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.notice", + "m.mentions": {}, + body: "| ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145" + + "\n| \n| ## https://twitter.com/i/status/1707484191963648161" + + "\n| \n| does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?" + + "\n| \n| ### Retweets" + + "\n| 119" + + "\n| \n| ### Likes" + + "\n| 5581" + + "\n| — Twitter", + format: "org.matrix.custom.html", + formatted_body: '

⏺️ dynastic (@dynastic)

' + + '

https://twitter.com/i/status/1707484191963648161' + + '

does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?' + + '

Retweets
119

Likes
5581

— Twitter
' + }]) +}) + +test("message2event embeds: image embed and attachment", async t => { + const events = await messageToEvent(data.message_with_embeds.image_embed_and_attachment, data.guild.general, {}, { + api: { + async getJoinedMembers(roomID) { + return {joined: []} + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "https://tootsuite.net/Warp-Gate2.gif\ntanget: @ monster spawner", + format: "org.matrix.custom.html", + formatted_body: 'https://tootsuite.net/Warp-Gate2.gif
tanget: @ monster spawner', + "m.mentions": {} + }, { + $type: "m.room.message", + msgtype: "m.image", + url: "mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR", + body: "Screenshot_20231001_034036.jpg", + external_url: "https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg?ex=651a1faa&is=6518ce2a&hm=eb5ca80a3fa7add8765bf404aea2028a28a2341e4a62435986bcdcf058da82f3&", + filename: "Screenshot_20231001_034036.jpg", + info: { + h: 1170, + w: 1080, + size: 51981, + mimetype: "image/jpeg" + }, + "m.mentions": {} + }]) +}) + +test("message2event embeds: blockquote in embed", async t => { + let called = 0 + const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general, {}, { + api: { + async getStateEvent(roomID, type, key) { + called++ + t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return { + users: { + "@_ooye_bot:cadence.moe": 100 + } + } + }, + async getJoinedMembers(roomID) { + called++ + t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe") + return { + joined: { + "@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null}, + "@user:example.invalid": {display_name: null, avatar_url: null} + } + } + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: ":emoji: **4 |** #wonderland", + format: "org.matrix.custom.html", + formatted_body: `\":emoji:\" 4 | #wonderland`, + "m.mentions": {} + }, { + $type: "m.room.message", + msgtype: "m.notice", + body: "| ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n| \n| reply draft\n| > The following is a message composed via consensus of the Stinker Council.\n| > \n| > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n| > \n| > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n| > \n| > There will be no further communication.\n| \n| [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)", + format: "org.matrix.custom.html", + formatted_body: "

⏺️ minimus

reply draft

The following is a message composed via consensus of the Stinker Council.

For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.

Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.

There will be no further communication.

Go to Message

", + "m.mentions": {} + }]) + t.equal(called, 2, "should call getStateEvent and getJoinedMembers once each") +}) + +test("message2event embeds: crazy html is all escaped", async t => { + const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.notice", + body: "| ## ⏺️ [Hey