From ff7af39802e34ac3972396cda67d0442dabf00c3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 25 Mar 2024 18:05:19 +1300 Subject: [PATCH 1/3] Exclude generated embeds for discord.com --- .../message-to-event.embeds.test.js | 34 ++++++++++++++++ d2m/converters/message-to-event.js | 4 ++ test/data.js | 40 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js index fc02a3f..61a0822 100644 --- a/d2m/converters/message-to-event.embeds.test.js +++ b/d2m/converters/message-to-event.embeds.test.js @@ -282,3 +282,37 @@ test("message2event embeds: youtube video", async t => { "m.mentions": {} }]) }) + +test("message2event embeds: if discord creates an embed preview for a discord channel link, don't copy that embed", async t => { + const events = await messageToEvent(data.message_with_embeds.discord_server_included_punctuation_bad_discord, data.guild.general, {}, { + api: { + async getStateEvent(roomID, type, key) { + t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return { + users: { + "@_ooye_bot:cadence.moe": 100 + } + } + }, + async getJoinedMembers(roomID) { + t.equal(roomID, "!TqlyQmifxGUggEmdBN:cadence.moe") + return { + joined: { + "@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null}, + "@user:matrix.org": {display_name: null, avatar_url: null} + } + } + } + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "(test https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&via=matrix.org)", + format: "org.matrix.custom.html", + formatted_body: `(test https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU?via=cadence.moe&via=matrix.org)`, + "m.mentions": {} + }]) +}) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 518d041..45e43dd 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -516,6 +516,10 @@ async function messageToEvent(message, guild, options = {}, di) { continue // Matrix's own URL previews are fine for images. } + if (embed.url?.startsWith("https://discord.com/")) { + continue // If discord creates an embed preview for a discord channel link, don't copy that embed + } + // Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once const rep = new mxUtils.MatrixStringBuilder() diff --git a/test/data.js b/test/data.js index 17b92cf..77b2ada 100644 --- a/test/data.js +++ b/test/data.js @@ -2870,6 +2870,46 @@ module.exports = { } }, webhook_id: "1109360903096369153" + }, + discord_server_included_punctuation_bad_discord: { + id: "1221672425792606349", + type: 0, + content: "(test https://discord.com/channels/1160894080998461480/1160894080998461480/1202543413652881428)", + channel_id: "1160894080998461480", + author: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "4b5c4b28051144e4c111f0113a0f1cf1", + discriminator: "0", + public_flags: 0, + premium_type: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [ + { + type: "article", + url: "https://discord.com/channels/1160894080998461480/1160894080998461480/1202543413652881428)", + title: "Discord - A New Way to Chat with Friends & Communities", + description: "Discord is the easiest way to communicate over voice, video, and text. Chat, hang out, and stay close with your friends and communities.", + provider: { name: "Discord" }, + content_scan_version: 0 + } + ], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2024-03-25T04:10:03.885000+00:00", + edited_timestamp: null, + flags: 0, + components: [] } }, message_update: { From 642be26313bd69fbb563a3fbe2dabb4679e62f79 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 26 Mar 2024 01:10:29 +1300 Subject: [PATCH 2/3] Enumerate child rooms with hierarchy endpoint --- d2m/actions/create-space.js | 14 +++++++++++--- matrix/api.js | 14 ++++++++++++++ types.d.ts | 17 +++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index a7390ed..3bad2a1 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -3,6 +3,7 @@ const assert = require("assert").strict const {isDeepStrictEqual} = require("util") const DiscordTypes = require("discord-api-types/v10") +const Ty = require("../../types") const reg = require("../../matrix/read-registration") const passthrough = require("../../passthrough") @@ -181,9 +182,16 @@ async function syncSpaceFully(guildID) { const spaceDiff = ks.diffKState(spaceKState, guildKState) await createRoom.applyKStateDiffToRoom(spaceID, spaceDiff) - const childRooms = ks.kstateToState(spaceKState).filter(({type, content}) => { - return type === "m.space.child" && "via" in content - }).map(({state_key}) => state_key) + /** @type {string[]} room IDs */ + let childRooms = [] + /** @type {string | undefined} */ + let nextBatch = undefined + do { + /** @type {Ty.HierarchyPagination} */ + const res = await api.getHierarchy(spaceID, {from: nextBatch}) + childRooms.push(...res.rooms.map(room => room.room_id)) + nextBatch = res.next_batch + } while (nextBatch) for (const roomID of childRooms) { const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() diff --git a/matrix/api.js b/matrix/api.js index baa5d96..82b1c10 100644 --- a/matrix/api.js +++ b/matrix/api.js @@ -121,6 +121,19 @@ function getJoinedMembers(roomID) { return mreq.mreq("GET", `/client/v3/rooms/${roomID}/joined_members`) } +/** + * @param {string} roomID + * @param {{from?: string, limit?: any}} pagination + * @returns {Promise>} + */ +function getHierarchy(roomID, pagination) { + let path = `/client/v1/rooms/${roomID}/hierarchy` + if (!pagination.from) delete pagination.from + if (!pagination.limit) pagination.limit = 50 + path += `?${new URLSearchParams(pagination)}` + return mreq.mreq("GET", path) +} + /** * @param {string} roomID * @param {string} eventID @@ -239,6 +252,7 @@ module.exports.getEventForTimestamp = getEventForTimestamp module.exports.getAllState = getAllState module.exports.getStateEvent = getStateEvent module.exports.getJoinedMembers = getJoinedMembers +module.exports.getHierarchy = getHierarchy module.exports.getRelations = getRelations module.exports.sendState = sendState module.exports.sendEvent = sendEvent diff --git a/types.d.ts b/types.d.ts index 2788f60..68430d9 100644 --- a/types.d.ts +++ b/types.d.ts @@ -257,6 +257,18 @@ export namespace R { export type EventRedacted = { event_id: string } + + export type Hierarchy = { + avatar_url?: string + canonical_alias?: string + children_state: {} + guest_can_join: boolean + join_rule?: string + name?: string + num_joined_members: number + room_id: string + room_type?: string + } } export type Pagination = { @@ -264,3 +276,8 @@ export type Pagination = { next_batch?: string prev_match?: string } + +export type HierarchyPagination = { + rooms: T[] + next_batch?: string +} From 5f0e765934bbd1dd3e5765e19d9624f0356ae9e6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 26 Mar 2024 01:11:13 +1300 Subject: [PATCH 3/3] Bridge forums as spaces --- d2m/actions/create-room.js | 40 +++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js index a8d2846..4559ffa 100644 --- a/d2m/actions/create-room.js +++ b/d2m/actions/create-room.js @@ -59,13 +59,16 @@ function applyKStateDiffToRoom(roomID, kstate) { } /** - * @param {{id: string, name: string, topic?: string?, type: number}} channel + * @param {{id: string, name: string, topic?: string?, type: number, parent_id?: string?}} channel * @param {{id: string}} guild * @param {string | null | undefined} customName */ function convertNameAndTopic(channel, guild, customName) { + // @ts-ignore + const parentChannel = discord.channels.get(channel.parent_id) let channelPrefix = - ( channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] " + ( parentChannel?.type === DiscordTypes.ChannelType.GuildForum ? "" + : channel.type === DiscordTypes.ChannelType.PublicThread ? "[⛓️] " : channel.type === DiscordTypes.ChannelType.PrivateThread ? "[🔒⛓️] " : channel.type === DiscordTypes.ChannelType.GuildVoice ? "[🔊] " : "") @@ -88,9 +91,24 @@ function convertNameAndTopic(channel, guild, customName) { * @param {DiscordTypes.APIGuild} guild */ async function channelToKState(channel, guild) { - const spaceID = await createSpace.ensureSpace(guild) - assert(typeof spaceID === "string") - const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get() + // @ts-ignore + const parentChannel = discord.channels.get(channel.parent_id) + /** Used for membership/permission checks. */ + let guildSpaceID + /** Used as the literal parent on Matrix, for categorisation. Will be the same as `guildSpaceID` unless it's a forum channel's thread, in which case a different space is used to group those threads. */ + let parentSpaceID + let privacyLevel + if (parentChannel?.type === DiscordTypes.ChannelType.GuildForum) { // it's a forum channel's thread, so use a different space to group those threads + guildSpaceID = await createSpace.ensureSpace(guild) + parentSpaceID = await ensureRoom(channel.parent_id) + privacyLevel = select("guild_space", "privacy_level", {space_id: guildSpaceID}).pluck().get() + } else { // otherwise use the guild's space like usual + parentSpaceID = await createSpace.ensureSpace(guild) + guildSpaceID = parentSpaceID + privacyLevel = select("guild_space", "privacy_level", {space_id: parentSpaceID}).pluck().get() + } + assert(typeof parentSpaceID === "string") + assert(typeof guildSpaceID === "string") assert(typeof privacyLevel === "number") const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get() @@ -114,7 +132,7 @@ async function channelToKState(channel, guild) { join_rule: "restricted", allow: [{ type: "m.room_membership", - room_id: spaceID + room_id: guildSpaceID }] } if (PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel] !== "restricted") { @@ -130,7 +148,7 @@ async function channelToKState(channel, guild) { "m.room.avatar/": avatarEventContent, "m.room.guest_access/": {guest_access: PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]}, "m.room.history_visibility/": {history_visibility}, - [`m.space.parent/${spaceID}`]: { + [`m.space.parent/${parentSpaceID}`]: { via: [reg.ooye.server_name], canonical: true }, @@ -167,7 +185,7 @@ async function channelToKState(channel, guild) { } } - return {spaceID, privacyLevel, channelKState} + return {spaceID: parentSpaceID, privacyLevel, channelKState} } /** @@ -183,6 +201,9 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { let threadParent = null if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id + let spaceCreationContent = {} + if (channel.type === DiscordTypes.ChannelType.GuildForum) spaceCreationContent = {creation_content: {type: "m.space"}} + // Name and topic can be done earlier in room creation rather than in initial_state // https://spec.matrix.org/latest/client-server-api/#creation const name = kstate["m.room.name/"].name @@ -199,7 +220,8 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) { preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel], invite: [], - initial_state: ks.kstateToState(kstate) + initial_state: ks.kstateToState(kstate), + ...spaceCreationContent }) db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent)