diff --git a/docs/developer-orientation.md b/docs/developer-orientation.md index bff64c2..0f5e0c0 100644 --- a/docs/developer-orientation.md +++ b/docs/developer-orientation.md @@ -113,6 +113,7 @@ Total transitive production dependencies: 144 * (0) ansi-colors: Helps with interactive prompting for the initial setup, and it's already pulled in by enquirer. * (1) chunk-text: It does what I want. * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust. +* (0) denque: Much faster than using arrays as a queue. * (0) discord-api-types: Bitfields needed at runtime and types needed for development. * (0) domino: DOM implementation that's already pulled in by turndown. * (2) enquirer: Interactive prompting for the initial setup rather than forcing users to edit YAML non-interactively. diff --git a/package-lock.json b/package-lock.json index eb07b4d..04dbbe4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "out-of-your-element", - "version": "3.5.1", + "version": "3.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "out-of-your-element", - "version": "3.5.1", + "version": "3.6.0", "license": "AGPL-3.0-or-later", "dependencies": { "@chriscdn/promise-semaphore": "^3.0.1", @@ -18,13 +18,14 @@ "@cloudrac3r/pngjs": "^7.0.3", "@cloudrac3r/pug": "^4.0.4", "@cloudrac3r/stream-type": "^1.0.0", - "@cloudrac3r/turndown": "^7.1.4", + "@cloudrac3r/turndown": "^7.1.5", "@stackoverflow/stacks": "^2.5.4", "@stackoverflow/stacks-icons": "^6.0.2", "ansi-colors": "^4.1.3", "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.17.1", + "cloudstorm": "^0.19.0", + "denque": "^2.1.0", "discord-api-types": "^0.38.38", "domino": "^2.1.6", "enquirer": "^2.4.1", @@ -37,7 +38,7 @@ "mime-types": "^2.1.35", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", - "snowtransfer": "^0.17.5", + "snowtransfer": "^0.19.0", "try-to-catch": "^4.0.5", "uqr": "^0.1.2", "xxhash-wasm": "^1.0.2", @@ -267,9 +268,9 @@ } }, "node_modules/@cloudrac3r/turndown": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/@cloudrac3r/turndown/-/turndown-7.1.4.tgz", - "integrity": "sha512-bQAwcvcSqBTdEHPMt+IAZWIoDh+2eRuy9TgD0FUdxVurbvj3CUHTxLfzlmsO0UTi+GHpgYqDSsVdV7kYTNq5Qg==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/@cloudrac3r/turndown/-/turndown-7.1.5.tgz", + "integrity": "sha512-pENoG62B4UM2ecsg57cuSRDa1OV/YFYTFGg1aoQsKUdHBBglhE9I7EwEMYpS96QoM/auyglfaxjgTeqhcSGCIA==", "license": "MIT", "dependencies": { "domino": "^2.1.6" @@ -1316,13 +1317,13 @@ } }, "node_modules/cloudstorm": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.17.1.tgz", - "integrity": "sha512-LYUwzHagRYRd93XocOqi+HCHdzPYI9cW7Yf7pYqinxgG+Qka1OiqBKWTCcLiEuiqXaOV30kr8c6aZ/c1QcDP4Q==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.19.0.tgz", + "integrity": "sha512-mjuP5o2nS/CgSljajh08KjN6jYMfNBBttSPHc1YUN74f7OkmhUhv8QE8JFDXomxCfdnLNJcqBQtjRLrs8nkfVQ==", "license": "MIT", "dependencies": { - "discord-api-types": "^0.38.47", - "snowtransfer": "^0.17.5" + "discord-api-types": "^0.38.49", + "snowtransfer": "^0.19.0" }, "engines": { "node": ">=22.0.0" @@ -1457,6 +1458,15 @@ "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", "license": "MIT" }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/destr": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz", @@ -1473,9 +1483,9 @@ } }, "node_modules/discord-api-types": { - "version": "0.38.47", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.47.tgz", - "integrity": "sha512-XgXQodHQBAE6kfD7kMvVo30863iHX1LHSqNq6MGUTDwIFCCvHva13+rwxyxVXDqudyApMNAd32PGjgVETi5rjA==", + "version": "0.38.49", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.49.tgz", + "integrity": "sha512-XnqcWmnFZFAE8ZM8SHAw9DIV8D3Or00rMQ8iQLotrEA2PmXhl+ykaf6L6q4l474hrSUH1JaYcv+iOMRWp2p6Tg==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" @@ -2579,12 +2589,12 @@ } }, "node_modules/snowtransfer": { - "version": "0.17.7", - "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.17.7.tgz", - "integrity": "sha512-scbOjYezo1Ycfk21atCEkeXIISTT7R7JTHCdiZ/7m7k4XbSb6o5q8Mu2fev5IqFpNyqIVjA0d/MZQ+eP/gtwfg==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.19.0.tgz", + "integrity": "sha512-Vebj4FCdpUsEKAzqC1p50ol5o09CLnrffvBCpTv0FNFefvXPXwogxR8kr4zJApIRt7oWQVZZVkxAq4XnGlICjQ==", "license": "MIT", "dependencies": { - "discord-api-types": "^0.38.47" + "discord-api-types": "^0.38.49" }, "engines": { "node": ">=22.0.0" diff --git a/package.json b/package.json index 9dfd2a8..c086517 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "out-of-your-element", - "version": "3.5.1", + "version": "3.6.0", "description": "A bridge between Matrix and Discord", "main": "index.js", "repository": { @@ -27,13 +27,14 @@ "@cloudrac3r/pngjs": "^7.0.3", "@cloudrac3r/pug": "^4.0.4", "@cloudrac3r/stream-type": "^1.0.0", - "@cloudrac3r/turndown": "^7.1.4", + "@cloudrac3r/turndown": "^7.1.5", "@stackoverflow/stacks": "^2.5.4", "@stackoverflow/stacks-icons": "^6.0.2", "ansi-colors": "^4.1.3", "better-sqlite3": "^12.2.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.17.1", + "cloudstorm": "^0.19.0", + "denque": "^2.1.0", "discord-api-types": "^0.38.38", "domino": "^2.1.6", "enquirer": "^2.4.1", @@ -46,7 +47,7 @@ "mime-types": "^2.1.35", "prettier-bytes": "^1.0.4", "sharp": "^0.34.5", - "snowtransfer": "^0.17.5", + "snowtransfer": "^0.19.0", "try-to-catch": "^4.0.5", "uqr": "^0.1.2", "xxhash-wasm": "^1.0.2", diff --git a/scripts/setup.js b/scripts/setup.js index 69b62a2..6ce4c9f 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -359,7 +359,7 @@ function defineEchoHandler() { console.log("โœ… Emojis are ready...") // set profile data on discord... - await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + avatarBuffer.toString("base64")}) + await discord.snow.user.editSelf({avatar: "data:image/png;base64," + avatarBuffer.toString("base64")}) console.log("โœ… Discord profile updated...") // set profile data on homeserver... diff --git a/src/d2m/actions/create-space.js b/src/d2m/actions/create-space.js index 7a751e2..403fdb2 100644 --- a/src/d2m/actions/create-space.js +++ b/src/d2m/actions/create-space.js @@ -8,6 +8,8 @@ const {reg} = require("../../matrix/read-registration") const passthrough = require("../../passthrough") const {discord, sync, db, select} = passthrough +/** @type {import("../../matrix/mreq")} */ +const mreq = sync.require("../../matrix/mreq") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** @type {import("../../matrix/file")} */ @@ -237,8 +239,12 @@ async function syncSpaceExpressions(data, checkBeforeSync) { try { existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey) } catch (e) { - // State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake. - existing = fn([], guild) + if (e instanceof mreq.MatrixServerError && e.httpStatus < 400) { + // State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake. + existing = fn([], guild) + } else { + throw e + } } if (isDeepStrictEqual(existing, content)) return } diff --git a/src/d2m/actions/retrigger.js b/src/d2m/actions/retrigger.js index 66ef19e..43f400d 100644 --- a/src/d2m/actions/retrigger.js +++ b/src/d2m/actions/retrigger.js @@ -2,7 +2,15 @@ const {EventEmitter} = require("events") const passthrough = require("../../passthrough") -const {select} = passthrough +const {select, sync, from} = passthrough +/** @type {import("../../matrix/utils")} */ +const utils = sync.require("../../matrix/utils") + +/* + Due to Eventual Consistency(TM) an update/delete may arrive before the original message arrives + (or before the it has finished being bridged to an event). + In this case, wait until the original message has finished bridging, then retrigger the passed function. +*/ const DEBUG_RETRIGGER = false @@ -12,81 +20,140 @@ function debugRetrigger(message) { } } -const paused = new Set() -const emitter = new EventEmitter() +const storage = new class { + /** @private @type {Set} */ + paused = new Set() + /** @private @type {Map any)[]>} id -> list of resolvers */ + resolves = new Map() + /** @private @type {Map>} id -> timer */ + timers = new Map() -/** - * Due to Eventual Consistency(TM) an update/delete may arrive before the original message arrives - * (or before the it has finished being bridged to an event). - * In this case, wait until the original message has finished bridging, then retrigger the passed function. - * @template {(...args: any[]) => any} T - * @param {string} inputID - * @param {T} fn - * @param {Parameters} rest - * @returns {boolean} false if the event was found and the function will be ignored, true if the event was not found and the function will be retriggered - */ -function eventNotFoundThenRetrigger(inputID, fn, ...rest) { - if (!paused.has(inputID)) { - if (inputID.match(/^[0-9]+$/)) { - const eventID = select("event_message", "event_id", {message_id: inputID}).pluck().get() - if (eventID) { - debugRetrigger(`[retrigger] OK mid <-> eid = ${inputID} <-> ${eventID}`) - return false // event was found so don't retrigger - } - } else if (inputID.match(/^\$/)) { - const messageID = select("event_message", "message_id", {event_id: inputID}).pluck().get() - if (messageID) { - debugRetrigger(`[retrigger] OK eid <-> mid = ${inputID} <-> ${messageID}`) - return false // message was found so don't retrigger - } + /** + * The purpose of storage is to store `resolve` and call it at a later time. + * @param {string} id + * @param {(found: Boolean) => any} resolve + */ + store(id, resolve) { + debugRetrigger(`[retrigger] STORE id = ${id}`) + this.resolves.set(id, (this.resolves.get(id) || []).concat(resolve)) // add to list in map value + if (!this.timers.has(id)) { + debugRetrigger(`[retrigger] SET TIMER id = ${id}`) + this.timers.set(id, setTimeout(() => this.resolve(id, false), 60 * 1000).unref()) // 1 minute } } + + /** @param {string} id */ + isNotPaused(id) { + return !storage.paused.has(id) + } - debugRetrigger(`[retrigger] WAIT id = ${inputID}`) - emitter.once(inputID, () => { - debugRetrigger(`[retrigger] TRIGGER id = ${inputID}`) - fn(...rest) - }) - // if the event never arrives, don't trigger the callback, just clean up - setTimeout(() => { - if (emitter.listeners(inputID).length) { - debugRetrigger(`[retrigger] EXPIRE id = ${inputID}`) + /** @param {string} id */ + pause(id) { + debugRetrigger(`[retrigger] PAUSE id = ${id}`) + this.paused.add(id) + } + + /** + * Go through `resolves` storage and resolve them all. (Also resets timer/paused.) + * @param {string} id + * @param {boolean} value + */ + resolve(id, value) { + if (this.paused.has(id)) { + debugRetrigger(`[retrigger] RESUME id = ${id}`) + this.paused.delete(id) } - emitter.removeAllListeners(inputID) - }, 60 * 1000) // 1 minute - return true // event was not found, then retrigger + + if (this.resolves.has(id)) { + debugRetrigger(`[retrigger] RESOLVE ${value} id = ${id}`) + const fns = this.resolves.get(id) || [] + this.resolves.delete(id) + for (const fn of fns) { + fn(value) + } + } + + if (this.timers.has(id)) { + clearTimeout(this.timers.get(id)) + this.timers.delete(id) + } + } +} + +/** + * @param {string} id + * @param {(found: Boolean) => any} resolve + * @param {boolean} existsInDatabase + */ +function waitFor(id, resolve, existsInDatabase) { + if (existsInDatabase && storage.isNotPaused(id)) { // if event already exists and isn't paused then resolve immediately + debugRetrigger(`[retrigger] EXISTS id = ${id}`) + return resolve(true) + } + + // doesn't exist. wait for it to exist. storage will resolve true if it exists or false if it timed out + return storage.store(id, resolve) +} + +const GET_EVENT_PREPARED = from("event_message").select("event_id").and("WHERE event_id = ?").prepare().raw() +/** + * @param {string} eventID + * @returns {Promise} if true then the message did not arrive + */ +function waitForEvent(eventID) { + const {promise, resolve} = Promise.withResolvers() + waitFor(eventID, resolve, !!GET_EVENT_PREPARED.get(eventID)) + return promise +} + +const GET_MESSAGE_PREPARED = from("event_message").select("message_id").and("WHERE message_id = ?").prepare().raw() +/** + * @param {string} messageID + * @returns {Promise} if true then the message did not arrive + */ +function waitForMessage(messageID) { + const {promise, resolve} = Promise.withResolvers() + waitFor(messageID, resolve, !!GET_MESSAGE_PREPARED.get(messageID)) + return promise +} + +const GET_REACTION_EVENT_PREPARED = from("reaction").select("hashed_event_id").and("WHERE hashed_event_id = ?").prepare().raw() +/** + * @param {string} eventID + * @returns {Promise} if true then the message did not arrive + */ +function waitForReactionEvent(eventID) { + const {promise, resolve} = Promise.withResolvers() + waitFor(eventID, resolve, !!GET_REACTION_EVENT_PREPARED.get(utils.getEventIDHash(eventID))) + return promise } /** * Anything calling retrigger during the callback will be paused and retriggered after the callback resolves. * @template T - * @param {string} messageID + * @param {string} id * @param {Promise} promise * @returns {Promise} */ -async function pauseChanges(messageID, promise) { +async function pauseChanges(id, promise) { try { - debugRetrigger(`[retrigger] PAUSE id = ${messageID}`) - paused.add(messageID) + storage.pause(id) return await promise } finally { - debugRetrigger(`[retrigger] RESUME id = ${messageID}`) - paused.delete(messageID) - messageFinishedBridging(messageID) + finishedBridging(id) } } /** * Triggers any pending operations that were waiting on the corresponding event ID. - * @param {string} messageID + * @param {string} id */ -function messageFinishedBridging(messageID) { - if (emitter.listeners(messageID).length) { - debugRetrigger(`[retrigger] EMIT id = ${messageID}`) - } - emitter.emit(messageID) +function finishedBridging(id) { + storage.resolve(id, true) } -module.exports.eventNotFoundThenRetrigger = eventNotFoundThenRetrigger -module.exports.messageFinishedBridging = messageFinishedBridging +module.exports.waitForMessage = waitForMessage +module.exports.waitForEvent = waitForEvent +module.exports.waitForReactionEvent = waitForReactionEvent module.exports.pauseChanges = pauseChanges +module.exports.finishedBridging = finishedBridging \ No newline at end of file diff --git a/src/d2m/actions/send-message.js b/src/d2m/actions/send-message.js index 8550d43..5b3b4f3 100644 --- a/src/d2m/actions/send-message.js +++ b/src/d2m/actions/send-message.js @@ -60,8 +60,7 @@ async function sendMessage(message, channel, guild, row) { const detailedResultsMessage = await pollEnd.endPoll(message) if (detailedResultsMessage) { const threadParent = select("channel_room", "thread_parent", {channel_id: message.channel_id}).pluck().get() - const channelID = threadParent ? threadParent : message.channel_id - const threadID = threadParent ? message.channel_id : undefined + const {channelID, threadID} = dUtils.swapThreadID(message.channel_id, threadParent) sentResultsMessage = await channelWebhook.sendMessageWithWebhook(channelID, detailedResultsMessage, threadID) } } diff --git a/src/d2m/actions/speedbump.js b/src/d2m/actions/speedbump.js index 218f046..42e3a35 100644 --- a/src/d2m/actions/speedbump.js +++ b/src/d2m/actions/speedbump.js @@ -1,6 +1,5 @@ // @ts-check -const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") const {discord, select, db} = passthrough @@ -70,12 +69,18 @@ async function doSpeedbump(messageID) { * Check whether to slow down a message, and do it. After it passes the speedbump, return whether it's okay or if it's been deleted. * @param {string} channelID * @param {string} messageID + * @param {string} [userID] if provided, only slow down the message when the user has used PK before * @returns whether it was deleted, and data about the channel's (not thread's) speedbump */ -async function maybeDoSpeedbump(channelID, messageID) { - let row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: channelID}).get() - if (row?.thread_parent) row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: row.thread_parent}).get() // webhooks belong to the channel, not the thread - if (!row?.speedbump_webhook_id) return {affected: false, row: null} // not affected, no speedbump +async function maybeDoSpeedbump(channelID, messageID, userID) { + let row = select("channel_room", ["room_id", "thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: channelID}).get() + if (row?.thread_parent) row = select("channel_room", ["room_id", "thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: row.thread_parent}).get() // webhooks belong to the channel, not the thread + if (!row?.speedbump_webhook_id) return {affected: false, row: null} // channel not affected, no speedbump + if (userID) { + if (row.speedbump_webhook_id === userID) return {affected: false, row} // shortcut + const userHasProxy = select("sim_proxy", "user_id", {proxy_owner_id: userID}).pluck().get() + if (!userHasProxy) return {affected: false, row} // user has not used PK before, no speedbump + } const affected = await doSpeedbump(messageID) return {affected, row} // maybe affected, and there is a speedbump } diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 7229d3d..aeda572 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -265,8 +265,9 @@ function getFormattedInteraction(interaction, isThinkingInteraction) { * @param {any} newEvents merge into events * @param {any} events will be modified * @param {boolean} forceSameMsgtype whether m.text may only be combined with m.text, etc + * @param {boolean} [forceMerge] if true, must merge event, will error if it had to append */ -function mergeTextEvents(newEvents, events, forceSameMsgtype) { +function mergeTextEvents(newEvents, events, forceSameMsgtype, forceMerge = false) { let prev = events.at(-1) for (const ne of newEvents) { const isAllText = prev?.body && prev?.formatted_body && ["m.text", "m.notice"].includes(ne.msgtype) && ["m.text", "m.notice"].includes(prev?.msgtype) @@ -278,6 +279,8 @@ function mergeTextEvents(newEvents, events, forceSameMsgtype) { rep.addLine(ne.body, ne.formatted_body) prev.body = rep.body prev.formatted_body = rep.formattedBody + } else if (forceMerge) { + throw new Error("Unable to merge events") } else { events.push(ne) } @@ -554,7 +557,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter. // So we scan the message ahead of time for all its emojis and ensure they are in the DB. - const emojiMatches = [...content.matchAll(/<(a?):([^:>]{1,64}):([0-9]+)>/g)] + const emojiMatches = [...content.matchAll(/<(a?):([^:>]+):([0-9]+)>/g)] await Promise.all(emojiMatches.map(match => { const id = match[3] const name = match[2] @@ -967,7 +970,8 @@ async function messageToEvent(message, guild, options = {}, di) { // May only be a section accessory or in an action row (up to 5) if (component.style === DiscordTypes.ButtonStyle.Link) { assert(component.label) // required for Discord to validate link buttons - stack.msb.add(`[${component.label} ${component.url}] `, tag`${component.label} `) + const link = await transformContentMessageLinks(component.url) + stack.msb.add(`[${component.label} ${link}] `, tag`${component.label} `) } } @@ -980,7 +984,19 @@ async function messageToEvent(message, guild, options = {}, di) { const {body, formatted_body} = stack.msb.get() if (body.trim().length) { - await addTextEvent(body, formatted_body, "m.text") + // Create new message if Components V2 (cannot have regular content) + if ((message.flags ?? 0) & DiscordTypes.MessageFlags.IsComponentsV2) { + await addTextEvent(body, formatted_body, "m.text") + } + // Add to existing message if legacy components https://docs.discord.com/developers/components/reference#legacy-message-component-behavior + else { + mergeTextEvents([{ + msgtype: "m.text", + body, + format: "org.matrix.custom.html", + formatted_body + }], events, false, true) + } } } diff --git a/src/d2m/converters/message-to-event.test.components.js b/src/d2m/converters/message-to-event.test.components.js index 137b63b..1ef83c3 100644 --- a/src/d2m/converters/message-to-event.test.components.js +++ b/src/d2m/converters/message-to-event.test.components.js @@ -1,6 +1,7 @@ const {test} = require("supertape") const {messageToEvent} = require("./message-to-event") const data = require("../../../test/data") +const {mockGetEffectivePower} = require("../../matrix/utils.test") test("message2event components: pk question mark output", async t => { const events = await messageToEvent(data.message_with_components.pk_question_mark_response, data.guild.general, {}) @@ -77,3 +78,24 @@ test("message2event components: pk question mark output", async t => { msgtype: "m.text", }]) }) + +test("message2event components: pk ping message legacy components", async t => { + const events = await messageToEvent(data.message_with_components.pk_ping_components_v1, data.guild.general, {}, { + api: { + async getJoinedMembers() { + return {joined: {}} + }, + getEffectivePower: mockGetEffectivePower() + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "โญ cadence used `/๐Ÿ”” Ping author`" + + "\nPsst, **Red** (@cadence.worm:), you have been pinged by @cadence.worm:." + + "\n[Jump https://matrix.to/#/!TqlyQmifxGUggEmdBN:cadence.moe/$l9FMmsEbh9K0NUReeEpWOMZYGRlUOE8yLcm6P-TYHSM?via=cadence.moe] ", + format: "org.matrix.custom.html", + formatted_body: "
โญ cadence used /๐Ÿ”” Ping author
Psst, Red (@cadence.worm), you have been pinged by @cadence.worm.
Jump ", + "m.mentions": {} + }]) +}) diff --git a/src/d2m/converters/remove-reaction.js b/src/d2m/converters/remove-reaction.js index 4ca22b6..b6b0407 100644 --- a/src/d2m/converters/remove-reaction.js +++ b/src/d2m/converters/remove-reaction.js @@ -34,7 +34,7 @@ function removeReaction(data, reactions, key) { // Even though the bridge bot only reacted once on Discord-side, multiple Matrix users may have // reacted on Matrix-side. Semantically, we want to remove the reaction from EVERY Matrix user. // Also need to clean up the database. - const hash = utils.getEventIDHash(event.event_id) + const hash = utils.getEventIDHash(eventID) removals.push({eventID, mxid: null, hash}) } if (!lookingAtMatrixReaction && !wantToRemoveMatrixReaction) { diff --git a/src/d2m/discord-client.js b/src/d2m/discord-client.js index c2a0549..b9b45e2 100644 --- a/src/d2m/discord-client.js +++ b/src/d2m/discord-client.js @@ -14,6 +14,8 @@ const {sync} = passthrough /** @type {import("./discord-packets")} */ const discordPackets = sync.require("./discord-packets") +const CONNECTION_DEBUG = false + class DiscordClient { /** * @param {string} discordToken @@ -24,7 +26,7 @@ class DiscordClient { const intents = [ "DIRECT_MESSAGES", "DIRECT_MESSAGE_REACTIONS", "DIRECT_MESSAGE_TYPING", "GUILDS", "GUILD_EMOJIS_AND_STICKERS", "GUILD_MESSAGES", "GUILD_MESSAGE_REACTIONS", "GUILD_MESSAGE_TYPING", "GUILD_WEBHOOKS", "GUILD_MESSAGE_POLLS", - "MESSAGE_CONTENT" + "MESSAGE_CONTENT", "GUILD_MEMBERS" ] if (reg.ooye.receive_presences !== false) intents.push("GUILD_PRESENCES") this.discordToken = discordToken @@ -59,15 +61,17 @@ class DiscordClient { }) } - const addEventLogger = (eventName, logName) => { - this.cloud.on(eventName, (...args) => { - const d = new Date().toISOString().slice(0, 19) - console.error(`[${d} Client ${logName}]`, ...args) - }) + if (CONNECTION_DEBUG) { + const addEventLogger = (eventName, logName) => { + this.cloud.on(eventName, (...args) => { + const d = new Date().toISOString().slice(0, 19) + console.error(`[${d} Client ${logName}]`, ...args) + }) + } + addEventLogger("error", "Error") + addEventLogger("disconnected", "Disconnected") + addEventLogger("ready", "Ready") } - addEventLogger("error", "Error") - addEventLogger("disconnected", "Disconnected") - addEventLogger("ready", "Ready") } } diff --git a/src/d2m/discord-packets.js b/src/d2m/discord-packets.js index afea9ea..b17ad4e 100644 --- a/src/d2m/discord-packets.js +++ b/src/d2m/discord-packets.js @@ -1,173 +1,222 @@ // @ts-check -// Discord library internals type beat - -const DiscordTypes = require("discord-api-types/v10") +const assert = require("assert") +const {scheduler} = require("timers/promises") const passthrough = require("../passthrough") -const {sync, db} = passthrough +const {sync} = passthrough -const utils = { - /** - * @param {import("./discord-client")} client - * @param {import("cloudstorm").IGatewayMessage} message - * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate - */ - async onPacket(client, message, listen) { - // requiring this later so that the client is already constructed by the time event-dispatcher is loaded - /** @type {typeof import("./event-dispatcher")} */ - const eventDispatcher = sync.require("./event-dispatcher") - /** @type {import("../discord/register-interactions")} */ - const interactions = sync.require("../discord/register-interactions") +/** @type {import("../matrix/homeserver-status")} */ +const homeserverStatus = sync.require("../matrix/homeserver-status") - // Client internals, keep track of the state we need - if (message.t === "READY") { - if (client.ready) return - client.ready = true - client.user = message.d.user - client.application = message.d.application - console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`) - interactions.registerInteractions() +let checkedHomeserver = false - } else if (message.t === "GUILD_CREATE") { - message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web - client.guilds.set(message.d.id, message.d) - const arr = [] - client.guildChannelMap.set(message.d.id, arr) - for (const channel of message.d.channels || []) { - // @ts-ignore - channel.guild_id = message.d.id - arr.push(channel.id) - client.channels.set(channel.id, channel) - } - for (const thread of message.d.threads || []) { - // @ts-ignore - thread.guild_id = message.d.id - arr.push(thread.id) - client.channels.set(thread.id, thread) - } +/** + * @param {import("./discord-client")} client + * @param {import("cloudstorm").IGatewayMessage} message + * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate + */ +async function onPacket(client, message, listen) { + // requiring this later so that the client is already constructed by the time event-dispatcher is loaded + /** @type {typeof import("./event-dispatcher")} */ + const eventDispatcher = sync.require("./event-dispatcher") + /** @type {import("../discord/register-interactions")} */ + const interactions = sync.require("../discord/register-interactions") - if (listen === "full") { - try { - await eventDispatcher.checkMissedExpressions(message.d) - await eventDispatcher.checkMissedMessages(client, message.d) - await eventDispatcher.checkMissedPins(client, message.d) - await eventDispatcher.checkMissedLeaves(client, message.d) - } catch (e) { - console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:") - console.error(e) + // Client internals, keep track of the state we need + if (message.t === "READY") { + if (client.ready) return + client.ready = true + client.user = message.d.user + client.application = message.d.application + console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`) + interactions.registerInteractions() + + } else if (message.t === "GUILD_CREATE") { + message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web + client.guilds.set(message.d.id, message.d) + const arr = [] + client.guildChannelMap.set(message.d.id, arr) + for (const channel of message.d.channels || []) { + // @ts-ignore + channel.guild_id = message.d.id + arr.push(channel.id) + client.channels.set(channel.id, channel) + } + for (const thread of message.d.threads || []) { + // @ts-ignore + thread.guild_id = message.d.id + arr.push(thread.id) + client.channels.set(thread.id, thread) + } + + if (listen === "full") { + try { + /* + Info about guilds is populated one guild at a time. + For m->d bridging to work, the guild needs to be populated, so we need to have GUILD_CREATE for the guild. + If we ping the homeserver, it will send us any pending events, so we need to wait for all GUILD_CREATES before we ping. + We must attempt a ping because we don't want to try sending missed d->m messages to an offline homeserver. + This delay can be removed if ONE of the following is done: + 1. m->d can queue incoming events until their guild exists in memory + 2. d->m missed messages can have their errors handled and added to queue, rather than pinging first + */ + let isMainCharacter = false + if (!checkedHomeserver) { + checkedHomeserver = true + isMainCharacter = true + console.log("Warming up guilds~") } - } - - } else if (message.t === "GUILD_UPDATE") { - const guild = client.guilds.get(message.d.id) - if (guild) { - for (const prop of Object.keys(message.d)) { - if (!["channels", "threads"].includes(prop)) { - guild[prop] = message.d[prop] - } - } - } - - } else if (message.t === "GUILD_EMOJIS_UPDATE") { - const guild = client.guilds.get(message.d.guild_id) - if (guild) { - guild.emojis = message.d.emojis - } - - } else if (message.t === "GUILD_STICKERS_UPDATE") { - const guild = client.guilds.get(message.d.guild_id) - if (guild) { - guild.stickers = message.d.stickers - } - - } else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") { - const guild = client.guilds.get(message.d.guild_id) - /** Delete this in case of UPDATE or DELETE */ - const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id - /** Add this in case of CREATE or UPDATE */ - const newRoles = [] - if ("role" in message.d) newRoles.push(message.d.role) - if (guild) { - const targetIndex = guild.roles.findIndex(r => r.id === targetID) - if (targetIndex !== -1) { - // Role already exists. Delete it and maybe replace it. - guild.roles.splice(targetIndex, 1, ...newRoles) + await scheduler.wait(5000) + if (isMainCharacter) { + checkedHomeserver = true + process.stdout.write("Connecting to homeserver... ") + await homeserverStatus.homeserverStatus.waitForOnline(true) + console.log("ok.\nReplaying past events. Welcome to Out Of Your Element.") } else { - // Role doesn't already exist. - guild.roles.push(...newRoles) + await homeserverStatus.homeserverStatus.waitForOnline(false) } + + await eventDispatcher.checkMissedExpressions(message.d) + await eventDispatcher.checkMissedMessages(client, message.d) + await eventDispatcher.checkMissedPins(client, message.d) + await eventDispatcher.checkMissedLeaves(client, message.d) + } catch (e) { + console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:") + console.error(e) } + } - } else if (message.t === "GUILD_MEMBER_UPDATE") { - const guild = client.guilds.get(message.d.guild_id) - const member = guild?.members.find(m => m.user.id === message.d.user.id) - if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones - Object.assign(member, message.d) - } - - } else if (message.t === "THREAD_CREATE") { - client.channels.set(message.d.id, message.d) - if (message.d["guild_id"]) { - const channels = client.guildChannelMap.get(message.d["guild_id"]) - if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) - } - - } else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") { - client.channels.set(message.d.id, message.d) - - - } else if (message.t === "CHANNEL_PINS_UPDATE") { - const channel = client.channels.get(message.d.channel_id) - if (channel) { - channel["last_pin_timestamp"] = message.d.last_pin_timestamp - } - - - } else if (message.t === "GUILD_DELETE") { - client.guilds.delete(message.d.id) - const channels = client.guildChannelMap.get(message.d.id) - if (channels) { - for (const id of channels) client.channels.delete(id) - } - client.guildChannelMap.delete(message.d.id) - - - } else if (message.t === "CHANNEL_CREATE") { - client.channels.set(message.d.id, message.d) - if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have - const channels = client.guildChannelMap.get(message.d["guild_id"]) - if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) - } - - } else if (message.t === "CHANNEL_DELETE") { - client.channels.delete(message.d.id) - if (message.d["guild_id"]) { - const channels = client.guildChannelMap.get(message.d["guild_id"]) - if (channels) { - const previous = channels.indexOf(message.d.id) - if (previous !== -1) channels.splice(previous, 1) + } else if (message.t === "GUILD_UPDATE") { + const guild = client.guilds.get(message.d.id) + if (guild) { + for (const prop of Object.keys(message.d)) { + if (!["channels", "threads"].includes(prop)) { + guild[prop] = message.d[prop] } } } - // Event dispatcher for OOYE bridge operations - if (listen === "full" && message.t) { - try { - if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") { - await eventDispatcher.onSomeReactionsRemoved(client, message.d) + } else if (message.t === "GUILD_EMOJIS_UPDATE") { + const guild = client.guilds.get(message.d.guild_id) + if (guild) { + guild.emojis = message.d.emojis + } - } else if (message.t === "INTERACTION_CREATE") { - await interactions.dispatchInteraction(message.d) + } else if (message.t === "GUILD_STICKERS_UPDATE") { + const guild = client.guilds.get(message.d.guild_id) + if (guild) { + guild.stickers = message.d.stickers + } - } else if (message.t in eventDispatcher) { - await eventDispatcher[message.t](client, message.d) - } - } catch (e) { - // Let OOYE try to handle errors too - await eventDispatcher.onError(client, e, message) + } else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") { + const guild = client.guilds.get(message.d.guild_id) + /** Delete this in case of UPDATE or DELETE */ + const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id + /** Add this in case of CREATE or UPDATE */ + const newRoles = [] + if ("role" in message.d) newRoles.push(message.d.role) + if (guild) { + const targetIndex = guild.roles.findIndex(r => r.id === targetID) + if (targetIndex !== -1) { + // Role already exists. Delete it and maybe replace it. + guild.roles.splice(targetIndex, 1, ...newRoles) + } else { + // Role doesn't already exist. + guild.roles.push(...newRoles) } } + + } else if (message.t === "GUILD_MEMBER_UPDATE") { + const guild = client.guilds.get(message.d.guild_id) + const member = guild?.members.find(m => m.user.id === message.d.user.id) + if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones + Object.assign(member, message.d) + } + + } else if (message.t === "THREAD_CREATE") { + client.channels.set(message.d.id, message.d) + if (message.d["guild_id"]) { + const channels = client.guildChannelMap.get(message.d["guild_id"]) + if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) + } + + } else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") { + client.channels.set(message.d.id, message.d) + + + } else if (message.t === "CHANNEL_PINS_UPDATE") { + const channel = client.channels.get(message.d.channel_id) + if (channel) { + channel["last_pin_timestamp"] = message.d.last_pin_timestamp + } + + + } else if (message.t === "GUILD_DELETE") { + client.guilds.delete(message.d.id) + const channels = client.guildChannelMap.get(message.d.id) + if (channels) { + for (const id of channels) client.channels.delete(id) + } + client.guildChannelMap.delete(message.d.id) + + + } else if (message.t === "CHANNEL_CREATE") { + client.channels.set(message.d.id, message.d) + if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have + const channels = client.guildChannelMap.get(message.d["guild_id"]) + if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) + } + + } else if (message.t === "CHANNEL_DELETE") { + client.channels.delete(message.d.id) + if (message.d["guild_id"]) { + const channels = client.guildChannelMap.get(message.d["guild_id"]) + if (channels) { + const previous = channels.indexOf(message.d.id) + if (previous !== -1) channels.splice(previous, 1) + } + } + } + + // Event dispatcher for OOYE bridge operations + if (listen === "full" && message.t) { + const alwaysRealTimeEvents = ["PRESENCE_UPDATE"] + if (alwaysRealTimeEvents.includes(message.t) || homeserverStatus.homeserverStatus.isRealTime()) { + dispatchPacketToBridge(client, message) + } else { + homeserverStatus.homeserverStatus.queuePacket(message) + } } } -module.exports = utils +/** + * @param {import("./discord-client")} client + * @param {import("cloudstorm").IGatewayMessage} message + */ +async function dispatchPacketToBridge(client, message) { + // requiring this later so that the client is already constructed by the time event-dispatcher is loaded + /** @type {typeof import("./event-dispatcher")} */ + const eventDispatcher = sync.require("./event-dispatcher") + /** @type {import("../discord/register-interactions")} */ + const interactions = sync.require("../discord/register-interactions") + + assert(message.t) // checked above + + try { + if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") { + await eventDispatcher.onSomeReactionsRemoved(client, message.d) + + } else if (message.t === "INTERACTION_CREATE") { + await interactions.dispatchInteraction(message.d) + + } else if (message.t in eventDispatcher) { + await eventDispatcher[message.t](client, message.d) + } + } catch (e) { + // Let OOYE try to handle errors too + await eventDispatcher.onError(client, e, message) + } +} + +module.exports.onPacket = onPacket +module.exports.dispatchPacketToBridge = dispatchPacketToBridge diff --git a/src/d2m/event-dispatcher.js b/src/d2m/event-dispatcher.js index 90824ac..31ef7ea 100644 --- a/src/d2m/event-dispatcher.js +++ b/src/d2m/event-dispatcher.js @@ -2,6 +2,7 @@ const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") +const {id: botID} = require("../../addbot") const {sync, db, select, from} = require("../passthrough") /** @type {import("./actions/send-message")}) */ @@ -38,6 +39,8 @@ const removeMember = sync.require("./actions/remove-member") const vote = sync.require("./actions/poll-vote") /** @type {import("../m2d/event-dispatcher")} */ const matrixEventDispatcher = sync.require("../m2d/event-dispatcher") +/** @type {import("../m2d/actions/redact.js")} */ +const redact = sync.require("../m2d/actions/redact.js") /** @type {import("../discord/interactions/matrix-info")} */ const matrixInfoInteraction = sync.require("../discord/interactions/matrix-info") @@ -57,7 +60,7 @@ module.exports = { matrixEventDispatcher.printError(gatewayMessage.t, "Discord", e, gatewayMessage) - const channelID = gatewayMessage.d["channel_id"] + const channelID = gatewayMessage.d?.["channel_id"] if (!channelID) return const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() if (!roomID) return @@ -310,13 +313,13 @@ module.exports = { if (!createRoom.existsOrAutocreatable(channel, guild.id)) return // Check that the sending-to room exists or is autocreatable - const {affected, row} = await speedbump.maybeDoSpeedbump(message.channel_id, message.id) + const {affected, row} = await speedbump.maybeDoSpeedbump(message.channel_id, message.id, message.author.id) if (affected) return // @ts-ignore await sendMessage.sendMessage(message, channel, guild, row) - retrigger.messageFinishedBridging(message.id) + retrigger.finishedBridging(message.id) }, /** @@ -332,13 +335,11 @@ module.exports = { if (dUtils.isEphemeralMessage(data)) return // Ephemeral messages are for the eyes of the receiver only! // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. - const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id) + const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id, data.author.id) if (affected) return - if (!row) { - // Check that the sending-to room exists, and deal with Eventual Consistency(TM) - if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_UPDATE, client, data)) return - } + // Check that the sending-to room exists, and deal with Eventual Consistency(TM) + if (!await retrigger.waitForMessage(data.id)) return /** @type {DiscordTypes.GatewayMessageCreateDispatchData} */ // @ts-ignore @@ -375,6 +376,16 @@ module.exports = { * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData | DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData | DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data */ async onSomeReactionsRemoved(client, data) { + // Don't attempt to double-bridge our own m2d deleted reactions back to Matrix + if ("user_id" in data && data.user_id === botID) { + const emojiIdOrName = data.emoji.id || data.emoji.name + const i = redact.m2dDeletedReactions.findIndex(x => data.message_id === x.messageID && emojiIdOrName === x.emojiIdOrName) + if (i !== -1) { + redact.m2dDeletedReactions.splice(i, 1) + return + } + } + await removeReaction.removeSomeReactions(data) }, @@ -384,7 +395,7 @@ module.exports = { */ async MESSAGE_DELETE(client, data) { speedbump.onMessageDelete(data.id) - if (retrigger.eventNotFoundThenRetrigger(data.id, module.exports.MESSAGE_DELETE, client, data)) return + if (!await retrigger.waitForMessage(data.id)) return await deleteMessage.deleteMessage(data) }, @@ -432,12 +443,12 @@ module.exports = { * @param {DiscordTypes.GatewayMessagePollVoteDispatchData} data */ async MESSAGE_POLL_VOTE_ADD(client, data) { - if (retrigger.eventNotFoundThenRetrigger(data.message_id, module.exports.MESSAGE_POLL_VOTE_ADD, client, data)) return + if (!await retrigger.waitForMessage(data.message_id)) return await vote.addVote(data) }, async MESSAGE_POLL_VOTE_REMOVE(client, data) { - if (retrigger.eventNotFoundThenRetrigger(data.message_id, module.exports.MESSAGE_POLL_VOTE_REMOVE, client, data)) return + if (!await retrigger.waitForMessage(data.message_id)) return await vote.removeVote(data) }, diff --git a/src/db/orm.js b/src/db/orm.js index 4d9b6f1..8763314 100644 --- a/src/db/orm.js +++ b/src/db/orm.js @@ -104,6 +104,16 @@ class From { return r } + pluckUnsafe(col) { + /** @type {Pluck} */ + // @ts-ignore + const r = this + r.cols = [col] + r.makeColsSafe = false + r.isPluck = true + return r + } + /** * @param {string} sql */ diff --git a/src/db/orm.test.js b/src/db/orm.test.js index 6f6018e..4639090 100644 --- a/src/db/orm.test.js +++ b/src/db/orm.test.js @@ -68,3 +68,8 @@ test("orm: select unsafe works (to select complex column names that can't be typ .all() t.equal(results[0].power_level, 150) }) + +test("orm: pluck unsafe works (to select complex column names that can't be type verified)", t => { + const result = from("channel_room").where({guild_id: "112760669178241024"}).pluckUnsafe("count(*)").get() + t.equal(result, 7) +}) diff --git a/src/discord/register-interactions.js b/src/discord/register-interactions.js index e3d58c4..dc97287 100644 --- a/src/discord/register-interactions.js +++ b/src/discord/register-interactions.js @@ -16,7 +16,7 @@ const ping = sync.require("./interactions/ping.js") // User must have EVERY permission in default_member_permissions to be able to use the command function registerInteractions() { - discord.snow.interaction.bulkOverwriteApplicationCommands(id, [{ + discord.snow.interaction.editApplicationCommands(id, [{ name: "Matrix info", contexts: [DiscordTypes.InteractionContextType.Guild], type: DiscordTypes.ApplicationCommandType.Message, @@ -91,40 +91,32 @@ function registerInteractions() { async function dispatchInteraction(interaction) { const interactionId = interaction.data?.["custom_id"] || interaction.data?.["name"] try { - if (interaction.type === DiscordTypes.InteractionType.MessageComponent || interaction.type === DiscordTypes.InteractionType.ModalSubmit) { - // All we get is custom_id, don't know which context the button was clicked in. - // So we namespace these ourselves in the custom_id. Currently the only existing namespace is POLL_. - if (interaction.data.custom_id.startsWith("POLL_")) { - await poll.interact(interaction) + if (interactionId === "Matrix info") { + await matrixInfo.interact(interaction) + } else if (interactionId === "invite") { + await invite.interact(interaction) + } else if (interactionId === "invite_channel") { + await invite.interactButton(interaction) + } else if (interactionId === "Permissions") { + await permissions.interact(interaction) + } else if (interactionId === "permissions_edit") { + await permissions.interactEdit(interaction) + } else if (interactionId === "Responses") { + /** @type {DiscordTypes.APIMessageApplicationCommandGuildInteraction} */ // @ts-ignore + const messageInteraction = interaction + if (select("poll", "message_id", {message_id: messageInteraction.data.target_id}).get()) { + await pollResponses.interact(messageInteraction) } else { - throw new Error(`Unknown message component ${interaction.data.custom_id}`) + await reactions.interact(messageInteraction) } + } else if (interactionId === "ping") { + await ping.interact(interaction) + } else if (interactionId === "privacy") { + await privacy.interact(interaction) + } else if (interactionId.startsWith("POLL_")) { + await poll.interact(interaction) } else { - if (interactionId === "Matrix info") { - await matrixInfo.interact(interaction) - } else if (interactionId === "invite") { - await invite.interact(interaction) - } else if (interactionId === "invite_channel") { - await invite.interactButton(interaction) - } else if (interactionId === "Permissions") { - await permissions.interact(interaction) - } else if (interactionId === "permissions_edit") { - await permissions.interactEdit(interaction) - } else if (interactionId === "Responses") { - /** @type {DiscordTypes.APIMessageApplicationCommandGuildInteraction} */ // @ts-ignore - const messageInteraction = interaction - if (select("poll", "message_id", {message_id: messageInteraction.data.target_id}).get()) { - await pollResponses.interact(messageInteraction) - } else { - await reactions.interact(messageInteraction) - } - } else if (interactionId === "ping") { - await ping.interact(interaction) - } else if (interactionId === "privacy") { - await privacy.interact(interaction) - } else { - throw new Error(`Unknown interaction ${interactionId}`) - } + throw new Error(`Unknown interaction ${interactionId}`) } } catch (e) { let stackLines = null diff --git a/src/discord/utils.js b/src/discord/utils.js index 0d400f1..6fb0b43 100644 --- a/src/discord/utils.js +++ b/src/discord/utils.js @@ -182,6 +182,18 @@ function filterTo(xs, fn) { return filtered } +/** + * The parameters correspond to the columns of the channel_room table. + * @param {string} rowChannelID thread ID, OR channel ID if there is no thread + * @param {string | null | undefined} rowThreadParent channel ID if there is a thread + */ +function swapThreadID(rowChannelID, rowThreadParent) { + return { + channelID: rowThreadParent ? rowThreadParent : rowChannelID, + threadID: rowThreadParent ? rowChannelID : undefined + } +} + const supportedPlaintextPreviewExtensions = new Set([ "4d", "abnf", @@ -582,4 +594,5 @@ module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact module.exports.getPublicUrlForCdn = getPublicUrlForCdn module.exports.howOldUnbridgedMessage = howOldUnbridgedMessage module.exports.filterTo = filterTo +module.exports.swapThreadID = swapThreadID module.exports.supportedPlaintextPreviewExtensions = supportedPlaintextPreviewExtensions diff --git a/src/m2d/actions/add-reaction.js b/src/m2d/actions/add-reaction.js index e4981fb..c453244 100644 --- a/src/m2d/actions/add-reaction.js +++ b/src/m2d/actions/add-reaction.js @@ -17,7 +17,7 @@ const retrigger = sync.require("../../d2m/actions/retrigger") */ async function addReaction(event) { // Wait until the corresponding channel and message have already been bridged - if (retrigger.eventNotFoundThenRetrigger(event.content["m.relates_to"].event_id, () => as.emit("type:m.reaction", event))) return + if (!await retrigger.waitForEvent(event.content["m.relates_to"].event_id)) return // These will exist because it passed retrigger const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") @@ -50,6 +50,8 @@ async function addReaction(event) { } db.prepare("REPLACE INTO reaction (hashed_event_id, message_id, encoded_emoji, original_encoding) VALUES (?, ?, ?, ?)").run(utils.getEventIDHash(event.event_id), messageID, discordPreferredEncoding, key) + + retrigger.finishedBridging(event.event_id) } module.exports.addReaction = addReaction diff --git a/src/m2d/actions/emoji-sheet.js b/src/m2d/actions/emoji-sheet.js index ed5ab88..077240c 100644 --- a/src/m2d/actions/emoji-sheet.js +++ b/src/m2d/actions/emoji-sheet.js @@ -23,7 +23,7 @@ async function getAndConvertEmoji(mxc) { const res = await api.getMedia(mxc, {signal: abortController.signal}) if (res.status !== 200) { const root = await res.json() - throw new mreq.MatrixServerError(root, {mxc}) + throw new mreq.MatrixServerError(root, res.status, {mxc}) } const readable = stream.Readable.fromWeb(res.body) return emojiSheetConverter.convertImageStream(readable, () => { diff --git a/src/m2d/actions/redact.js b/src/m2d/actions/redact.js index 3135d31..7e49753 100644 --- a/src/m2d/actions/redact.js +++ b/src/m2d/actions/redact.js @@ -10,6 +10,9 @@ const utils = sync.require("../../matrix/utils") /** @type {import("../../d2m/actions/retrigger")} */ const retrigger = sync.require("../../d2m/actions/retrigger") +/** @type {{messageID: string, emojiIdOrName: string}[]} */ +const m2dDeletedReactions = [] + /** * @param {Ty.Event.Outer_M_Room_Redaction} event */ @@ -24,6 +27,21 @@ async function deleteMessage(event) { db.prepare("DELETE FROM message_room WHERE message_id = ?").run(rows[0].message_id) } +/** + * @param {Ty.Event.Outer_M_Room_Redaction} event + */ +async function removeMessageEvent(event) { + // Could be for removing a message or suppressing embeds. For more information, the message needs to be bridged first. + if (!await retrigger.waitForEvent(event.redacts)) return + + const row = select("event_message", ["event_type", "event_subtype", "part"], {event_id: event.redacts}).get() + if (row && row.event_type === "m.room.message" && row.event_subtype === "m.notice" && row.part === 1) { + await suppressEmbeds(event) + } else { + await deleteMessage(event) + } +} + /** * @param {Ty.Event.Outer_M_Room_Redaction} event */ @@ -41,11 +59,20 @@ async function suppressEmbeds(event) { * @param {Ty.Event.Outer_M_Room_Redaction} event */ async function removeReaction(event) { + if (!await retrigger.waitForReactionEvent(event.redacts)) return + const hash = utils.getEventIDHash(event.redacts) const row = from("reaction").join("message_room", "message_id").join("historical_channel_room", "historical_room_index") .select("reference_channel_id", "message_id", "encoded_emoji").where({hashed_event_id: hash}).get() if (!row) return - await discord.snow.channel.deleteReactionSelf(row.reference_channel_id, row.message_id, row.encoded_emoji) + // See how many Matrix-side reactions there are, and delete if it's the last one + const numberOfReactions = from("reaction").where({message_id: row.message_id, encoded_emoji: row.encoded_emoji}).pluckUnsafe("count(*)").get() + if (numberOfReactions === 1) { + // If a unicode emoji, the name is already the Discord preferred version because that's what was added and stored to encoded_emoji + const emojiIdOrName = decodeURIComponent(row.encoded_emoji).split(":").slice(-1)[0] + m2dDeletedReactions.push({messageID: row.message_id, emojiIdOrName}) + await discord.snow.channel.deleteReactionSelf(row.reference_channel_id, row.message_id, row.encoded_emoji) + } db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(hash) } @@ -54,18 +81,12 @@ async function removeReaction(event) { * @param {Ty.Event.Outer_M_Room_Redaction} event */ async function handle(event) { - // If this is for removing a reaction, try it - await removeReaction(event) - - // Or, it might be for removing a message or suppressing embeds. But to do that, the message needs to be bridged first. - if (retrigger.eventNotFoundThenRetrigger(event.redacts, () => as.emit("type:m.room.redaction", event))) return - - const row = select("event_message", ["event_type", "event_subtype", "part"], {event_id: event.redacts}).get() - if (row && row.event_type === "m.room.message" && row.event_subtype === "m.notice" && row.part === 1) { - await suppressEmbeds(event) - } else { - await deleteMessage(event) - } + // Don't know if it's a redaction for a reaction or an event, try both at the same time (otherwise waitFor will block) + await Promise.all([ + removeMessageEvent(event), + removeReaction(event) + ]) } module.exports.handle = handle +module.exports.m2dDeletedReactions = m2dDeletedReactions \ No newline at end of file diff --git a/src/m2d/actions/sticker.js b/src/m2d/actions/sticker.js index 8eeb5d2..f5a1e16 100644 --- a/src/m2d/actions/sticker.js +++ b/src/m2d/actions/sticker.js @@ -22,7 +22,7 @@ async function getAndResizeSticker(mxc) { const res = await api.getMedia(mxc) if (res.status !== 200) { const root = await res.json() - throw new mreq.MatrixServerError(root, {mxc}) + throw new mreq.MatrixServerError(root, res.status, {mxc}) } const streamIn = Readable.fromWeb(res.body) diff --git a/src/m2d/actions/update-pins.js b/src/m2d/actions/update-pins.js index 1ff2bb9..1b3f1bd 100644 --- a/src/m2d/actions/update-pins.js +++ b/src/m2d/actions/update-pins.js @@ -16,9 +16,9 @@ async function updatePins(pins, prev) { .select("reference_channel_id", "message_id").where({event_id}).and("ORDER BY part ASC").get() if (!row) continue if (added) { - discord.snow.channel.addChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message pinned on Matrix") + discord.snow.channel.createChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message pinned on Matrix") } else { - discord.snow.channel.removeChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message unpinned on Matrix") + discord.snow.channel.deleteChannelPinnedMessage(row.reference_channel_id, row.message_id, "Message unpinned on Matrix") } } } diff --git a/src/m2d/actions/vote.js b/src/m2d/actions/vote.js index 926b957..84f8cc7 100644 --- a/src/m2d/actions/vote.js +++ b/src/m2d/actions/vote.js @@ -1,18 +1,12 @@ // @ts-check const Ty = require("../../types") -const DiscordTypes = require("discord-api-types/v10") -const {Readable} = require("stream") const assert = require("assert").strict -const crypto = require("crypto") const passthrough = require("../../passthrough") -const {sync, discord, db, select} = passthrough +const {sync, db, select} = passthrough -const {reg} = require("../../matrix/read-registration") -/** @type {import("../../matrix/api")} */ -const api = sync.require("../../matrix/api") -/** @type {import("../../matrix/utils")} */ -const utils = sync.require("../../matrix/utils") +/** @type {import("../../discord/utils")} */ +const dUtils = sync.require("../../discord/utils") /** @type {import("../converters/poll-components")} */ const pollComponents = sync.require("../converters/poll-components") /** @type {import("./channel-webhook")} */ @@ -33,10 +27,11 @@ async function updateVote(event) { // If poll was started on Matrix, the Discord version is using components, so we can update that to the current status if (messageRow.source === 0) { - const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get() - assert(channelID) - await webhook.editMessageWithWebhook(channelID, messageID, pollComponents.getPollComponentsFromDatabase(messageID)) + const row = select("channel_room", ["channel_id", "thread_parent"], {room_id: event.room_id}).get() + assert(row) + const {channelID, threadID} = dUtils.swapThreadID(row.channel_id, row.thread_parent) + await webhook.editMessageWithWebhook(channelID, messageID, pollComponents.getPollComponentsFromDatabase(messageID), threadID) } } -module.exports.updateVote = updateVote \ No newline at end of file +module.exports.updateVote = updateVote diff --git a/src/m2d/converters/event-to-message.js b/src/m2d/converters/event-to-message.js index 0a18a14..fbaf769 100644 --- a/src/m2d/converters/event-to-message.js +++ b/src/m2d/converters/event-to-message.js @@ -10,6 +10,7 @@ const domino = require("domino") const assert = require("assert").strict const entities = require("entities") const pb = require("prettier-bytes") +const mimeTypes = require("mime-types") const {tag} = require("@cloudrac3r/html-template-tag") const passthrough = require("../../passthrough") @@ -97,15 +98,6 @@ turndownService.addRule("underline", { } }) -turndownService.addRule("blockquote", { - filter: "blockquote", - replacement: function (content) { - content = content.replace(/^\n+|\n+$/g, "") - content = content.replace(/^/gm, "> ") - return content - } -}) - turndownService.addRule("spoiler", { filter: function (node, options) { return node.tagName === "SPAN" && node.hasAttribute("data-mx-spoiler") @@ -622,13 +614,20 @@ async function eventToMessage(event, guild, channel, di) { captionContent.addLine(`(Spoiler: ${fileSpoilerReason})`) } + // If filename is not specified, create one from mimetype + let filename = event.content.filename || event.content.body + if (!filename) { + const extension = mimeTypes.extension(event.content.info?.mimetype) + if (!extension) throw new Error("Filename and mimetype missing. Please report this bug to your client.") + filename = `file.${extension}` + } + // File link as alternative to uploading if (!("file" in event.content) && event.content.info?.size > getFileSizeForGuild(guild)) { // Upload (unencrypted) file as link, because it's too large for Discord // Do this by constructing a sample Matrix message with the link and then use the text processor to convert that + the original caption. const url = mxUtils.getPublicUrlForMxc(event.content.url) assert(url) - const filename = event.content.filename || event.content.body const emoji = attachmentEmojis.has(event.content.msgtype) ? attachmentEmojis.get(event.content.msgtype) + " " : "" if (fileIsSpoiler) { captionContent.addLine(`${emoji}Uploaded SPOILER file: <${url}> (${pb(event.content.info.size)})`, tag`${emoji}Uploaded SPOILER file: ${filename} (${pb(event.content.info.size)})`) // the space is necessary to work around a bug in Discord's URL previewer. the preview still gets blurred in the client. @@ -637,7 +636,6 @@ async function eventToMessage(event, guild, channel, di) { } } else { // Upload file as file - let filename = event.content.filename || event.content.body if (fileIsSpoiler) filename = "SPOILER_" + filename if ("file" in event.content) { // Encrypted @@ -855,7 +853,7 @@ async function eventToMessage(event, guild, channel, di) { input = input.replace(/("https:\/\/matrix.to.*?<\/a>):?/g, "$1") // Element adds a bunch of
before but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those. - input = input.replace(/(?:\n|
\s*)*<\/blockquote>/g, "") + input = input.replace(/(?:\n|
\s*)*<\/blockquote>(?:\n|
\s*)*/g, "") // The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason. // But I should not count it if it's between block elements. diff --git a/src/m2d/converters/event-to-message.test.js b/src/m2d/converters/event-to-message.test.js index 650e442..158c2ef 100644 --- a/src/m2d/converters/event-to-message.test.js +++ b/src/m2d/converters/event-to-message.test.js @@ -855,6 +855,40 @@ test("event2message: html lines are bridged correctly", async t => { ) }) +test("event2message: breaks between consecutive blockquotes are preserved", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "this was inspired by someone telling me\n\n>betrothed is an insane vocab pull\n\n> i think thats part of why im so scared of cadence\n\n> if an insult comes my way, not only do i not know when, but i'll have to humiliate myself and look it up", + format: "org.matrix.custom.html", + formatted_body: "

this was inspired by someone telling me

\n
\n

betrothed is an insane vocab pull

\n
\n
\n

i think thats part of why im so scared of cadence

\n
\n
\n

if an insult comes my way, not only do i not know when, but i'll have to humiliate myself and look it up

\n
" + }, + origin_server_ts: 1783571303430, + event_id: "$cdltlkLTUp-ma2uYmLKy7o3RkDgpDbtdYfngST_igWE", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + allowed_mentions: { + parse: ["users", "roles"] + }, + username: "cadence [they]", + avatar_url: "https://bridge.example.org/download/letter-avatar?letter=C&hue=90", + content: "this was inspired by someone telling me" + + "\n\n> betrothed is an insane vocab pull" + + "\n\n> i think thats part of why im so scared of cadence" + + "\n\n> if an insult comes my way, not only do i not know when, but i'll have to humiliate myself and look it up" + }] + } + ) +}) + /*test("event2message: whitespace is retained", async t => { t.deepEqual( await eventToMessage({ @@ -1294,7 +1328,7 @@ test("event2message: characters are encoded properly in code blocks", async t => ) }) -test("event2message: quotes have an appropriate amount of whitespace", async t => { +test("event2message: quotes have an appropriate amount of whitespace (br following)", async t => { t.deepEqual( await eventToMessage({ content: { @@ -1328,6 +1362,40 @@ test("event2message: quotes have an appropriate amount of whitespace", async t = ) }) +test("event2message: quotes have an appropriate amount of whitespace (p following)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: "
\n

short story

\n
\n

by whose standards? LOL

\n

I basically went all the way back to childhood (which analysts always like you to do)

\n
\n

laughed out loud

" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "> short story\nby whose standards? LOL\n\n> I basically went all the way back to childhood (which analysts always like you to do)\nlaughed out loud", + avatar_url: "https://bridge.example.org/download/letter-avatar?letter=C&hue=90", + allowed_mentions: { + parse: ["users", "roles"] + } + }] + } + ) +}) + test("event2message: lists have appropriate line breaks", async t => { t.deepEqual( await eventToMessage({ diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 352ca41..129fca5 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -24,6 +24,8 @@ const vote = sync.require("./actions/vote") const matrixCommandHandler = sync.require("../matrix/matrix-command-handler") /** @type {import("../matrix/utils")} */ const utils = sync.require("../matrix/utils") +/** @type {import("../matrix/mreq")}) */ +const mreq = sync.require("../matrix/mreq") /** @type {import("../matrix/api")}) */ const api = sync.require("../matrix/api") /** @type {import("../d2m/actions/create-room")} */ @@ -32,6 +34,8 @@ const createRoom = sync.require("../d2m/actions/create-room") const roomUpgrade = require("../matrix/room-upgrade") /** @type {import("../d2m/actions/retrigger")} */ const retrigger = sync.require("../d2m/actions/retrigger") +/** @type {import("../matrix/homeserver-status")} */ +const homeserverStatus = sync.require("../matrix/homeserver-status") const {reg} = require("../matrix/read-registration") let lastReportedEvent = 0 @@ -166,7 +170,12 @@ async function sendError(roomID, source, type, e, payload) { key: "๐Ÿ”" } }) - } catch (e) {} + } catch (e) { + if (e instanceof mreq.MatrixServerError && [502, 503].includes(e.httpStatus) && e.errcode === "CX_SERVER_ERROR") { + // Matrix homeserver is down (reverse proxy indicated failure) + homeserverStatus.homeserverStatus.setErrorWithPacket(payload) + } + } } function guard(type, fn) { @@ -225,7 +234,7 @@ async event => { // @ts-ignore await matrixCommandHandler.execute(event) } - retrigger.messageFinishedBridging(event.event_id) + retrigger.finishedBridging(event.event_id) await api.ackEvent(event) })) @@ -236,7 +245,7 @@ sync.addTemporaryListener(as, "type:m.sticker", guard("m.sticker", async event => { if (utils.eventSenderIsFromDiscord(event.sender)) return const messageResponses = await sendEvent.sendEvent(event) - retrigger.messageFinishedBridging(event.event_id) + retrigger.finishedBridging(event.event_id) await api.ackEvent(event) })) diff --git a/src/matrix/api.js b/src/matrix/api.js index 9b7f280..47fc565 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -452,7 +452,8 @@ async function ping() { headers: { Authorization: `Bearer ${reg.as_token}` }, - body: "{}" + body: "{}", + signal: AbortSignal.timeout(15e3) }) const root = await res.json() return { @@ -474,7 +475,7 @@ async function ping() { async function getMedia(mxc, init = {}) { init = {...init} - const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) + const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/([a-zA-Z0-9_-]+)$/) assert(mediaParts) let route = "download" diff --git a/src/matrix/homeserver-status.js b/src/matrix/homeserver-status.js new file mode 100644 index 0000000..3ceac84 --- /dev/null +++ b/src/matrix/homeserver-status.js @@ -0,0 +1,134 @@ +// @ts-check + +const assert = require("assert") +const Denque = require("denque") +const StateMachine = require("snowtransfer").StateMachine + +const passthrough = require("../passthrough") +const {sync} = passthrough +/** @type {import("../d2m/discord-packets")} */ +const discordPackets = sync.require("../d2m/discord-packets") +/** @type {import("../matrix/api")} */ +const api = sync.require("../matrix/api") + +const DEBUG_HOMESERVER_STATUS = false + +function debugHomeserverStatus(message) { + if (DEBUG_HOMESERVER_STATUS) { + console.log(message) + } +} + +const homeserverStatus = new class HomeserverStatus { + constructor() { + /** @private */ + this.queue = new Denque() + + /** @private */ + this.pingInterval = undefined + + /** @private */ + this.sm = new StateMachine("online") + .defineState("online") + + .defineState("checking", { + onEnter: [async () => { + const pingResult = await api.ping().catch(e => ({ok: false})) + if (pingResult.ok) { + this.sm.doTransition("check ok") + } else { + this.sm.doTransition("check fail") + } + }], + onLeave: [], + transitions: new Map() + }) + + .defineState("offline", { + onEnter: [() => { + this.pingInterval = setInterval(async () => { + const pingResult = await api.ping().catch(e => ({ok: false, status: "net", root: e.message})) + if (pingResult.ok) { + this.sm.doTransition("ping ok") + } + }, 15e3) + }], + onLeave: [() => { + clearInterval(this.pingInterval) + }], + transitions: new Map() + }) + + .defineState("recovering", { + onEnter: [async () => { // Drain queue. + while (!this.queue.isEmpty()) { + const packet = this.queue.peekFront() // same position as .shift() + debugHomeserverStatus(`homeserver status: ${new Date().toISOString()} dq packet ${packet.t} ${packet.d?.content}`) + await discordPackets.dispatchPacketToBridge(passthrough.discord, packet) + if (this.sm.currentStateName !== "recovering") return // got kicked out due to another error + this.queue.shift() + } + this.sm.doTransition("recovered") + }], + onLeave: [], + transitions: new Map() + }) + + .defineUniversalTransition("error", "offline") + .defineTransition("offline", "ping ok", "recovering") + .defineTransition("recovering", "recovered", "online") + .defineTransition("online", "check", "checking") + .defineTransition("checking", "check ok", "recovering") + .defineTransition("checking", "check fail", "offline") + + this.sm.on("enter", st => debugHomeserverStatus(`homeserver status: ${st}`)) + this.sm.setMaxListeners(101) + this.sm.freeze() + } + + isRealTime() { + return this.sm.currentStateName === "online" + } + + /** @param {boolean} forceCheck */ + waitForOnline(forceCheck) { + const onlinePromise = new Promise(resolve => { + // Already online? Start check or just done + if (this.sm.currentStateName === "online") { + if (forceCheck) { + this.sm.doTransition("check") + } else { + return resolve(null) + } + } + + // Checking or not online. Wait for online. + const onlineListener = stateName => { + if (stateName === "online") { + this.sm.removeListener("enter", onlineListener) + resolve(null) + } + } + this.sm.on("enter", onlineListener) + }) + return onlinePromise + } + + /** + * When offline or recovering, call this for incoming packets to queue them to be sent in order later. + */ + queuePacket(packet) { + assert(["offline", "recovering"].includes(this.sm.currentStateName)) + this.queue.push(packet) + } + + setErrorWithPacket(packet) { + const wasRecovering = this.sm.currentStateName === "recovering" + this.sm.doTransition("error") + if (!wasRecovering) { // if was recovering then packet is already in the right place in queue + this.queuePacket(packet) + } + } +} + +module.exports.homeserverStatus = homeserverStatus diff --git a/src/matrix/mreq.js b/src/matrix/mreq.js index bb59506..bf238f2 100644 --- a/src/matrix/mreq.js +++ b/src/matrix/mreq.js @@ -9,9 +9,12 @@ const {reg} = require("./read-registration.js") const baseUrl = `${reg.ooye.server_origin}/_matrix` class MatrixServerError extends Error { - constructor(data, opts) { + /** @param {number} httpStatus} */ + constructor(data, httpStatus, opts) { super(data.error || data.errcode) this.data = data + /** @type {number} */ + this.httpStatus = httpStatus /** @type {string} */ this.errcode = data.errcode this.opts = opts @@ -44,11 +47,11 @@ async function _convertBody(body) { async function makeMatrixServerError(res, opts = {}) { delete opts.headers?.["Authorization"] if (res.headers.get("content-type") === "application/json") { - return new MatrixServerError(await res.json(), opts) + return new MatrixServerError(await res.json(), res.status, opts) } else if (res.headers.get("content-type")?.startsWith("text/")) { - return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, opts) + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, res.status, opts) } else { - return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, opts) + return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, res.status, opts) } } @@ -78,12 +81,12 @@ async function mreq(method, url, bodyIn, extra = {}) { var root = JSON.parse(text) } catch (e) { delete opts.headers?.["Authorization"] - throw new MatrixServerError(text, {baseUrl, url, ...opts}) + throw new MatrixServerError(text, res.status, {baseUrl, url, ...opts}) } if (!res.ok || root.errcode) { delete opts.headers?.["Authorization"] - throw new MatrixServerError(root, {baseUrl, url, ...opts}) + throw new MatrixServerError(root, res.status, {baseUrl, url, ...opts}) } return root } diff --git a/src/matrix/utils.js b/src/matrix/utils.js index eee635b..fed71c9 100644 --- a/src/matrix/utils.js +++ b/src/matrix/utils.js @@ -250,7 +250,7 @@ function getPublicUrlForMxc(mxc) { */ function makeMxcPublic(mxc) { assert(hasher, "xxhash is not ready yet") - const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/) + const mediaParts = mxc?.match(/^mxc:\/\/([^/]+)\/([a-zA-Z0-9_-]+)$/) if (!mediaParts) return undefined const serverAndMediaID = `${mediaParts[1]}/${mediaParts[2]}` diff --git a/src/stdin.js b/src/stdin.js index 04b0151..43f9607 100644 --- a/src/stdin.js +++ b/src/stdin.js @@ -15,6 +15,7 @@ const mreq = sync.require("./matrix/mreq") const api = sync.require("./matrix/api") const file = sync.require("./matrix/file") const sendEvent = sync.require("./m2d/actions/send-event") +const redact = sync.require("./m2d/actions/redact") const eventDispatcher = sync.require("./d2m/event-dispatcher") const updatePins = sync.require("./d2m/actions/update-pins") const speedbump = sync.require("./d2m/actions/speedbump") @@ -22,7 +23,7 @@ const ks = sync.require("./matrix/kstate") const setPresence = sync.require("./d2m/actions/set-presence") const channelWebhook = sync.require("./m2d/actions/channel-webhook") const dUtils = sync.require("./discord/utils") -const mUtils = sync.require("./matrix/utils") +const mxUtils = sync.require("./matrix/utils") const guildID = "112760669178241024" async function ping() { diff --git a/src/web/pug/guild.pug b/src/web/pug/guild.pug index 9791ae3..7411a1e 100644 --- a/src/web/pug/guild.pug +++ b/src/web/pug/guild.pug @@ -122,7 +122,7 @@ block body #role-add.s-popover(popover style="display: revert").ws2.px0.py4.bs-lg.overflow-visible .s-popover--arrow.s-popover--arrow__tc +add-roles-menu(guild, guild_id) - p.fc-medium.mb0.mt8 Matrix users will start with these roles. If your main channels are gated by a role, use this to let Matrix users skip the gate. + p.fc-light.mb0.mt8 Matrix users will start with these roles. If your main channels are gated by a role, use this to let Matrix users skip the gate. h3.mt32.fs-category Features .s-card.d-grid.px0.g16 @@ -191,14 +191,14 @@ block body label.s-btn.s-btn__muted.ta-left.truncate(for=channel.id) +discord(channel, true, "Announcement") else - .s-empty-state.p8 All Discord channels are linked. + .s-empty-state.p8 No Discord channels available. .fl-grow1.s-btn-group.fd-column.w30 each room in unlinkedRooms input.s-btn--radio(type="radio" name="matrix" required id=room.room_id value=room.room_id) label.s-btn.s-btn__muted.ta-left.truncate(for=room.room_id) +matrix(room, true) else - .s-empty-state.p8 All Matrix rooms are linked. + .s-empty-state.p8 No Matrix rooms available. input(type="hidden" name="guild_id" value=guild_id) div button.s-btn.s-btn__icon.s-btn__filled#link-button diff --git a/start.js b/start.js index 39e8ea0..51942c6 100755 --- a/start.js +++ b/start.js @@ -31,8 +31,9 @@ sync.require("./src/m2d/event-dispatcher") ;(async () => { await migrate.migrate(db) + process.stdout.write("Connecting to Discord... ") await discord.cloud.connect() - console.log("Discord gateway started") + console.log("ok.") sync.require("./src/web/server") await power.applyPower() diff --git a/test/data.js b/test/data.js index f3092bc..eab9a63 100644 --- a/test/data.js +++ b/test/data.js @@ -5473,6 +5473,189 @@ module.exports = { content: '-# Original Message ID: 1466556003645657118 ยท ' } ] + }, + pk_ping_components_v1: { + type: 23, + content: "Psst, **Red** (<@772659086046658620>), you have been pinged by <@772659086046658620>.", + mentions: [ + { + id: "772659086046658620", + username: "cadence.worm", + avatar: "466df0c98b1af1e1388f595b4c1ad1b9", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + collectibles: null, + display_name_styles: null, + banner_color: null, + clan: { + identity_guild_id: "532245108070809601", + identity_enabled: true, + tag: "doll", + badge: "dba08126b4e810a0e096cc7cd5bc37f0" + }, + primary_guild: { + identity_guild_id: "532245108070809601", + identity_enabled: true, + tag: "doll", + badge: "dba08126b4e810a0e096cc7cd5bc37f0" + } + } + ], + mention_roles: [], + attachments: [], + embeds: [], + timestamp: "2026-03-25T07:07:02.626000+00:00", + edited_timestamp: null, + flags: 0, + components: [ + { + type: 1, + id: 1, + components: [ + { + type: 2, + id: 2, + style: 5, + label: "Jump", + url: "https://discord.com/channels/1160893336324931584/1160894080998461480/1440549403667468320" + } + ] + } + ], + id: "1486260105908457653", + channel_id: "1160894080998461480", + author: { + id: "466378653216014359", + username: "PluralKit", + avatar: "b78ef67a081737a830b60aa47d9ebcd9", + discriminator: "4020", + public_flags: 65536, + flags: 65536, + bot: true, + banner: null, + accent_color: null, + global_name: null, + avatar_decoration_data: null, + collectibles: null, + display_name_styles: null, + banner_color: null, + clan: null, + primary_guild: null + }, + pinned: false, + mention_everyone: false, + tts: false, + application_id: "466378653216014359", + interaction: { + id: "1486260103928614932", + type: 2, + name: "๐Ÿ”” Ping author", + user: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "466df0c98b1af1e1388f595b4c1ad1b9", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + collectibles: null, + display_name_styles: null, + banner_color: null, + clan: { + identity_guild_id: "532245108070809601", + identity_enabled: true, + tag: "doll", + badge: "dba08126b4e810a0e096cc7cd5bc37f0" + }, + primary_guild: { + identity_guild_id: "532245108070809601", + identity_enabled: true, + tag: "doll", + badge: "dba08126b4e810a0e096cc7cd5bc37f0" + } + } + }, + webhook_id: "466378653216014359", + message_reference: { + type: 0, + channel_id: "1160894080998461480", + message_id: "1440549403667468320", + guild_id: "1160893336324931584" + }, + interaction_metadata: { + id: "1486260103928614932", + type: 2, + user: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "466df0c98b1af1e1388f595b4c1ad1b9", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + collectibles: null, + display_name_styles: null, + banner_color: null, + clan: { + identity_guild_id: "532245108070809601", + identity_enabled: true, + tag: "doll", + badge: "dba08126b4e810a0e096cc7cd5bc37f0" + }, + primary_guild: { + identity_guild_id: "532245108070809601", + identity_enabled: true, + tag: "doll", + badge: "dba08126b4e810a0e096cc7cd5bc37f0" + } + }, + authorizing_integration_owners: { "0": "1160893336324931584" }, + name: "๐Ÿ”” Ping author", + command_type: 3, + target_message_id: "1440549403667468320" + }, + referenced_message: { + type: 0, + content: "test", + mentions: [], + mention_roles: [], + attachments: [], + embeds: [], + timestamp: "2025-11-19T03:49:01.948000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + id: "1440549403667468320", + channel_id: "1160894080998461480", + author: { + id: "1195662438662680720", + username: "special name", + avatar: "a82347890f2739e5880cd82b8c1a708e", + discriminator: "0000", + public_flags: 0, + flags: 0, + bot: true, + global_name: null, + clan: null, + primary_guild: null + }, + pinned: false, + mention_everyone: false, + tts: false, + application_id: "466378653216014359", + webhook_id: "1195662438662680720" + } } }, message_update: { diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 8dd71cd..1662320 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -95,7 +95,8 @@ WITH a (message_id, channel_id) AS (VALUES ('1381212840957972480', '112760669178241024'), ('1401760355339862066', '112760669178241024'), ('1439351590262800565', '1438284564815548418'), -('1404133238414376971', '112760669178241024')) +('1404133238414376971', '112760669178241024'), +('1440549403667468320', '1160894080998461480')) SELECT message_id, max(historical_room_index) as historical_room_index FROM a INNER JOIN historical_channel_room ON historical_channel_room.reference_channel_id = a.channel_id GROUP BY message_id; INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES @@ -143,7 +144,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part ('$7P2O_VTQNHvavX5zNJ35DV-dbJB1Ag80tGQP_JzGdhk', 'm.room.message', 'm.text', '1401760355339862066', 0, 0, 0), ('$ielAnR6geu0P1Tl5UXfrbxlIf-SV9jrNprxrGXP3v7M', 'm.room.message', 'm.image', '1439351590262800565', 0, 0, 0), ('$uUKLcTQvik5tgtTGDKuzn0Ci4zcCvSoUcYn2X7mXm9I', 'm.room.message', 'm.text', '1404133238414376971', 0, 1, 1), -('$LhmoWWvYyn5_AHkfb6FaXmLI6ZOC1kloql5P40YDmIk', 'm.room.message', 'm.notice', '1404133238414376971', 1, 0, 1); +('$LhmoWWvYyn5_AHkfb6FaXmLI6ZOC1kloql5P40YDmIk', 'm.room.message', 'm.notice', '1404133238414376971', 1, 0, 1), +('$l9FMmsEbh9K0NUReeEpWOMZYGRlUOE8yLcm6P-TYHSM', 'm.room.message', 'm.text', '1440549403667468320', 0, 0, 1); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),