From 1d7ec4315ed046cd6ba2f0215e8171a0dde42638 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 16 Oct 2023 00:12:00 +1300 Subject: [PATCH 001/452] Support alternate Discord hosts --- d2m/discord-client.js | 6 +++++- registration.example.yaml | 2 ++ types.d.ts | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/d2m/discord-client.js b/d2m/discord-client.js index b1a1e811..6597889d 100644 --- a/d2m/discord-client.js +++ b/d2m/discord-client.js @@ -1,8 +1,12 @@ // @ts-check -const { SnowTransfer } = require("snowtransfer") +const registration = require("../matrix/read-registration") +const { Endpoints, SnowTransfer } = require("snowtransfer") const { Client: CloudStorm } = require("cloudstorm") +// @ts-ignore +Endpoints.BASE_HOST = registration.ooye.discord_origin; Endpoints.CDN_URL = registration.ooye.discord_cdn_origin + const passthrough = require("../passthrough") const { sync } = passthrough diff --git a/registration.example.yaml b/registration.example.yaml index 9e7cd2c5..879081d6 100644 --- a/registration.example.yaml +++ b/registration.example.yaml @@ -18,6 +18,8 @@ ooye: max_file_size: 5000000 server_name: [the part after the colon in your matrix id, like cadence.moe] server_origin: [the full protocol and domain of your actual matrix server's location, with no trailing slash, like https://matrix.cadence.moe] + discord_origin: https://discord.com + discord_cdn_origin: https://cdn.discordapp.com invite: # uncomment this to auto-invite the named user to newly created spaces and mark them as admin (PL 100) everywhere # - @cadence:cadence.moe diff --git a/types.d.ts b/types.d.ts index e93f3790..656c22ae 100644 --- a/types.d.ts +++ b/types.d.ts @@ -21,6 +21,8 @@ export type AppServiceRegistrationConfig = { max_file_size: number server_name: string server_origin: string + discord_origin: string + discord_cdn_origin: string invite: string[] } old_bridge?: { From a542bbdca79ba0825e08fe589f85224e212c9b88 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 16 Oct 2023 16:24:48 +1300 Subject: [PATCH 002/452] Update discord-markdown to support list and header --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index a0056451..11585f87 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.9.0", - "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", @@ -1058,8 +1058,8 @@ "integrity": "sha512-5BELXTsv7becqVHrD81nZrqT4oEyXXWBwbsO/kwDDu6X3u19VV1tYDB5I5vaVAK+c1chcDeheI9zACBLm41LiQ==" }, "node_modules/discord-markdown": { - "version": "2.4.1", - "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "version": "2.5.1", + "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "license": "MIT", "dependencies": { "simple-markdown": "^0.7.2" diff --git a/package.json b/package.json index 29f6f1cc..42988c8a 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.9.0", - "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", From 762e48230c682fb9256119a51d600dc6bbafbadf Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 16 Oct 2023 16:47:42 +1300 Subject: [PATCH 003/452] Ensure the appservice bot user is registered Synapse does it automatically, but Conduit requires this HTTP call. --- d2m/actions/register-user.js | 2 +- scripts/seed.js | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js index 9b5527f6..b605c3ae 100644 --- a/d2m/actions/register-user.js +++ b/d2m/actions/register-user.js @@ -36,7 +36,7 @@ async function createSim(user) { await api.register(localpart) } catch (e) { // If user creation fails, manually undo the database change. Still isn't perfect, but should help. - // (A transaction would be preferable, but I don't think it's safe to leave transaction open across event loop ticks.) + // (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.) db.prepare("DELETE FROM sim WHERE user_id = ?").run(user.id) throw e } diff --git a/scripts/seed.js b/scripts/seed.js index 0f2f23d6..4fc704c1 100644 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -67,6 +67,18 @@ async function uploadAutoEmoji(guild, name, filename) { console.log("✅ Database is ready...") + // ensure appservice bot user is registered... + try { + await api.register(reg.sender_localpart) + } catch (e) { + if (e.data?.error !== "Internal server error") throw e // "Internal server error" is the only OK error because Synapse says this if you try to register the same username twice. + } + + // upload initial images... + const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png") + + console.log("✅ Matrix appservice login works...") + // upload the L1 L2 emojis to some guild const emojis = db.prepare("SELECT name FROM auto_emoji WHERE name = 'L1' OR name = 'L2'").pluck().all() if (emojis.length !== 2) { @@ -104,11 +116,6 @@ async function uploadAutoEmoji(guild, name, filename) { } console.log("✅ Emojis are ready...") - // ensure homeserver well-known is valid and returns reg.ooye.server_name... - - // upload initial images... - const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png") - // set profile data on discord... const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer()) await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")}) From afbbe0da3d9bf561e3f5394c0fb6009a5d9f6711 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 28 Oct 2023 00:24:42 +1300 Subject: [PATCH 004/452] Fix more edge-case embed formatting --- .../message-to-event.embeds.test.js | 68 ++++++-- d2m/converters/message-to-event.js | 149 ++++++++++++------ m2d/converters/emoji.js | 1 + m2d/converters/event-to-message.js | 18 +++ m2d/converters/event-to-message.test.js | 43 ++++- m2d/converters/utils.js | 65 ++++++++ matrix/matrix-command-handler.js | 53 +------ package-lock.json | 23 ++- package.json | 4 +- readme.md | 8 +- test/data.js | 108 +++++++++++++ test/ooye-test-data.sql | 15 +- 12 files changed, 428 insertions(+), 127 deletions(-) diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js index 93d189c4..173e0167 100644 --- a/d2m/converters/message-to-event.embeds.test.js +++ b/d2m/converters/message-to-event.embeds.test.js @@ -35,14 +35,14 @@ test("message2event embeds: nothing but a field", async t => { $type: "m.room.message", "m.mentions": {}, msgtype: "m.notice", - body: "> **Amanda 🎵#2192 :online:" - + "\n> willow tree, branch 0**" + body: "> ### Amanda 🎵#2192 :online:" + + "\n> willow tree, branch 0" + "\n> **❯ Uptime:**\n> 3m 55s\n> **❯ Memory:**\n> 64.45MB", format: "org.matrix.custom.html", - formatted_body: '
Amanda 🎵#2192 \":online:\"' + formatted_body: '

Amanda 🎵#2192 \":online:\"' + '
willow tree, branch 0' + '
❯ Uptime:
3m 55s' - + '
❯ Memory:
64.45MB

' + + '
❯ Memory:
64.45MB

' }]) }) @@ -52,19 +52,19 @@ test("message2event embeds: reply with just an embed", async t => { $type: "m.room.message", msgtype: "m.notice", "m.mentions": {}, - body: "> [**⏺️ dynastic (@dynastic)**](https://twitter.com/i/user/719631291747078145)" - + "\n> \n> **https://twitter.com/i/status/1707484191963648161**" + body: "> ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145" + + "\n> \n> ## https://twitter.com/i/status/1707484191963648161" + "\n> \n> does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?" - + "\n> \n> **Retweets**" + + "\n> \n> ### Retweets" + "\n> 119" - + "\n> \n> **Likes**" + + "\n> \n> ### Likes" + "\n> 5581" - + "\n> \n> — Twitter", + + "\n> — Twitter", format: "org.matrix.custom.html", - formatted_body: '
⏺️ dynastic (@dynastic)' - + '

https://twitter.com/i/status/1707484191963648161' - + '

does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?' - + '

Retweets
119

Likes
5581

— Twitter
' + formatted_body: '

⏺️ dynastic (@dynastic)

' + + '

https://twitter.com/i/status/1707484191963648161' + + '

does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?' + + '

Retweets
119

Likes
5581

— Twitter
' }]) }) @@ -99,3 +99,45 @@ test("message2event embeds: image embed and attachment", async t => { "m.mentions": {} }]) }) + +test("message2event embeds: blockquote in embed", async t => { + const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: ":emoji: **4 |** #wonderland", + format: "org.matrix.custom.html", + formatted_body: `\":emoji:\" 4 | #wonderland`, + "m.mentions": {} + }, { + $type: "m.room.message", + msgtype: "m.notice", + body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo)", + format: "org.matrix.custom.html", + formatted_body: "

⏺️ minimus

reply draft

The following is a message composed via consensus of the Stinker Council.

For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.

Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.

There will be no further communication.

Go to Message

", + "m.mentions": {} + }]) +}) + +test("message2event embeds: crazy html is all escaped", async t => { + const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.notice", + body: "> ## ⏺️ [Hey