add readme

index.js

@@ -16,7 +16,7 @@ const DiscordClient = require("./d2m/discord-client")
 const discord = new DiscordClient(config.discordToken, "full")
 passthrough.discord = discord
 
-const as = require("./m2d/appservice")
+const as = require("./matrix/appservice")
 passthrough.as = as
 
 sync.require("./m2d/event-dispatcher")

package-lock.json

@@ -17,7 +17,6 @@
         "heatsync": "^2.4.1",
         "js-yaml": "^4.1.0",
         "matrix-appservice": "^2.0.0",
-        "matrix-js-sdk": "^24.1.0",
         "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
         "node-fetch": "^2.6.7",
         "prettier-bytes": "^1.0.4",
@@ -35,17 +34,6 @@
         "tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d"
       }
     },
-    "node_modules/@babel/runtime": {
-      "version": "7.21.5",
-      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz",
-      "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==",
-      "dependencies": {
-        "regenerator-runtime": "^0.13.11"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      }
-    },
     "node_modules/@bcoe/v8-coverage": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz",
@@ -203,14 +191,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
-    "node_modules/@matrix-org/matrix-sdk-crypto-js": {
-      "version": "0.1.0-alpha.8",
-      "resolved": "https://registry.npmjs.org/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.8.tgz",
-      "integrity": "sha512-hdmbbGXKrN6JNo3wdBaR5Zs3lXlzllT3U43ViNTlabB3nKkOZQnEAN/Isv+4EQSgz1+8897veI9Q8sqlQX22oA==",
-      "engines": {
-        "node": ">= 10"
-      }
-    },
     "node_modules/@putout/cli-keypress": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/@putout/cli-keypress/-/cli-keypress-1.0.0.tgz",
@@ -326,11 +306,6 @@
         "node": ">=16"
       }
     },
-    "node_modules/@types/events": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz",
-      "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g=="
-    },
     "node_modules/@types/istanbul-lib-coverage": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz",
@@ -368,11 +343,6 @@
         "csstype": "^3.0.2"
       }
     },
-    "node_modules/@types/retry": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
-      "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="
-    },
     "node_modules/@types/scheduler": {
       "version": "0.16.3",
       "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz",
@@ -402,11 +372,6 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/another-json": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/another-json/-/another-json-0.2.0.tgz",
-      "integrity": "sha512-/Ndrl68UQLhnCdsAzEXLMFuOR546o2qbYRqCglaNHbjXrwG1ayTcdwr3zkSGOGtGXDyR5X9nCFfnyG2AFJIsqg=="
-    },
     "node_modules/ansi-regex": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
@@ -494,11 +459,6 @@
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
       "dev": true
     },
-    "node_modules/base-x": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/base-x/-/base-x-4.0.0.tgz",
-      "integrity": "sha512-FuwxlW4H5kh37X/oW59pwTzzTKRzfrrQwhmyspRM7swOEZcHtDZSCt45U6oKgtuFE+WYPblePMVIPR4RZrh/hw=="
-    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -630,14 +590,6 @@
         "balanced-match": "^1.0.0"
       }
     },
-    "node_modules/bs58": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/bs58/-/bs58-5.0.0.tgz",
-      "integrity": "sha512-r+ihvQJvahgYT50JD05dyJNKlmmSlMoOGwn1lCcEzanPglg7TxYjioQUYehQ9mAR/+hOSd2jRc/Z2y5UxBymvQ==",
-      "dependencies": {
-        "base-x": "^4.0.0"
-      }
-    },
     "node_modules/buffer": {
       "version": "6.0.3",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
@@ -1164,6 +1116,7 @@
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
       "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "dev": true,
       "engines": {
         "node": ">=0.8.x"
       }
@@ -1965,18 +1918,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/loglevel": {
-      "version": "1.8.1",
-      "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.8.1.tgz",
-      "integrity": "sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg==",
-      "engines": {
-        "node": ">= 0.6.0"
-      },
-      "funding": {
-        "type": "tidelift",
-        "url": "https://tidelift.com/funding/github/npm/loglevel"
-      }
-    },
     "node_modules/lru-cache": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@@ -2017,42 +1958,6 @@
         "node": ">=18"
       }
     },
-    "node_modules/matrix-events-sdk": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz",
-      "integrity": "sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA=="
-    },
-    "node_modules/matrix-js-sdk": {
-      "version": "24.1.0",
-      "resolved": "https://registry.npmjs.org/matrix-js-sdk/-/matrix-js-sdk-24.1.0.tgz",
-      "integrity": "sha512-xEx2ZoNsS56dwgqLJ3rIv2SUpFxdQLrLKmJCpMatMUKCAg+NGuZfpQ3QXblIbGaqFNQZCH7fC7S48AeTMZp1Jw==",
-      "dependencies": {
-        "@babel/runtime": "^7.12.5",
-        "@matrix-org/matrix-sdk-crypto-js": "^0.1.0-alpha.5",
-        "another-json": "^0.2.0",
-        "bs58": "^5.0.0",
-        "content-type": "^1.0.4",
-        "loglevel": "^1.7.1",
-        "matrix-events-sdk": "0.0.1",
-        "matrix-widget-api": "^1.3.1",
-        "p-retry": "4",
-        "sdp-transform": "^2.14.1",
-        "unhomoglyph": "^1.0.6",
-        "uuid": "9"
-      },
-      "engines": {
-        "node": ">=16.0.0"
-      }
-    },
-    "node_modules/matrix-widget-api": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/matrix-widget-api/-/matrix-widget-api-1.4.0.tgz",
-      "integrity": "sha512-dw0dRylGQzDUoiaY/g5xx1tBbS7aoov31PRtFMAvG58/4uerYllV9Gfou7w+I1aglwB6hihTREzKltVjARWV6A==",
-      "dependencies": {
-        "@types/events": "^3.0.0",
-        "events": "^3.2.0"
-      }
-    },
     "node_modules/media-typer": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -2330,18 +2235,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/p-retry": {
-      "version": "4.6.2",
-      "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",
-      "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==",
-      "dependencies": {
-        "@types/retry": "0.12.0",
-        "retry": "^0.13.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -2558,11 +2451,6 @@
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
     },
-    "node_modules/regenerator-runtime": {
-      "version": "0.13.11",
-      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
-      "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg=="
-    },
     "node_modules/regexp.prototype.flags": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz",
@@ -2606,14 +2494,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/retry": {
-      "version": "0.13.1",
-      "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
-      "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
-      "engines": {
-        "node": ">= 4"
-      }
-    },
     "node_modules/rimraf": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -2703,14 +2583,6 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
-    "node_modules/sdp-transform": {
-      "version": "2.14.1",
-      "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.1.tgz",
-      "integrity": "sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==",
-      "bin": {
-        "sdp-verify": "checker.js"
-      }
-    },
     "node_modules/semver": {
       "version": "7.5.4",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
@@ -3280,11 +3152,6 @@
         "node": ">=14.0"
       }
     },
-    "node_modules/unhomoglyph": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/unhomoglyph/-/unhomoglyph-1.0.6.tgz",
-      "integrity": "sha512-7uvcWI3hWshSADBu4JpnyYbTVc7YlhF5GDW/oPD5AxIxl34k4wXR3WDkPnzLxkN32LiTCTKMQLtKVZiwki3zGg=="
-    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -3306,14 +3173,6 @@
         "node": ">= 0.4.0"
       }
     },
-    "node_modules/uuid": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
-      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
-      "bin": {
-        "uuid": "dist/bin/uuid"
-      }
-    },
     "node_modules/v8-to-istanbul": {
       "version": "9.1.0",
       "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.0.tgz",

package.json

@@ -23,7 +23,6 @@
     "heatsync": "^2.4.1",
     "js-yaml": "^4.1.0",
     "matrix-appservice": "^2.0.0",
-    "matrix-js-sdk": "^24.1.0",
     "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0",
     "node-fetch": "^2.6.7",
     "prettier-bytes": "^1.0.4",

readme.md

@@ -0,0 +1,112 @@
+# Out Of Your Element
+
+Modern Matrix-to-Discord appservice bridge.
+
+## Why a new bridge?
+
+* Modern: Supports new Discord features like replies, threads and stickers, and new Matrix features like edits, spaces and space membership.
+* Reliable: Any errors on either side are notified on Matrix and can be retried.
+* Tested: A test suite and code coverage make sure all the core logic works.
+* Simple development: No build step (it's JavaScript, not TypeScript), minimal/lightweight dependencies, and abstraction only where necessary so that less background knowledge is required. No need to learn about Intents or library functions.
+
+## What works?
+
+Most things you'd expect:
+
+* Messages
+* Edits
+* Deletions
+* Reactions
+* Replies
+
+## Caveats
+
+* Custom emojis don't fully work yet.
+* Embeds don't work yet.
+* Some aspects of this bridge are customised for my homeserver. I'm working over time to make it more general. Please please reach out to @cadence:cadence.moe if you would like to run this, and I'll work with you to get it running!
+
+# Development information
+
+## You will need
+
+* Discord bot
+* Access to the homeserver's configuration
+* (For now) Help and support from @cadence:cadence.moe. Message me and tell me you're interested in OOYE!
+* The L1 and L2 emojis
+
+## Initial setup
+
+Node.js version 18 or later is required: https://nodejs.org/en/download/releases (the matrix-appservice dependency demands 18)
+
+Install dependencies: `npm install --save-dev`
+
+Copy `config.example.js` to `config.js` and fill in Discord token.
+
+Copy `registration.example.yaml` to `registration.yaml` and fill in bracketed values. Register it in Synapse's `homeserver.yaml` through the usual appservice installation process, then restart Synapse.
+
+If developing on a different computer to the one running the homeserver, use SSH port forwarding so that Synapse can connect on its `localhost:6693` to reach the running bridge on your computer. Example: `ssh -T -v -R 6693:localhost:6693 username@matrix.cadence.moe`
+
+Make sure the tests work: `npm t`
+
+I recommend developing in Visual Studio Code so that the JSDoc x TypeScript annotations work. I don't know what other editors or language servers support annotations and type inference.
+
+## Repository structure
+
+    .
+    * Entrypoint:
+    ├── index.js
+    * Runtime configuration, like tokens and user info:
+    ├── config.js
+    ├── registration.yaml
+    * The bridge's SQLite database is stored here:
+    ├── db
+    │   └── *.sql, *.db
+    * Discord-to-Matrix bridging:
+    ├── d2m
+    │   * Execute actions through the whole flow, like sending a Discord message to Matrix:
+    │   ├── actions
+    │   │   └── *.js
+    │   * Convert data from one form to another without depending on bridge state. Called by actions:
+    │   ├── converters
+    │   │   └── *.js
+    │   * Making Discord work:
+    │   ├── discord-*.js
+    │   * Listening to events from Discord and dispatching them to the correct `action`:
+    │   └── event-dispatcher.js
+    * Matrix-to-Discord bridging:
+    ├── m2d
+    │   * Execute actions through the whole flow, like sending a Matrix message to Discord:
+    │   ├── actions
+    │   │   ├── *.js
+    │   ├── converters
+    │   │   └── *.js
+    │   └── event-dispatcher.js
+    * We aren't using the matrix-js-sdk, so here's all the stuff we need to call the Matrix CS API:
+    ├── matrix
+    │   └── *.js
+    * Various files you can run once if you need them. Hopefully you won't need them.
+    ├── scripts
+    │   └── *.js
+    * First time running a new bridge? Run this file to plant a seed, which will flourish into state for the bridge:
+    ├── seed.js
+    * You are here :)
+    └── readme.md
+
+## Dependency justification
+
+(transitive dependency count) dependency name: explanation
+
+* (0) @chriscdn/promise-semaphore
+* (50) better-sqlite3: SQLite3 is the best database, and this is the best library for it. Really! I love it.
+* (1) chunk-text: It does what I want.
+* (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
+* (8) snowtransfer: Discord API library with bring-your-own-caching that I trust.
+* (1) discord-markdown: This is my fork! I make sure it does what I want.
+* (1) heatsync: Module hot-reloader that I trust.
+* (1) js-yaml: It seems to do what I want, and it's already pulled in by matrix-appservice.
+* (115) matrix-appservice: I wish it didn't pull in express :(
+* (0) mixin-deep: This is my fork! It fixes a bug in regular mixin-deep.
+* (3) node-fetch@2: I like it and it does what I want.
+* (0) prettier-bytes: It does what I want and has no dependencies.
+* (0) try-to-catch: Not strictly necessary, but it does what I want and has no dependencies.
+* (1) turndown: I need an HTML-to-Markdown converter and this one looked suitable enough. It has some bugs that I've worked around, so I might switch away from it later.

registration.example.yaml

@@ -0,0 +1,19 @@
+id: de8c56117637cb5d9f4ac216f612dc2adb1de4c09ae8d13553f28c33a28147c7
+hs_token: [a unique 64 character hex string]
+as_token: [a unique 64 character hex string]
+url: http://localhost:6693
+sender_localpart: _ooye_bot
+protocols:
+  - discord
+namespaces:
+  users:
+    - exclusive: true
+      regex: '@_ooye_.*'
+  aliases:
+    - exclusive: true
+      regex: '#_ooye_.*'
+rate_limited: false
+ooye:
+  namespace_prefix: _ooye_
+  max_file_size: 5000000
+  server_name: [the part after the colon in your matrix id, like cadence.moe]