From 01af302796a378faba0b24fab0a75920ccf1da48 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 15 Sep 2024 01:09:47 +1200 Subject: [PATCH] Move ooye.db to current working dir --- .gitignore | 2 +- package.json | 1 + readme.md | 17 +++++++++-------- scripts/check-migrate.js | 2 +- scripts/migrate-from-old-bridge.js | 2 +- scripts/save-channel-names-to-db.js | 2 +- scripts/save-event-types-to-db.js | 2 +- scripts/seed.js | 6 +++--- scripts/wal.js | 2 +- start.js | 2 +- 10 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 3781b9b..a462935 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,6 @@ node_modules config.js registration.yaml coverage -src/db/ooye.db* +ooye.db* test/res/* !test/res/lottie* diff --git a/package.json b/package.json index 69101c8..5933ba9 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "supertape": "^10.4.0" }, "scripts": { + "start": "node start.js", "addbot": "node addbot.js", "test": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap test/test.js | tap-dot", "test-slow": "cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap --no-worker test/test.js -- --slow | tap-dot", diff --git a/readme.md b/readme.md index 695c26a..d9ade3c 100644 --- a/readme.md +++ b/readme.md @@ -55,7 +55,7 @@ For more information about features, [see the user guide.](https://gitdab.com/ca Using WeatherStack as a thin layer between the bridge application and the Discord API lets us control exactly what data is cached in memory. Only necessary information is cached. For example, member data, user data, message content, and past edits are never stored in memory. This keeps the memory usage low and also prevents it ballooning in size over the bridge's runtime. -The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 100,000 messages require 16.1 MB of storage space in the SQLite database. +The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs and URLs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 300,000 messages (representing a year of conversations) requires 47.3 MB of storage space in the SQLite database. Only necessary data and columns are queried from the database. We only contact the homeserver API if the database doesn't contain what we need. @@ -83,7 +83,7 @@ Follow these steps: 1. Run `node scripts/seed.js` to check your setup and set the bot's initial state. It will prompt you for information. You only need to run this once ever. -1. Start the bridge: `node start.js` +1. Start the bridge: `npm run start` 1. Add the bot to a server - use any *one* of the following commands for an invite link: * (in the REPL) `addbot` @@ -106,18 +106,19 @@ To get into the rooms on your Matrix account, either add yourself to `invite` in ## Repository structure . - * Run this to start the bridge: - ├── start.js * Runtime configuration, like tokens and user info: ├── registration.yaml * You are here! :) - └── readme.md + ├── readme.md + * The bridge's SQLite database is stored here: + ├── ooye.db* * Source code └── src - * The bridge's SQLite database is stored here: + * Database schema: ├── db - │   ├── *.sql, *.db - │   * Migrations change the database schema when you update to a newer version of OOYE: + │   ├── orm.js, orm-defs.d.ts + │   * Migrations change the database schema when you update to a newer version of OOYE: + │   ├── migrate.js │   └── migrations │       └── *.sql, *.js * Discord-to-Matrix bridging: diff --git a/scripts/check-migrate.js b/scripts/check-migrate.js index 82f2832..04a4402 100755 --- a/scripts/check-migrate.js +++ b/scripts/check-migrate.js @@ -7,7 +7,7 @@ const sqlite = require("better-sqlite3") const passthrough = require("../src/passthrough") -const db = new sqlite("db/ooye.db") +const db = new sqlite("ooye.db") const migrate = require("../src/db/migrate") Object.assign(passthrough, {db}) diff --git a/scripts/migrate-from-old-bridge.js b/scripts/migrate-from-old-bridge.js index c47bb05..c5e50a1 100755 --- a/scripts/migrate-from-old-bridge.js +++ b/scripts/migrate-from-old-bridge.js @@ -17,7 +17,7 @@ const oldAT = reg.old_bridge.as_token const newAT = reg.as_token const oldDB = new sqlite(reg.old_bridge.database) -const db = new sqlite("db/ooye.db") +const db = new sqlite("ooye.db") db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration ( discord_channel TEXT NOT NULL, diff --git a/scripts/save-channel-names-to-db.js b/scripts/save-channel-names-to-db.js index 7a72949..1f36a73 100755 --- a/scripts/save-channel-names-to-db.js +++ b/scripts/save-channel-names-to-db.js @@ -6,7 +6,7 @@ const HeatSync = require("heatsync") const {reg} = require("../src/matrix/read-registration") const passthrough = require("../src/passthrough") -const db = new sqlite("db/ooye.db") +const db = new sqlite("ooye.db") const sync = new HeatSync({watchFS: false}) diff --git a/scripts/save-event-types-to-db.js b/scripts/save-event-types-to-db.js index db14e07..edfbb9c 100755 --- a/scripts/save-event-types-to-db.js +++ b/scripts/save-event-types-to-db.js @@ -5,7 +5,7 @@ const sqlite = require("better-sqlite3") const HeatSync = require("heatsync") const passthrough = require("../src/passthrough") -const db = new sqlite("db/ooye.db") +const db = new sqlite("ooye.db") const sync = new HeatSync({watchFS: false}) diff --git a/scripts/seed.js b/scripts/seed.js index 991eadc..16be900 100755 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -21,11 +21,11 @@ const args = require("minimist")(process.argv.slice(2), {string: ["emoji-guild"] // Move database file if it's still in the old location if (fs.existsSync("db")) { if (fs.existsSync("db/ooye.db")) { - fs.renameSync("db/ooye.db", "src/db/ooye.db") + fs.renameSync("db/ooye.db", "ooye.db") } const files = fs.readdirSync("db") if (files.length) { - console.error("You must manually move or delete the files in the db folder:") + console.error("The db folder is deprecated and must be removed. Your ooye.db database file has already been moved to the root of the repo. You must manually move or delete the remaining files:") for (const file of files) { console.error(file) } @@ -35,7 +35,7 @@ if (fs.existsSync("db")) { } const passthrough = require("../src/passthrough") -const db = new sqlite("src/db/ooye.db") +const db = new sqlite("ooye.db") const migrate = require("../src/db/migrate") const sync = new HeatSync({watchFS: false}) diff --git a/scripts/wal.js b/scripts/wal.js index 352ac9d..625f2ba 100755 --- a/scripts/wal.js +++ b/scripts/wal.js @@ -2,6 +2,6 @@ // @ts-check const sqlite = require("better-sqlite3") -const db = new sqlite("db/ooye.db", {fileMustExist: true}) +const db = new sqlite("ooye.db", {fileMustExist: true}) db.pragma("journal_mode = wal") db.close() diff --git a/start.js b/start.js index caa01c2..be434f0 100755 --- a/start.js +++ b/start.js @@ -7,7 +7,7 @@ const HeatSync = require("heatsync") const {reg} = require("./src/matrix/read-registration") const passthrough = require("./src/passthrough") -const db = new sqlite("src/db/ooye.db") +const db = new sqlite("ooye.db") /** @type {import("heatsync").default} */ // @ts-ignore const sync = new HeatSync()