From c00d518f6ed068cafb59930cd8f4507663cf20cf Mon Sep 17 00:00:00 2001 From: samhza Date: Sat, 26 Nov 2022 16:31:00 -0500 Subject: [PATCH] simplify postgres and sqlite database initialization, remove psqlinit.sh (#331) Before this change, uninitialized databases would be initialized with the old schema and then migrated to the latest version. After this change, unintialized databases are initizialized with the latest database schema immediately, without having to run any of the migrations. This change has no effect on existing databases. Before this change, Postgres database initialization was done manually using utils/psqlinit.sh. This is inconsistent with SQLite, which the bot initializes itself. It also requires shell access to the server running the Postgres instance, which means it cannot be used on managed Postgres instances. After this change, the bot initializes Postgres databases as it does with SQLite, and utils/psqlinit.sh has been removed as it is now unecessary. --- docker-compose.yml | 1 - docs/postgresql.md | 6 +-- utils/database/postgresql.js | 83 ++++++++++++++++++++++++++---------- utils/database/sqlite.js | 79 ++++++++++++++++++++++++---------- utils/psqlinit.sh | 11 ----- 5 files changed, 118 insertions(+), 62 deletions(-) delete mode 100644 utils/psqlinit.sh diff --git a/docker-compose.yml b/docker-compose.yml index f85fd07..93fa2c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,7 +52,6 @@ services: restart: unless-stopped volumes: - pg-data:/var/lib/postgresql/data - - ./utils/psqlinit.sh:/docker-entrypoint-initdb.d/psqlinit.sh environment: POSTGRES_PASSWORD: verycoolpass100 POSTGRES_USER: esmbot diff --git a/docs/postgresql.md b/docs/postgresql.md index d06d8fb..7868d7f 100644 --- a/docs/postgresql.md +++ b/docs/postgresql.md @@ -49,11 +49,7 @@ Once you're inside the shell, you'll need to make sure the bot owns the database ```sql ALTER DATABASE esmbot OWNER TO esmbot; ``` -The database is now accessible by the bot, but the bot may not function yet as the tables to add/get data are still missing. To fix that, you'll need to add them. Luckily, the bot comes with a script to automate this. First, exit the PostgreSQL shell by typing `\q`, then make the script executable and run it by entering the following commands: -```sh -chmod +x utils/psqlinit.sh -POSTGRES_USER=esmbot POSTGRES_DB=esmbot utils/psqlinit.sh -``` + You're done! *** diff --git a/utils/database/postgresql.js b/utils/database/postgresql.js index 7a0426b..cf64ce0 100644 --- a/utils/database/postgresql.js +++ b/utils/database/postgresql.js @@ -2,9 +2,39 @@ import { prefixCache, disabledCmdCache, disabledCache, commands, messageCommands import * as logger from "../logger.js"; import Postgres from "postgres"; -const sql = Postgres(process.env.DB); +const sql = Postgres(process.env.DB, { + onnotice: () => {} +}); -const psqlUpdates = [ +const settingsSchema = ` +CREATE TABLE IF NOT EXISTS settings ( + id smallint PRIMARY KEY, + version integer NOT NULL, CHECK(id = 1) +); +`; + +const schema = ` +ALTER TABLE settings ADD COLUMN broadcast text; +CREATE TABLE guilds ( + guild_id VARCHAR(30) NOT NULL PRIMARY KEY, + prefix VARCHAR(15) NOT NULL, + disabled text ARRAY NOT NULL, + disabled_commands text ARRAY NOT NULL +); +CREATE TABLE counts ( + command VARCHAR NOT NULL PRIMARY KEY, + count integer NOT NULL +); +CREATE TABLE tags ( + guild_id VARCHAR(30) NOT NULL, + name text NOT NULL, + content text NOT NULL, + author VARCHAR(30) NOT NULL, + UNIQUE(guild_id, name) +); +`; + +const updates = [ "", // reserved "CREATE TABLE IF NOT EXISTS settings ( id smallint PRIMARY KEY, version integer NOT NULL, CHECK(id = 1) );\nALTER TABLE guilds ADD COLUMN accessed timestamp;", "ALTER TABLE guilds DROP COLUMN accessed", @@ -44,29 +74,38 @@ export async function setup() { } export async function upgrade(logger) { - let version; try { - version = (await sql`SELECT version FROM settings WHERE id = 1`)[0].version; - } catch { - version = 0; - } - if (version < (psqlUpdates.length - 1)) { - logger.warn(`Migrating PostgreSQL database, which is currently at version ${version}...`); - try { - await sql.begin(async (db) => { - while (version < (psqlUpdates.length - 1)) { + await sql.begin(async (tx) => { + await tx.unsafe(settingsSchema); + let version; + const settingsrow = (await tx`SELECT version FROM settings WHERE id = 1`); + if (settingsrow.length == 0) { + version = 0; + } else { + version = settingsrow[0].version; + }; + const latestVersion = updates.length - 1; + if (version === 0) { + logger.info(`Initializing PostgreSQL database...`); + await tx.unsafe(schema); + } else if (version < latestVersion) { + logger.info(`Migrating PostgreSQL database, which is currently at version ${version}...`); + while (version < latestVersion) { version++; - logger.warn(`Running version ${version} update script (${psqlUpdates[version]})...`); - await db.unsafe(psqlUpdates[version]); + logger.info(`Running version ${version} update script...`); + await tx.unsafe(updates[version]); } - }); - const ver = psqlUpdates.length - 1; - await sql`INSERT INTO settings ${sql({ id: 1, version: ver })} ON CONFLICT (id) DO UPDATE SET version = ${ver}`; - } catch (e) { - logger.error(`PostgreSQL migration failed: ${e}`); - logger.error("Unable to start the bot, quitting now."); - return 1; - } + } else if (version > latestVersion) { + throw new Error(`PostgreSQL database is at version ${version}, but this version of the bot only supports up to version ${latestVersion}.`); + } else { + return; + } + await tx`INSERT INTO settings ${sql({ id: 1, version: latestVersion })} ON CONFLICT (id) DO UPDATE SET version = ${latestVersion}`; + }); + } catch (e) { + logger.error(`PostgreSQL migration failed: ${e}`); + logger.error("Unable to start the bot, quitting now."); + return 1; } } diff --git a/utils/database/sqlite.js b/utils/database/sqlite.js index 5ec7d85..4ec6826 100644 --- a/utils/database/sqlite.js +++ b/utils/database/sqlite.js @@ -4,11 +4,42 @@ import * as logger from "../logger.js"; import sqlite3 from "better-sqlite3"; const connection = sqlite3(process.env.DB.replace("sqlite://", "")); -const sqliteUpdates = [ +const schema = ` +CREATE TABLE guilds ( + guild_id VARCHAR(30) NOT NULL PRIMARY KEY, + prefix VARCHAR(15) NOT NULL, + disabled text NOT NULL, + disabled_commands text NOT NULL +); +CREATE TABLE counts ( + command VARCHAR NOT NULL PRIMARY KEY, + count integer NOT NULL +); +CREATE TABLE tags ( + guild_id VARCHAR(30) NOT NULL, + name text NOT NULL, + content text NOT NULL, + author VARCHAR(30) NOT NULL, + UNIQUE(guild_id, name) +); +CREATE TABLE settings ( + id smallint PRIMARY KEY, + broadcast VARCHAR, + CHECK(id = 1) +); +INSERT INTO settings (id) VALUES (1); +`; + +const updates = [ "", // reserved "ALTER TABLE guilds ADD COLUMN accessed int", "ALTER TABLE guilds DROP COLUMN accessed", - "CREATE TABLE IF NOT EXISTS settings ( id smallint PRIMARY KEY, broadcast VARCHAR, CHECK(id = 1) );\nINSERT INTO settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING;" + `CREATE TABLE settings ( + id smallint PRIMARY KEY, + broadcast VARCHAR, + CHECK(id = 1) + ); + INSERT INTO settings (id) VALUES (1);`, ]; export async function setup() { @@ -42,31 +73,33 @@ export async function stop() { } export async function upgrade(logger) { - connection.prepare("CREATE TABLE IF NOT EXISTS guilds ( guild_id VARCHAR(30) NOT NULL PRIMARY KEY, prefix VARCHAR(15) NOT NULL, disabled text NOT NULL, disabled_commands text NOT NULL )").run(); - connection.prepare("CREATE TABLE IF NOT EXISTS counts ( command VARCHAR NOT NULL PRIMARY KEY, count integer NOT NULL )").run(); - connection.prepare("CREATE TABLE IF NOT EXISTS tags ( guild_id VARCHAR(30) NOT NULL, name text NOT NULL, content text NOT NULL, author VARCHAR(30) NOT NULL, UNIQUE(guild_id, name) )").run(); - - let version = connection.pragma("user_version", { simple: true }); - if (version < (sqliteUpdates.length - 1)) { - logger.warn(`Migrating SQLite database at ${process.env.DB}, which is currently at version ${version}...`); - connection.prepare("BEGIN TRANSACTION").run(); - try { - while (version < (sqliteUpdates.length - 1)) { + connection.exec("BEGIN TRANSACTION"); + try { + let version = connection.pragma("user_version", { simple: true }); + const latestVersion = updates.length - 1; + if (version == 0) { + logger.info(`Initializing SQLite database...`); + connection.exec(schema); + } else if (version < latestVersion) { + logger.info(`Migrating SQLite database at ${process.env.DB}, which is currently at version ${version}...`); + while (version < latestVersion) { version++; - logger.warn(`Running version ${version} update script (${sqliteUpdates[version]})...`); - for (const statement of sqliteUpdates[version].split("\n")) { - connection.prepare(statement).run(); - } + logger.info(`Running version ${version} update script...`); + connection.exec(updates[version]); } - connection.pragma(`user_version = ${version}`); // insecure, but the normal templating method doesn't seem to work here - connection.prepare("COMMIT").run(); - } catch (e) { - logger.error(`SQLite migration failed: ${e}`); - connection.prepare("ROLLBACK").run(); - logger.error("Unable to start the bot, quitting now."); - return 1; + } else if (version > latestVersion) { + throw new Error(`SQLite database is at version ${version}, but this version of the bot only supports up to version ${latestVersion}.`); + } else { + return; } + connection.pragma(`user_version = ${latestVersion}`); // prepared statements don't seem to work here + } catch (e) { + logger.error(`SQLite migration failed: ${e}`); + connection.exec("ROLLBACK"); + logger.error("Unable to start the bot, quitting now."); + return 1; } + connection.exec("COMMIT"); } export async function fixGuild(guild) { diff --git a/utils/psqlinit.sh b/utils/psqlinit.sh deleted file mode 100644 index cdbe63f..0000000 --- a/utils/psqlinit.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE TABLE guilds ( guild_id VARCHAR(30) NOT NULL PRIMARY KEY, prefix VARCHAR(15) NOT NULL, disabled text ARRAY NOT NULL, disabled_commands text ARRAY NOT NULL ); - CREATE TABLE counts ( command VARCHAR NOT NULL PRIMARY KEY, count integer NOT NULL ); - CREATE TABLE tags ( guild_id VARCHAR(30) NOT NULL, name text NOT NULL, content text NOT NULL, author VARCHAR(30) NOT NULL, UNIQUE(guild_id, name) ); - - CREATE TABLE settings ( id smallint PRIMARY KEY, version integer NOT NULL, broadcast VARCHAR, CHECK(id = 1) ); - INSERT INTO settings (id, version) VALUES (1, 2) ON CONFLICT (id) DO NOTHING; -EOSQL