simplify postgres and sqlite database initialization, remove psqlinit.sh (#331)

Before this change, uninitialized databases would be initialized with the old
schema and then migrated to the latest version. After this change, unintialized
databases are initizialized with the latest database schema immediately, without
having to run any of the migrations. This change has no effect on existing
databases.

Before this change, Postgres database initialization was done manually using
utils/psqlinit.sh. This is inconsistent with SQLite, which the bot initializes
itself. It also requires shell access to the server running the Postgres
instance, which means it cannot be used on managed Postgres instances. After
this change, the bot initializes Postgres databases as it does with SQLite, and
utils/psqlinit.sh has been removed as it is now unecessary.

docker-compose.yml

@@ -52,7 +52,6 @@ services:
     restart: unless-stopped
     volumes:
       - pg-data:/var/lib/postgresql/data
-      - ./utils/psqlinit.sh:/docker-entrypoint-initdb.d/psqlinit.sh
     environment:
       POSTGRES_PASSWORD: verycoolpass100
       POSTGRES_USER: esmbot

docs/postgresql.md

@@ -49,11 +49,7 @@ Once you're inside the shell, you'll need to make sure the bot owns the database
 ```sql
 ALTER DATABASE esmbot OWNER TO esmbot;
 ```
-The database is now accessible by the bot, but the bot may not function yet as the tables to add/get data are still missing. To fix that, you'll need to add them. Luckily, the bot comes with a script to automate this. First, exit the PostgreSQL shell by typing `\q`, then make the script executable and run it by entering the following commands:
-```sh
-chmod +x utils/psqlinit.sh
-POSTGRES_USER=esmbot POSTGRES_DB=esmbot utils/psqlinit.sh
-```
+
 You're done!
 
 ***

utils/database/postgresql.js

@@ -2,9 +2,39 @@ import { prefixCache, disabledCmdCache, disabledCache, commands, messageCommands
 import * as logger from "../logger.js";
 
 import Postgres from "postgres";
-const sql = Postgres(process.env.DB);
+const sql = Postgres(process.env.DB, {
+  onnotice: () => {}
+});
 
-const psqlUpdates = [
+const settingsSchema = `
+CREATE TABLE IF NOT EXISTS settings (
+  id smallint PRIMARY KEY,
+  version integer NOT NULL, CHECK(id = 1)
+);
+`;
+
+const schema = `
+ALTER TABLE settings ADD COLUMN broadcast text;
+CREATE TABLE guilds (
+  guild_id VARCHAR(30) NOT NULL PRIMARY KEY,
+  prefix VARCHAR(15) NOT NULL,
+  disabled text ARRAY NOT NULL,
+  disabled_commands text ARRAY NOT NULL
+);
+CREATE TABLE counts (
+  command VARCHAR NOT NULL PRIMARY KEY,
+  count integer NOT NULL
+);
+CREATE TABLE tags (
+  guild_id VARCHAR(30) NOT NULL,
+  name text NOT NULL,
+  content text NOT NULL,
+  author VARCHAR(30) NOT NULL,
+  UNIQUE(guild_id, name)
+);
+`;
+
+const updates = [
   "", // reserved
   "CREATE TABLE IF NOT EXISTS settings ( id smallint PRIMARY KEY, version integer NOT NULL, CHECK(id = 1) );\nALTER TABLE guilds ADD COLUMN accessed timestamp;",
   "ALTER TABLE guilds DROP COLUMN accessed",
@@ -44,31 +74,40 @@ export async function setup() {
 }
 
 export async function upgrade(logger) {
+  try {
+    await sql.begin(async (tx) => {
+      await tx.unsafe(settingsSchema);
       let version;
-  try {
-    version = (await sql`SELECT version FROM settings WHERE id = 1`)[0].version;
-  } catch {
+      const settingsrow = (await tx`SELECT version FROM settings WHERE id = 1`);
+      if (settingsrow.length == 0) {
         version = 0;
-  }
-  if (version < (psqlUpdates.length - 1)) {
-    logger.warn(`Migrating PostgreSQL database, which is currently at version ${version}...`);
-    try {
-      await sql.begin(async (db) => {
-        while (version < (psqlUpdates.length - 1)) {
+      } else {
+        version = settingsrow[0].version;
+      };
+      const latestVersion = updates.length - 1;
+      if (version === 0) {
+        logger.info(`Initializing PostgreSQL database...`);
+        await tx.unsafe(schema);
+      } else if (version < latestVersion) {
+        logger.info(`Migrating PostgreSQL database, which is currently at version ${version}...`);
+        while (version < latestVersion) {
           version++;
-          logger.warn(`Running version ${version} update script (${psqlUpdates[version]})...`);
-          await db.unsafe(psqlUpdates[version]);
+          logger.info(`Running version ${version} update script...`);
+          await tx.unsafe(updates[version]);
         }
+      } else if (version > latestVersion) {
+        throw new Error(`PostgreSQL database is at version ${version}, but this version of the bot only supports up to version ${latestVersion}.`);
+      } else {
+        return;
+      }
+      await tx`INSERT INTO settings ${sql({ id: 1, version: latestVersion })} ON CONFLICT (id) DO UPDATE SET version = ${latestVersion}`;
     });
-      const ver = psqlUpdates.length - 1;
-      await sql`INSERT INTO settings ${sql({ id: 1, version: ver })} ON CONFLICT (id) DO UPDATE SET version = ${ver}`;
   } catch (e) {
     logger.error(`PostgreSQL migration failed: ${e}`);
     logger.error("Unable to start the bot, quitting now.");
     return 1;
   }
 }
-}
 
 export async function getGuild(query) {
   return (await sql`SELECT * FROM guilds WHERE guild_id = ${query}`)[0];

utils/database/sqlite.js

@@ -4,11 +4,42 @@ import * as logger from "../logger.js";
 import sqlite3 from "better-sqlite3";
 const connection = sqlite3(process.env.DB.replace("sqlite://", ""));
 
-const sqliteUpdates = [
+const schema = `
+CREATE TABLE guilds (
+  guild_id VARCHAR(30) NOT NULL PRIMARY KEY,
+  prefix VARCHAR(15) NOT NULL,
+  disabled text NOT NULL,
+  disabled_commands text NOT NULL
+);
+CREATE TABLE counts (
+  command VARCHAR NOT NULL PRIMARY KEY,
+  count integer NOT NULL
+);
+CREATE TABLE tags (
+  guild_id VARCHAR(30) NOT NULL,
+  name text NOT NULL,
+  content text NOT NULL,
+  author VARCHAR(30) NOT NULL,
+  UNIQUE(guild_id, name)
+);
+CREATE TABLE settings (
+  id smallint PRIMARY KEY,
+  broadcast VARCHAR,
+  CHECK(id = 1)
+);
+INSERT INTO settings (id) VALUES (1);
+`;
+
+const updates = [
   "", // reserved
   "ALTER TABLE guilds ADD COLUMN accessed int",
   "ALTER TABLE guilds DROP COLUMN accessed",
-  "CREATE TABLE IF NOT EXISTS settings ( id smallint PRIMARY KEY, broadcast VARCHAR, CHECK(id = 1) );\nINSERT INTO settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING;"
+  `CREATE TABLE settings (
+    id smallint PRIMARY KEY,
+    broadcast VARCHAR,
+    CHECK(id = 1)
+  );
+  INSERT INTO settings (id) VALUES (1);`,
 ];
 
 export async function setup() {
@@ -42,31 +73,33 @@ export async function stop() {
 }
 
 export async function upgrade(logger) {
-  connection.prepare("CREATE TABLE IF NOT EXISTS guilds ( guild_id VARCHAR(30) NOT NULL PRIMARY KEY, prefix VARCHAR(15) NOT NULL, disabled text NOT NULL, disabled_commands text NOT NULL )").run();
-  connection.prepare("CREATE TABLE IF NOT EXISTS counts ( command VARCHAR NOT NULL PRIMARY KEY, count integer NOT NULL )").run();
-  connection.prepare("CREATE TABLE IF NOT EXISTS tags ( guild_id VARCHAR(30) NOT NULL, name text NOT NULL, content text NOT NULL, author VARCHAR(30) NOT NULL, UNIQUE(guild_id, name) )").run();
-
-  let version = connection.pragma("user_version", { simple: true });
-  if (version < (sqliteUpdates.length - 1)) {
-    logger.warn(`Migrating SQLite database at ${process.env.DB}, which is currently at version ${version}...`);
-    connection.prepare("BEGIN TRANSACTION").run();
+  connection.exec("BEGIN TRANSACTION");
   try {
-      while (version < (sqliteUpdates.length - 1)) {
+    let version = connection.pragma("user_version", { simple: true });
+    const latestVersion = updates.length - 1;
+    if (version == 0) {
+      logger.info(`Initializing SQLite database...`);
+      connection.exec(schema);
+    } else if (version < latestVersion) {
+      logger.info(`Migrating SQLite database at ${process.env.DB}, which is currently at version ${version}...`);
+      while (version < latestVersion) {
         version++;
-        logger.warn(`Running version ${version} update script (${sqliteUpdates[version]})...`);
-        for (const statement of sqliteUpdates[version].split("\n")) {
-          connection.prepare(statement).run();
+        logger.info(`Running version ${version} update script...`);
+        connection.exec(updates[version]);
       }
+    } else if (version > latestVersion) {
+      throw new Error(`SQLite database is at version ${version}, but this version of the bot only supports up to version ${latestVersion}.`);
+    } else {
+      return;
     }
-      connection.pragma(`user_version = ${version}`); // insecure, but the normal templating method doesn't seem to work here
-      connection.prepare("COMMIT").run();
+    connection.pragma(`user_version = ${latestVersion}`); // prepared statements don't seem to work here
   } catch (e) {
     logger.error(`SQLite migration failed: ${e}`);
-      connection.prepare("ROLLBACK").run();
+    connection.exec("ROLLBACK");
     logger.error("Unable to start the bot, quitting now.");
     return 1;
   }
-  }
+  connection.exec("COMMIT");
 }
 
 export async function fixGuild(guild) {

utils/psqlinit.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-set -e
-
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    CREATE TABLE guilds ( guild_id VARCHAR(30) NOT NULL PRIMARY KEY, prefix VARCHAR(15) NOT NULL, disabled text ARRAY NOT NULL, disabled_commands text ARRAY NOT NULL );
-    CREATE TABLE counts ( command VARCHAR NOT NULL PRIMARY KEY, count integer NOT NULL );
-    CREATE TABLE tags ( guild_id VARCHAR(30) NOT NULL, name text NOT NULL, content text NOT NULL, author VARCHAR(30) NOT NULL, UNIQUE(guild_id, name) );
-
-    CREATE TABLE settings ( id smallint PRIMARY KEY, version integer NOT NULL, broadcast VARCHAR, CHECK(id = 1) );
-    INSERT INTO settings (id, version) VALUES (1, 2) ON CONFLICT (id) DO NOTHING;
-EOSQL