simplify postgres and sqlite database initialization, remove psqlinit.sh (#331)

Before this change, uninitialized databases would be initialized with the old
schema and then migrated to the latest version. After this change, unintialized
databases are initizialized with the latest database schema immediately, without
having to run any of the migrations. This change has no effect on existing
databases.

Before this change, Postgres database initialization was done manually using
utils/psqlinit.sh. This is inconsistent with SQLite, which the bot initializes
itself. It also requires shell access to the server running the Postgres
instance, which means it cannot be used on managed Postgres instances. After
this change, the bot initializes Postgres databases as it does with SQLite, and
utils/psqlinit.sh has been removed as it is now unecessary.
This commit is contained in:
samhza 2022-11-26 16:31:00 -05:00 committed by GitHub
parent 1830db6282
commit c00d518f6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 118 additions and 62 deletions

View file

@ -52,7 +52,6 @@ services:
restart: unless-stopped restart: unless-stopped
volumes: volumes:
- pg-data:/var/lib/postgresql/data - pg-data:/var/lib/postgresql/data
- ./utils/psqlinit.sh:/docker-entrypoint-initdb.d/psqlinit.sh
environment: environment:
POSTGRES_PASSWORD: verycoolpass100 POSTGRES_PASSWORD: verycoolpass100
POSTGRES_USER: esmbot POSTGRES_USER: esmbot

View file

@ -49,11 +49,7 @@ Once you're inside the shell, you'll need to make sure the bot owns the database
```sql ```sql
ALTER DATABASE esmbot OWNER TO esmbot; ALTER DATABASE esmbot OWNER TO esmbot;
``` ```
The database is now accessible by the bot, but the bot may not function yet as the tables to add/get data are still missing. To fix that, you'll need to add them. Luckily, the bot comes with a script to automate this. First, exit the PostgreSQL shell by typing `\q`, then make the script executable and run it by entering the following commands:
```sh
chmod +x utils/psqlinit.sh
POSTGRES_USER=esmbot POSTGRES_DB=esmbot utils/psqlinit.sh
```
You're done! You're done!
*** ***

View file

@ -2,9 +2,39 @@ import { prefixCache, disabledCmdCache, disabledCache, commands, messageCommands
import * as logger from "../logger.js"; import * as logger from "../logger.js";
import Postgres from "postgres"; import Postgres from "postgres";
const sql = Postgres(process.env.DB); const sql = Postgres(process.env.DB, {
onnotice: () => {}
});
const psqlUpdates = [ const settingsSchema = `
CREATE TABLE IF NOT EXISTS settings (
id smallint PRIMARY KEY,
version integer NOT NULL, CHECK(id = 1)
);
`;
const schema = `
ALTER TABLE settings ADD COLUMN broadcast text;
CREATE TABLE guilds (
guild_id VARCHAR(30) NOT NULL PRIMARY KEY,
prefix VARCHAR(15) NOT NULL,
disabled text ARRAY NOT NULL,
disabled_commands text ARRAY NOT NULL
);
CREATE TABLE counts (
command VARCHAR NOT NULL PRIMARY KEY,
count integer NOT NULL
);
CREATE TABLE tags (
guild_id VARCHAR(30) NOT NULL,
name text NOT NULL,
content text NOT NULL,
author VARCHAR(30) NOT NULL,
UNIQUE(guild_id, name)
);
`;
const updates = [
"", // reserved "", // reserved
"CREATE TABLE IF NOT EXISTS settings ( id smallint PRIMARY KEY, version integer NOT NULL, CHECK(id = 1) );\nALTER TABLE guilds ADD COLUMN accessed timestamp;", "CREATE TABLE IF NOT EXISTS settings ( id smallint PRIMARY KEY, version integer NOT NULL, CHECK(id = 1) );\nALTER TABLE guilds ADD COLUMN accessed timestamp;",
"ALTER TABLE guilds DROP COLUMN accessed", "ALTER TABLE guilds DROP COLUMN accessed",
@ -44,30 +74,39 @@ export async function setup() {
} }
export async function upgrade(logger) { export async function upgrade(logger) {
try {
await sql.begin(async (tx) => {
await tx.unsafe(settingsSchema);
let version; let version;
try { const settingsrow = (await tx`SELECT version FROM settings WHERE id = 1`);
version = (await sql`SELECT version FROM settings WHERE id = 1`)[0].version; if (settingsrow.length == 0) {
} catch {
version = 0; version = 0;
} } else {
if (version < (psqlUpdates.length - 1)) { version = settingsrow[0].version;
logger.warn(`Migrating PostgreSQL database, which is currently at version ${version}...`); };
try { const latestVersion = updates.length - 1;
await sql.begin(async (db) => { if (version === 0) {
while (version < (psqlUpdates.length - 1)) { logger.info(`Initializing PostgreSQL database...`);
await tx.unsafe(schema);
} else if (version < latestVersion) {
logger.info(`Migrating PostgreSQL database, which is currently at version ${version}...`);
while (version < latestVersion) {
version++; version++;
logger.warn(`Running version ${version} update script (${psqlUpdates[version]})...`); logger.info(`Running version ${version} update script...`);
await db.unsafe(psqlUpdates[version]); await tx.unsafe(updates[version]);
} }
} else if (version > latestVersion) {
throw new Error(`PostgreSQL database is at version ${version}, but this version of the bot only supports up to version ${latestVersion}.`);
} else {
return;
}
await tx`INSERT INTO settings ${sql({ id: 1, version: latestVersion })} ON CONFLICT (id) DO UPDATE SET version = ${latestVersion}`;
}); });
const ver = psqlUpdates.length - 1;
await sql`INSERT INTO settings ${sql({ id: 1, version: ver })} ON CONFLICT (id) DO UPDATE SET version = ${ver}`;
} catch (e) { } catch (e) {
logger.error(`PostgreSQL migration failed: ${e}`); logger.error(`PostgreSQL migration failed: ${e}`);
logger.error("Unable to start the bot, quitting now."); logger.error("Unable to start the bot, quitting now.");
return 1; return 1;
} }
}
} }
export async function getGuild(query) { export async function getGuild(query) {

View file

@ -4,11 +4,42 @@ import * as logger from "../logger.js";
import sqlite3 from "better-sqlite3"; import sqlite3 from "better-sqlite3";
const connection = sqlite3(process.env.DB.replace("sqlite://", "")); const connection = sqlite3(process.env.DB.replace("sqlite://", ""));
const sqliteUpdates = [ const schema = `
CREATE TABLE guilds (
guild_id VARCHAR(30) NOT NULL PRIMARY KEY,
prefix VARCHAR(15) NOT NULL,
disabled text NOT NULL,
disabled_commands text NOT NULL
);
CREATE TABLE counts (
command VARCHAR NOT NULL PRIMARY KEY,
count integer NOT NULL
);
CREATE TABLE tags (
guild_id VARCHAR(30) NOT NULL,
name text NOT NULL,
content text NOT NULL,
author VARCHAR(30) NOT NULL,
UNIQUE(guild_id, name)
);
CREATE TABLE settings (
id smallint PRIMARY KEY,
broadcast VARCHAR,
CHECK(id = 1)
);
INSERT INTO settings (id) VALUES (1);
`;
const updates = [
"", // reserved "", // reserved
"ALTER TABLE guilds ADD COLUMN accessed int", "ALTER TABLE guilds ADD COLUMN accessed int",
"ALTER TABLE guilds DROP COLUMN accessed", "ALTER TABLE guilds DROP COLUMN accessed",
"CREATE TABLE IF NOT EXISTS settings ( id smallint PRIMARY KEY, broadcast VARCHAR, CHECK(id = 1) );\nINSERT INTO settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING;" `CREATE TABLE settings (
id smallint PRIMARY KEY,
broadcast VARCHAR,
CHECK(id = 1)
);
INSERT INTO settings (id) VALUES (1);`,
]; ];
export async function setup() { export async function setup() {
@ -42,31 +73,33 @@ export async function stop() {
} }
export async function upgrade(logger) { export async function upgrade(logger) {
connection.prepare("CREATE TABLE IF NOT EXISTS guilds ( guild_id VARCHAR(30) NOT NULL PRIMARY KEY, prefix VARCHAR(15) NOT NULL, disabled text NOT NULL, disabled_commands text NOT NULL )").run(); connection.exec("BEGIN TRANSACTION");
connection.prepare("CREATE TABLE IF NOT EXISTS counts ( command VARCHAR NOT NULL PRIMARY KEY, count integer NOT NULL )").run();
connection.prepare("CREATE TABLE IF NOT EXISTS tags ( guild_id VARCHAR(30) NOT NULL, name text NOT NULL, content text NOT NULL, author VARCHAR(30) NOT NULL, UNIQUE(guild_id, name) )").run();
let version = connection.pragma("user_version", { simple: true });
if (version < (sqliteUpdates.length - 1)) {
logger.warn(`Migrating SQLite database at ${process.env.DB}, which is currently at version ${version}...`);
connection.prepare("BEGIN TRANSACTION").run();
try { try {
while (version < (sqliteUpdates.length - 1)) { let version = connection.pragma("user_version", { simple: true });
const latestVersion = updates.length - 1;
if (version == 0) {
logger.info(`Initializing SQLite database...`);
connection.exec(schema);
} else if (version < latestVersion) {
logger.info(`Migrating SQLite database at ${process.env.DB}, which is currently at version ${version}...`);
while (version < latestVersion) {
version++; version++;
logger.warn(`Running version ${version} update script (${sqliteUpdates[version]})...`); logger.info(`Running version ${version} update script...`);
for (const statement of sqliteUpdates[version].split("\n")) { connection.exec(updates[version]);
connection.prepare(statement).run();
} }
} else if (version > latestVersion) {
throw new Error(`SQLite database is at version ${version}, but this version of the bot only supports up to version ${latestVersion}.`);
} else {
return;
} }
connection.pragma(`user_version = ${version}`); // insecure, but the normal templating method doesn't seem to work here connection.pragma(`user_version = ${latestVersion}`); // prepared statements don't seem to work here
connection.prepare("COMMIT").run();
} catch (e) { } catch (e) {
logger.error(`SQLite migration failed: ${e}`); logger.error(`SQLite migration failed: ${e}`);
connection.prepare("ROLLBACK").run(); connection.exec("ROLLBACK");
logger.error("Unable to start the bot, quitting now."); logger.error("Unable to start the bot, quitting now.");
return 1; return 1;
} }
} connection.exec("COMMIT");
} }
export async function fixGuild(guild) { export async function fixGuild(guild) {

View file

@ -1,11 +0,0 @@
#!/bin/sh
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
CREATE TABLE guilds ( guild_id VARCHAR(30) NOT NULL PRIMARY KEY, prefix VARCHAR(15) NOT NULL, disabled text ARRAY NOT NULL, disabled_commands text ARRAY NOT NULL );
CREATE TABLE counts ( command VARCHAR NOT NULL PRIMARY KEY, count integer NOT NULL );
CREATE TABLE tags ( guild_id VARCHAR(30) NOT NULL, name text NOT NULL, content text NOT NULL, author VARCHAR(30) NOT NULL, UNIQUE(guild_id, name) );
CREATE TABLE settings ( id smallint PRIMARY KEY, version integer NOT NULL, broadcast VARCHAR, CHECK(id = 1) );
INSERT INTO settings (id, version) VALUES (1, 2) ON CONFLICT (id) DO NOTHING;
EOSQL