Assorted db changes (#333)

* postgres: use transaction-scoped sql for upgrade

* database: combine fixGuild and addGuild into getGuild

* postgres, sqlite: simplify upgrade()

* allow running commands in DMs without prefix

this functionality was broken in
16095c0256 but is now fixed

* allow running esmBot without a database

Before this change, the only way to run esmBot without a database is to use the
dummy database driver which is broken but fails silently. THis can lead to a
confusing user experience. For instance, using `&command disable` with the
dummy database driver will tell you that the command has been disabled even
though it has not been.

This change adds support for running esmBot with no database driver by leaving
the DB= config option empty, and explicitly telling the user that some
functionality is now unavailable rather than failing silently like the dummy
driver.

* remove dummy database driver

app.js

@@ -97,9 +97,11 @@ esmBot ${esmBotVersion} (${process.env.GIT_REV})
     return process.exit(1);
   }
 
-  // database handling
-  const dbResult = await database.upgrade(logger);
-  if (dbResult === 1) return process.exit(1);
+  if (database) {
+    // database handling
+    const dbResult = await database.upgrade(logger);
+    if (dbResult === 1) return process.exit(1);
+  }
 
   // process the threshold into bytes early
   if (process.env.TEMPDIR && process.env.THRESHOLD) {
@@ -118,7 +120,9 @@ esmBot ${esmBotVersion} (${process.env.GIT_REV})
   }
   logger.log("info", "Finished loading commands.");
 
-  await database.setup();
+  if (database) {
+    await database.setup();
+  }
   if (process.env.API_TYPE === "ws") await reloadImageConnections();
 
   // create the oceanic client

commands/general/broadcast.js

@@ -46,6 +46,7 @@ class BroadcastCommand extends Command {
 
   static description = "Broadcasts a playing message until the command is run again or the bot restarts";
   static adminOnly = true;
+  static dbRequired = true;
 }
 
 export default BroadcastCommand;

commands/general/channel.js

@@ -47,6 +47,7 @@ class ChannelCommand extends Command {
   static arguments = ["[enable/disable]", "{id}"];
   static slashAllowed = false;
   static directAllowed = false;
+  static dbRequired = true;
 }
 
 export default ChannelCommand;

commands/general/command.js

@@ -38,6 +38,7 @@ class CommandCommand extends Command {
   static arguments = ["[enable/disable]", "[command]"];
   static slashAllowed = false;
   static directAllowed = false;
+  static dbRequired = true;
 }
 
 export default CommandCommand;

commands/general/count.js

@@ -48,6 +48,7 @@ class CountCommand extends Command {
   static description = "Gets how many times every command was used";
   static arguments = ["{mention/id}"];
   static aliases = ["counts"];
+  static dbRequired = true;
 }
 
 export default CountCommand;

commands/general/help.js

@@ -9,11 +9,15 @@ const tips = ["You can change the bot's prefix using the prefix command.", "Imag
 
 class HelpCommand extends Command {
   async run() {
-    const { prefix } = this.guild ? await database.getGuild(this.guild.id) : "N/A";
+    let prefix;
+    if (this.guild && database) {
+      prefix = (await database.getGuild(this.guild.id)).prefix;
+    } else {
+      prefix = process.env.PREFIX;
+    }
     if (this.args.length !== 0 && (collections.commands.has(this.args[0].toLowerCase()) || collections.aliases.has(this.args[0].toLowerCase()))) {
       const command = collections.aliases.get(this.args[0].toLowerCase()) ?? this.args[0].toLowerCase();
       const info = collections.info.get(command);
-      const counts = await database.getCounts();
       const embed = {
         embeds: [{
           author: {
@@ -27,10 +31,6 @@ class HelpCommand extends Command {
           fields: [{
             name: "Aliases",
             value: info.aliases.length !== 0 ? info.aliases.join(", ") : "None"
-          }, {
-            name: "Times Used",
-            value: counts[command],
-            inline: true
           }, {
             name: "Parameters",
             value: command === "tags" ? "[name]" : (info.params ? (info.params.length !== 0 ? info.params.join(" ") : "None") : "None"),
@@ -38,6 +38,13 @@ class HelpCommand extends Command {
           }]
         }]
       };
+      if (database) {
+        embed.embeds[0].fields.push({
+          name: "Times used",
+          value: (await database.getCounts())[command],
+          inline: true
+        });
+      }
       if (info.flags.length !== 0) {
         const flagInfo = [];
         for (const flag of info.flags) {
@@ -92,7 +99,7 @@ class HelpCommand extends Command {
             },
             fields: [{
               name: "Prefix",
-              value: this.guild ? prefix : "N/A"
+              value: prefix
             }, {
               name: "Tip",
               value: random(tips)

commands/general/prefix.js

@@ -6,6 +6,9 @@ class PrefixCommand extends Command {
     if (!this.guild) return `The current prefix is \`${process.env.PREFIX}\`.`;
     const guild = await database.getGuild(this.guild.id);
     if (this.args.length !== 0) {
+      if (!database) {
+        return "Setting a per-guild prefix is not possible on a stateless instance of esmBot!"
+      }
       const owners = process.env.OWNER.split(",");
       if (!this.member.permissions.has("ADMINISTRATOR") && !owners.includes(this.member.id)) {
         this.success = false;

commands/tags/tags.js

@@ -163,6 +163,7 @@ class TagsCommand extends Command {
     description: "Gets a random tag"
   }];
   static directAllowed = false;
+  static dbRequired = true;
 }
 
 export default TagsCommand;

docs/config.md

@@ -7,7 +7,7 @@ To make managing environment variables easier, an example `.env` file is include
 ### Required
 - `NODE_ENV`: Used for tuning the bot to different environments. If you don't know what to set it to, leave it as is.
 - `TOKEN`: Your bot's token. You can find this [here](https://discord.com/developers/applications) under your application's Bot tab.
-- `DB`: The database connection string. By default the `sqlite` and `postgresql` protocols are available, but this can be expanded by putting proper DB driver scripts into `utils/database/`. You can also set this to `dummy` to make the bot not use a database at all.
+- `DB`: The database connection string. By default the `sqlite` and `postgresql` protocols are available, but this can be expanded by putting proper DB driver scripts into `utils/database/`.
 - `OWNER`: Your Discord user ID. This is used for granting yourself access to certain management commands. Adding multiple users is supported by separating the IDs with a comma; however, this is not recommended for security purposes.
 - `PREFIX`: The bot's default command prefix for classic commands. Note that servers can set their own individual prefixes via the `prefix` command.
 
@@ -75,4 +75,4 @@ The JSON-based configuration files are located in `config/`.
     // Note: instances must support getting results over JSON
   ]
 }
-```
+```

events/guildCreate.js

@@ -1,9 +0,0 @@
-import db from "../utils/database.js";
-import { log } from "../utils/logger.js";
-
-// run when the bot is added to a guild
-export default async (client, guild) => {
-  log(`[GUILD JOIN] ${guild.name} (${guild.id}) added the bot.`);
-  const guildDB = await db.getGuild(guild.id);
-  if (!guildDB) await db.addGuild(guild.id);
-};

events/interactionCreate.js

@@ -15,13 +15,19 @@ export default async (client, interaction) => {
     cmd = messageCommands.get(command);
     if (!cmd) return;
   }
+  if (cmd.dbRequired && !database) {
+    await interaction["createMessage"]({ content: "This command is unavailable on stateless instances of esmBot.", flags: 64 });
+    return;
+  };
 
   const invoker = interaction.member ?? interaction.user;
 
   // actually run the command
   logger.log("log", `${invoker.username} (${invoker.id}) ran application command ${command}`);
   try {
-    await database.addCount(command);
+    if (database) {
+      await database.addCount(command);
+    }
     // eslint-disable-next-line no-unused-vars
     const commandClass = new cmd(client, { type: "application", interaction });
     const result = await commandClass.run();

events/messageCreate.js

@@ -31,15 +31,12 @@ export default async (client, message) => {
   const mentionResult = message.content.match(mentionRegex);
   if (mentionResult) {
     text = message.content.substring(mentionResult[0].length).trim();
-  } else if (message.guildID) {
+  } else if (message.guildID && database) {
     const cachedPrefix = prefixCache.get(message.guildID);
     if (cachedPrefix && message.content.startsWith(cachedPrefix)) {
       text = message.content.substring(cachedPrefix.length).trim();
     } else {
       guildDB = await database.getGuild(message.guildID);
-      if (!guildDB) {
-        guildDB = await database.fixGuild(message.guildID);
-      }
       if (message.content.startsWith(guildDB.prefix)) {
         text = message.content.substring(guildDB.prefix.length).trim();
         prefixCache.set(message.guildID, guildDB.prefix);
@@ -49,6 +46,8 @@ export default async (client, message) => {
     }
   } else if (message.content.startsWith(process.env.PREFIX)) {
     text = message.content.substring(process.env.PREFIX.length).trim();
+  } else if (!message.guildID) {
+    text = message.content;
   } else {
     return;
   }
@@ -62,8 +61,18 @@ export default async (client, message) => {
   const cmd = commands.get(aliased ?? command);
   if (!cmd) return;
 
+  // block certain commands from running in DMs
+  if (!cmd.directAllowed && !message.guildID) return;
+
+  if (cmd.dbRequired && !database) {
+    await client.rest.channels.createMessage(message.channelID, {
+      content: "This command is unavailable on stateless instances of esmBot."
+    })
+    return;
+  };
+
   // don't run if message is in a disabled channel
-  if (message.guildID) {
+  if (message.guildID && database) {
     let disabled = disabledCache.get(message.guildID);
     if (!disabled) {
       if (!guildDB) guildDB = await database.getGuild(message.guildID);
@@ -81,9 +90,6 @@ export default async (client, message) => {
     if (disabledCmds.includes(aliased ?? command)) return;
   }
 
-  // block certain commands from running in DMs
-  if (!cmd.directAllowed && !message.guildID) return;
-
   // actually run the command
   log("log", `${message.author.username} (${message.author.id}) ran classic command ${command}`);
   const reference = {
@@ -100,7 +106,9 @@ export default async (client, message) => {
   try {
     // parse args
     const parsed = parseCommand(preArgs);
-    await database.addCount(aliases.get(command) ?? command);
+    if (database) {
+      await database.addCount(aliases.get(command) ?? command);
+    }
     const startTime = new Date();
     // eslint-disable-next-line no-unused-vars
     const commandClass = new cmd(client, { type: "classic", message, args: parsed._, content: text.replace(command, "").trim(), specialArgs: (({ _, ...o }) => o)(parsed) }); // we also provide the message content as a parameter for cases where we need more accuracy

utils/database.js

@@ -2,4 +2,19 @@
 import { config } from "dotenv";
 config();
 
-export default await import(`./database/${process.env.DB ? process.env.DB.split("://")[0] : "dummy"}.js`);
+let db = null;
+
+if (process.env.DB) {
+  const dbtype = process.env.DB.split("://")[0];
+  try {
+    db = await import(`./database/${dbtype}.js`);
+  } catch (error) {
+    if (error.code == "ERR_MODULE_NOT_FOUND") {
+      console.error(`DB config option has unknown database type '${dbtype}'`);
+      process.exit(1);
+    }
+    throw error;
+  }
+}
+
+export default db;

utils/database/dummy.js

@@ -1,35 +0,0 @@
-// dummy (no-op) database handler
-import { warn } from "../logger.js";
-
-export async function setup() {
-  warn("Using dummy database adapter. If this isn't what you wanted, check your DB variable.");
-}
-export async function stop() {}
-export async function fixGuild() {}
-export async function addCount() {}
-export async function getCounts() {
-  return {};
-}
-export async function upgrade() {}
-export async function disableCommand() {}
-export async function enableCommand() {}
-export async function disableChannel() {}
-export async function enableChannel() {}
-export async function getTags() {}
-export async function getTag() {}
-export async function setTag() {}
-export async function removeTag() {}
-export async function editTag() {}
-export async function setBroadcast() {}
-export async function getBroadcast() {}
-export async function setPrefix() {}
-export async function addGuild(guild) {
-  return {
-    id: guild,
-    tags: {},
-    prefix: process.env.PREFIX,
-    disabled: [],
-    disabled_commands: []
-  };
-}
-export const getGuild = addGuild;

utils/database/postgresql.js

@@ -42,43 +42,26 @@ const updates = [
 ];
 
 export async function setup() {
-  let counts;
-  try {
-    counts = await sql`SELECT * FROM counts`;
-  } catch {
-    counts = [];
-  }
-
-  const merged = new Map([...commands, ...messageCommands]);
-
-  if (!counts.length) {
-    for (const command of merged.keys()) {
+  const existingCommands = (await sql`SELECT command FROM counts`).map(x => x.command);
+  const commandNames = [...commands.keys(), ...messageCommands.keys()];
+  for (const command of existingCommands) {
+    if (!commandNames.includes(command)) {
+      await sql`DELETE FROM counts WHERE command = ${command}`;
+    }
+  };
+  for (const command of commandNames) {
+    if (!existingCommands.includes(command)) {
       await sql`INSERT INTO counts ${sql({ command, count: 0 }, "command", "count")}`;
     }
-  } else {
-    const exists = [];
-    for (const command of merged.keys()) {
-      const count = await sql`SELECT * FROM counts WHERE command = ${command}`;
-      if (!count.length) {
-        await sql`INSERT INTO counts ${sql({ command, count: 0 }, "command", "count")}`;
-      }
-      exists.push(command);
-    }
-
-    for (const { command } of counts) {
-      if (!exists.includes(command)) {
-        await sql`DELETE FROM counts WHERE command = ${command}`;
-      }
-    }
-  }
+  };
 }
 
 export async function upgrade(logger) {
   try {
-    await sql.begin(async (tx) => {
-      await tx.unsafe(settingsSchema);
+    await sql.begin(async (sql) => {
+      await sql.unsafe(settingsSchema);
       let version;
-      const settingsrow = (await tx`SELECT version FROM settings WHERE id = 1`);
+      const settingsrow = (await sql`SELECT version FROM settings WHERE id = 1`);
       if (settingsrow.length == 0) {
         version = 0;
       } else {
@@ -87,20 +70,20 @@ export async function upgrade(logger) {
       const latestVersion = updates.length - 1;
       if (version === 0) {
         logger.info(`Initializing PostgreSQL database...`);
-        await tx.unsafe(schema);
+        await sql.unsafe(schema);
       } else if (version < latestVersion) {
         logger.info(`Migrating PostgreSQL database, which is currently at version ${version}...`);
         while (version < latestVersion) {
           version++;
           logger.info(`Running version ${version} update script...`);
-          await tx.unsafe(updates[version]);
+          await sql.unsafe(updates[version]);
         }
       } else if (version > latestVersion) {
         throw new Error(`PostgreSQL database is at version ${version}, but this version of the bot only supports up to version ${latestVersion}.`);
       } else {
         return;
       }
-      await tx`INSERT INTO settings ${sql({ id: 1, version: latestVersion })} ON CONFLICT (id) DO UPDATE SET version = ${latestVersion}`;
+      await sql`INSERT INTO settings ${sql({ id: 1, version: latestVersion })} ON CONFLICT (id) DO UPDATE SET version = ${latestVersion}`;
     });
   } catch (e) {
     logger.error(`PostgreSQL migration failed: ${e}`);
@@ -110,7 +93,15 @@ export async function upgrade(logger) {
 }
 
 export async function getGuild(query) {
-  return (await sql`SELECT * FROM guilds WHERE guild_id = ${query}`)[0];
+  let guild;
+  await sql.begin(async (sql) => {
+    guild = (await sql`SELECT * FROM guilds WHERE guild_id = ${query}`)[0];
+    if (guild == undefined) {
+      guild = { guild_id: query, prefix: process.env.PREFIX, disabled: [], disabled_commands: [] };
+      await sql`INSERT INTO guilds ${sql(guild)}`;
+    };
+  });
+  return guild;
 }
 
 export async function setPrefix(prefix, guild) {
@@ -192,25 +183,6 @@ export async function addCount(command) {
   await sql`INSERT INTO counts ${sql({ command, count: 1 }, "command", "count")} ON CONFLICT (command) DO UPDATE SET count = counts.count + 1 WHERE counts.command = ${command}`;
 }
 
-export async function addGuild(guild) {
-  const query = await this.getGuild(guild);
-  if (query) return query;
-  try {
-    await sql`INSERT INTO guilds ${sql({ guild_id: guild, prefix: process.env.PREFIX, disabled: [], disabled_commands: [] })}`;
-  } catch (e) {
-    logger.error(`Failed to register guild ${guild}: ${e}`);
-  }
-  return await this.getGuild(guild);
-}
-
-export async function fixGuild(guild) {
-  const guildDB = await sql`SELECT exists(SELECT 1 FROM guilds WHERE guild_id = ${guild})`;
-  if (!guildDB[0].exists) {
-    logger.log(`Registering guild database entry for guild ${guild}...`);
-    return await this.addGuild(guild);
-  }
-}
-
 export async function stop() {
   await sql.end();
 }

utils/database/sqlite.js

@@ -1,4 +1,4 @@
-import * as collections from "../collections.js";
+import { commands, messageCommands } from "../collections.js";
 import * as logger from "../logger.js";
 
 import sqlite3 from "better-sqlite3";
@@ -43,29 +43,18 @@ const updates = [
 ];
 
 export async function setup() {
-  const counts = connection.prepare("SELECT * FROM counts").all();
-  const merged = new Map([...collections.commands, ...collections.messageCommands]);
-
-  if (!counts || counts.length === 0) {
-    for (const command of merged.keys()) {
+  const existingCommands = connection.prepare("SELECT command FROM counts").all().map(x => x.command);
+  const commandNames = [...commands.keys(), ...messageCommands.keys()];
+  for (const command of existingCommands) {
+    if (!commandNames.includes(command)) {
+      connection.prepare("DELETE FROM counts WHERE command = ?").run(command);
+    }
+  };
+  for (const command of commandNames) {
+    if (!existingCommands.includes(command)) {
       connection.prepare("INSERT INTO counts (command, count) VALUES (?, ?)").run(command, 0);
     }
-  } else {
-    const exists = [];
-    for (const command of merged.keys()) {
-      const count = connection.prepare("SELECT * FROM counts WHERE command = ?").get(command);
-      if (!count) {
-        connection.prepare("INSERT INTO counts (command, count) VALUES (?, ?)").run(command, 0);
-      }
-      exists.push(command);
-    }
-
-    for (const { command } of counts) {
-      if (!exists.includes(command)) {
-        connection.prepare("DELETE FROM counts WHERE command = ?").run(command);
-      }
-    }
-  }
+  };
 }
 
 export async function stop() {
@@ -102,19 +91,6 @@ export async function upgrade(logger) {
   connection.exec("COMMIT");
 }
 
-export async function fixGuild(guild) {
-  let guildDB;
-  try {
-    guildDB = connection.prepare("SELECT * FROM guilds WHERE guild_id = ?").get(guild);
-  } catch {
-    connection.prepare("CREATE TABLE guilds ( guild_id VARCHAR(30) NOT NULL PRIMARY KEY, prefix VARCHAR(15) NOT NULL, disabled text NOT NULL, disabled_commands text NOT NULL )").run();
-  }
-  if (!guildDB) {
-    logger.log(`Registering guild database entry for guild ${guild}...`);
-    return await this.addGuild(guild);
-  }
-}
-
 export async function addCount(command) {
   connection.prepare("UPDATE counts SET count = count + 1 WHERE command = ?").run(command);
 }
@@ -201,23 +177,19 @@ export async function setPrefix(prefix, guild) {
   collections.prefixCache.set(guild.id, prefix);
 }
 
-export async function addGuild(guild) {
-  const query = await this.getGuild(guild);
-  if (query) return query;
-  const guildObject = {
-    id: guild,
-    prefix: process.env.PREFIX,
-    disabled: "[]",
-    disabledCommands: "[]"
-  };
-  connection.prepare("INSERT INTO guilds (guild_id, prefix, disabled, disabled_commands) VALUES (@id, @prefix, @disabled, @disabledCommands)").run(guildObject);
-  return guildObject;
-}
-
 export async function getGuild(query) {
-  try {
-    return connection.prepare("SELECT * FROM guilds WHERE guild_id = ?").get(query);
-  } catch {
-    return;
-  }
+  let guild;
+  connection.transaction(() => {   
+    guild = connection.prepare("SELECT * FROM guilds WHERE guild_id = ?").get(query);
+    if (!guild) {
+      guild = {
+        id: query,
+        prefix: process.env.PREFIX,
+        disabled: "[]",
+        disabledCommands: "[]"
+      };
+      connection.prepare("INSERT INTO guilds (guild_id, prefix, disabled, disabled_commands) VALUES (@id, @prefix, @disabled, @disabledCommands)").run(guild);
+    }
+  })();
+  return guild;
 }

utils/misc.js

@@ -64,6 +64,9 @@ export async function activityChanger(bot) {
 }
 
 export async function checkBroadcast(bot) {
+  if (!db) {
+    return;
+  }
   const message = await db.getBroadcast();
   if (message) {
     startBroadcast(bot, message);

utils/pm2/ext.js

@@ -131,9 +131,11 @@ if (process.env.METRICS && process.env.METRICS !== "") {
 # HELP esmbot_shard_count Number of shards the bot has
 # TYPE esmbot_shard_count gauge
 `);
-    const counts = await database.getCounts();
-    for (const [i, w] of Object.entries(counts)) {
-      res.write(`esmbot_command_count{command="${i}"} ${w}\n`);
+    if (database) {
+      const counts = await database.getCounts();
+      for (const [i, w] of Object.entries(counts)) {
+        res.write(`esmbot_command_count{command="${i}"} ${w}\n`);
+      }
     }
 
     res.write(`esmbot_server_count ${serverCount}\n`);