murm/mrmBot-Matrix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Replace eris-fleet with a pm2-based cluster system, overhaul image handling, removed azure image api

Browse source
This commit is contained in:
Essem 2022-09-21 00:05:03 -05:00
parent 5a3364736d
commit db0decf71a
No known key found for this signature in database
GPG key ID: 7D497397CC3A2A8C
45 changed files with 1777 additions and 857 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.env.example
View file

@ -48,12 +48,7 @@ METRICS=
# The image API type to be used
# Set this to `none` to process all images locally
# Set this to `ws` if you want to use the external image API script, located in api/index.js
# Set this to `azure` to use the Azure Functions API
API_TYPE=none
# If API_TYPE is `azure`, set this to your Azure webhook URL
AZURE_URL=
# If API_TYPE is `azure`, set an optional password for webhook responses
AZURE_PASS=
# Put ID of server to limit owner-only commands to
ADMIN_SERVER=

11
.gitignore vendored
View file

@ -118,14 +118,3 @@ libvips/
# Databases
data/
*.sqlite
# Azure Functions artifacts
bin
obj
appsettings.json
local.settings.json
# Azurite artifacts
__blobstorage__
__queuestorage__
__azurite_db*__.json

12
api/IMPLEMENTATION.md
View file

@ -6,6 +6,9 @@ The esmBot image API is a combined HTTP and WebSocket API. The default port to a
### GET `/image/?id=<job id>`
Get image data after job is finished running. The Content-Type header is properly set.
### GET `/count`
Get the current amount of running jobs. Response is a plaintext number value.
## WebSockets
A client sends *requests* (T-messages) to a server, which subsequently *replies* (R-messages) to the client.
### Message IDs
@ -24,11 +27,11 @@ A client sends *requests* (T-messages) to a server, which subsequently *replies*
[j] means JSON data that goes until the end of the message.
`tag` is used to identify a request/response pair, like `lock` in the original API. `jid` is used to identify a job. `job` is a job object.
- Rerror tag[2] error[s]
- Tqueue tag[2] jid[4] job[j]
- Tqueue tag[2] jid[8] job[j]
- Rqueue tag[2]
- Tcancel tag[2] jid[4]
- Tcancel tag[2] jid[8]
- Rcancel tag[2]
- Twait tag[2] jid[4]
- Twait tag[2] jid[8]
- Rwait tag[2]
- Rinit tag[2] max_jobs[2] running_jobs[2] formats[j]
@ -42,6 +45,7 @@ The job object is formatted like this:
"params": { // content varies depending on the command, some common parameters are listed here
"type": string, // mime type of output, should usually be the same as input
...
}
},
"name": string // filename of the image, without extension
}
```

13
api/index.js
View file

@ -107,8 +107,8 @@ wss.on("connection", (ws, request) => {
const tag = msg.slice(1, 3);
const req = msg.toString().slice(3);
if (opcode == Tqueue) {
const id = msg.readUInt32LE(3);
const obj = msg.slice(7);
const id = msg.readBigInt64LE(3);
const obj = msg.slice(11);
const job = { msg: obj, num: jobAmount, verifyEvent: new EventEmitter() };
jobs.set(id, job);
queue.push(id);
@ -128,7 +128,7 @@ wss.on("connection", (ws, request) => {
const cancelResponse = Buffer.concat([Buffer.from([Rcancel]), tag]);
ws.send(cancelResponse);
} else if (opcode == Twait) {
const id = msg.readUInt32LE(3);
const id = msg.readBigUInt64LE(3);
const job = jobs.get(id);
if (!job) {
const errorResponse = Buffer.concat([Buffer.from([Rerror]), tag, Buffer.from("Invalid job ID")]);
@ -178,7 +178,7 @@ httpServer.on("request", async (req, res) => {
res.statusCode = 400;
return res.end("400 Bad Request");
}
const id = parseInt(reqUrl.searchParams.get("id"));
const id = BigInt(reqUrl.searchParams.get("id"));
if (!jobs.has(id)) {
res.statusCode = 410;
return res.end("410 Gone");
@ -208,6 +208,11 @@ httpServer.on("request", async (req, res) => {
return res.end(data, (err) => {
if (err) error(err);
});
} else if (reqUrl.pathname === "/count" && req.method === "GET") {
log(`Sending job count to ${req.socket.remoteAddress}:${req.socket.remotePort} via HTTP`);
return res.end(jobAmount.toString(), (err) => {
if (err) error(err);
});
} else {
res.statusCode = 404;
return res.end("404 Not Found");

277
app.js
View file

@ -1,12 +1,15 @@
if (process.platform === "win32") console.error("\x1b[1m\x1b[31m\x1b[40m" + `WIN32 IS NOT OFFICIALLY SUPPORTED!
Although there's a (very) slim chance of it working, multiple aspects of the bot are built with UNIX-like systems in mind and could break on Win32-based systems. If you want to run the bot on Windows, using Windows Subsystem for Linux is highly recommended.
The bot will continue to run past this message, but keep in mind that it could break at any time. Continue running at your own risk; alternatively, stop the bot using Ctrl+C and install WSL.` + "\x1b[0m");
if (process.versions.node.split(".")[0] < 15) {
if (process.versions.node.split(".")[0] < 16) {
console.error(`You are currently running Node.js version ${process.version}.
esmBot requires Node.js version 15 or above.
esmBot requires Node.js version 16 or above.
Please refer to step 3 of the setup guide.`);
process.exit(1);
}
if (process.platform === "win32") {
console.error("\x1b[1m\x1b[31m\x1b[40m" + `WINDOWS IS NOT OFFICIALLY SUPPORTED!
Although there's a (very) slim chance of it working, multiple aspects of the bot are built with UNIX-like systems in mind and could break on Win32-based systems. If you want to run the bot on Windows, using Windows Subsystem for Linux is highly recommended.
The bot will continue to run past this message in 5 seconds, but keep in mind that it could break at any time. Continue running at your own risk; alternatively, stop the bot using Ctrl+C and install WSL.` + "\x1b[0m");
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5000);
}
// load config from .env file
import { resolve, dirname } from "path";
@ -14,32 +17,34 @@ import { fileURLToPath } from "url";
import { config } from "dotenv";
config({ path: resolve(dirname(fileURLToPath(import.meta.url)), ".env") });
// main sharding manager
import { Fleet } from "eris-fleet";
import { isMaster } from "cluster";
import { generateList, createPage } from "./utils/help.js";
import { reloadImageConnections } from "./utils/image.js";
// main services
import Shard from "./shard.js";
import ImageWorker from "./utils/services/image.js";
import PrometheusWorker from "./utils/services/prometheus.js";
import Eris from "eris";
import pm2 from "pm2";
// some utils
import { promises, readFileSync } from "fs";
import winston from "winston";
import "winston-daily-rotate-file";
import { logger } from "./utils/logger.js";
import { exec as baseExec } from "child_process";
import { promisify } from "util";
const exec = promisify(baseExec);
// initialize command loader
import { load, send } from "./utils/handler.js";
// command collections
import { paths } from "./utils/collections.js";
// database stuff
import database from "./utils/database.js";
// dbl posting
import { Api } from "@top-gg/sdk";
const dbl = process.env.NODE_ENV === "production" && process.env.DBL ? new Api(process.env.DBL) : null;
// lavalink stuff
import { checkStatus, connect, reload, status, connected } from "./utils/soundplayer.js";
// events
import { endBroadcast, startBroadcast, activityChanger, checkBroadcast } from "./utils/misc.js";
import { parseThreshold } from "./utils/tempimages.js";
const { types } = JSON.parse(readFileSync(new URL("./config/commands.json", import.meta.url)));
if (isMaster) {
const esmBotVersion = JSON.parse(readFileSync(new URL("./package.json", import.meta.url))).version;
const erisFleetVersion = JSON.parse(readFileSync(new URL("./node_modules/eris-fleet/package.json", import.meta.url))).version; // a bit of a hacky way to get the eris-fleet version
exec("git rev-parse HEAD").then(output => output.stdout.substring(0, 7), () => "unknown commit").then(o => process.env.GIT_REV = o).then(() => {
console.log(`
,*\`$ z\`"v
F zBw\`% A ,W "W
@ -61,14 +66,9 @@ k <BBBw BBBBEBBBBBBBBBBBBBBBBBQ4BM #
*+, " F'"'*^~~~^"^\` V+*^
\`"""
esmBot ${esmBotVersion} (${(await exec("git rev-parse HEAD").then(output => output.stdout.substring(0, 7), () => "unknown commit"))}), powered by eris-fleet ${erisFleetVersion}
esmBot ${esmBotVersion} (${process.env.GIT_REV})
`);
}
const services = [
{ name: "image", ServiceWorker: ImageWorker }
];
if (process.env.METRICS && process.env.METRICS !== "") services.push({ name: "prometheus", ServiceWorker: PrometheusWorker });
});
const intents = [
"guildVoiceStates",
@ -80,21 +80,69 @@ if (types.classic) {
intents.push("messageContent");
}
const Admiral = new Fleet({
BotWorker: Shard,
token: `Bot ${process.env.TOKEN}`,
fetchTimeout: 900000,
maxConcurrencyOverride: 1,
startingStatus: {
status: "idle",
game: {
name: "Starting esmBot..."
// PM2-specific handling
if (process.env.PM2_USAGE) {
pm2.launchBus((err, pm2Bus) => {
if (err) {
logger.error(err);
return;
}
pm2Bus.on("process:msg", async (packet) => {
switch (packet.data?.type) {
case "reload":
var path = paths.get(packet.data.message);
await load(bot, path, await checkStatus(), true);
break;
case "soundreload":
var soundStatus = await checkStatus();
if (!soundStatus) {
reload();
}
break;
case "imagereload":
await reloadImageConnections();
break;
case "broadcastStart":
startBroadcast(bot, packet.data.message);
break;
case "broadcastEnd":
endBroadcast(bot);
break;
case "serverCounts":
pm2.sendDataToProcessId(0, {
id: 0,
type: "process:msg",
data: {
type: "serverCounts",
guilds: bot.guilds.size,
shards: bot.shards.size
},
whatToLog: {
blacklist: ["stats_update"]
},
clientOptions: {
topic: true
}, (err) => {
if (err) logger.error(err);
});
break;
}
});
});
}
database.upgrade(logger).then(result => {
if (result === 1) return process.exit(1);
});
// process the threshold into bytes early
if (process.env.TEMPDIR && process.env.THRESHOLD) {
parseThreshold();
}
if (!types.classic && !types.application) {
logger.error("Both classic and application commands are disabled! Please enable at least one command type in config/commands.json.");
process.exit(1);
}
const bot = new Eris(`Bot ${process.env.TOKEN}`, {
allowedMentions: {
everyone: false,
roles: false,
@ -102,93 +150,78 @@ const Admiral = new Fleet({
repliedUser: true
},
restMode: true,
maxShards: "auto",
messageLimit: 50,
intents,
stats: {
requestTimeout: 30000
},
connectionTimeout: 30000
},
useCentralRequestHandler: process.env.DEBUG_LOG ? false : true, // workaround for eris-fleet weirdness
services
});
if (isMaster) {
const logger = winston.createLogger({
levels: {
error: 0,
warn: 1,
info: 2,
main: 3,
debug: 4
},
transports: [
new winston.transports.Console({ format: winston.format.colorize({ all: true }), stderrLevels: ["error", "warn"] }),
new winston.transports.DailyRotateFile({ filename: "logs/error-%DATE%.log", level: "error", zippedArchive: true, maxSize: 4194304, maxFiles: 8 }),
new winston.transports.DailyRotateFile({ filename: "logs/main-%DATE%.log", zippedArchive: true, maxSize: 4194304, maxFiles: 8 })
],
level: process.env.DEBUG_LOG ? "debug" : "main",
format: winston.format.combine(
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
winston.format.printf((info) => {
const {
timestamp, level, message, ...args
} = info;
return `[${timestamp}]: [${level.toUpperCase()}] - ${message} ${Object.keys(args).length ? JSON.stringify(args, null, 2) : ""}`;
}),
)
});
winston.addColors({
info: "green",
main: "gray",
debug: "magenta",
warn: "yellow",
error: "red"
});
database.upgrade(logger).then(result => {
if (result === 1) return process.exit(1);
});
Admiral.on("log", (m) => logger.main(m));
Admiral.on("info", (m) => logger.info(m));
Admiral.on("debug", (m) => logger.debug(m));
Admiral.on("warn", (m) => logger.warn(m));
Admiral.on("error", (m) => logger.error(m));
if (dbl) {
Admiral.on("stats", async (m) => {
await dbl.postStats({
serverCount: m.guilds,
shardCount: m.shardCount
});
});
}
// process the threshold into bytes early
if (process.env.TEMPDIR && process.env.THRESHOLD) {
const matched = process.env.THRESHOLD.match(/(\d+)([KMGT])/);
const sizes = {
K: 1024,
M: 1048576,
G: 1073741824,
T: 1099511627776
};
if (matched && matched[1] && matched[2]) {
process.env.THRESHOLD = matched[1] * sizes[matched[2]];
} else {
logger.error("Invalid THRESHOLD config.");
process.env.THRESHOLD = undefined;
}
const dirstat = (await promises.readdir(process.env.TEMPDIR)).map((file) => {
return promises.stat(`${process.env.TEMPDIR}/${file}`).then((stats) => stats.size);
});
const size = await Promise.all(dirstat);
const reduced = size.reduce((a, b) => {
return a + b;
}, 0);
Admiral.centralStore.set("dirSizeCache", reduced);
bot.once("ready", async () => {
// register commands and their info
const soundStatus = await checkStatus();
logger.log("info", "Attempting to load commands...");
for await (const commandFile of getFiles(resolve(dirname(fileURLToPath(import.meta.url)), "./commands/"))) {
logger.log("main", `Loading command from ${commandFile}...`);
try {
await load(bot, commandFile, soundStatus);
} catch (e) {
logger.error(`Failed to register command from ${commandFile}: ${e}`);
}
}
if (types.application) {
try {
await send(bot);
} catch (e) {
logger.log("error", e);
logger.log("error", "Failed to send command data to Discord, slash/message commands may be unavailable.");
}
}
logger.log("info", "Finished loading commands.");
if (process.env.API_TYPE === "ws") await reloadImageConnections();
await database.setup();
// register events
logger.log("info", "Attempting to load events...");
for await (const file of getFiles(resolve(dirname(fileURLToPath(import.meta.url)), "./events/"))) {
logger.log("main", `Loading event from ${file}...`);
const eventArray = file.split("/");
const eventName = eventArray[eventArray.length - 1].split(".")[0];
if (eventName === "interactionCreate" && !types.application) {
logger.log("warn", `Skipped loading event from ${file} because application commands are disabled`);
continue;
}
const { default: event } = await import(file);
bot.on(eventName, event.bind(null, bot));
}
logger.log("info", "Finished loading events.");
// generate docs
if (process.env.OUTPUT && process.env.OUTPUT !== "") {
generateList();
await createPage(process.env.OUTPUT);
logger.log("info", "The help docs have been generated.");
}
// connect to lavalink
if (!status && !connected) connect(bot);
checkBroadcast(bot);
activityChanger(bot);
logger.log("info", "Started esmBot.");
});
async function* getFiles(dir) {
const dirents = await promises.readdir(dir, { withFileTypes: true });
for (const dirent of dirents) {
const name = dir + (dir.charAt(dir.length - 1) !== "/" ? "/" : "") + dirent.name;
if (dirent.isDirectory()) {
yield* getFiles(name);
} else if (dirent.name.endsWith(".js")) {
yield name;
}
}
}
bot.connect();

7
classes/command.js
View file

@ -1,10 +1,7 @@
class Command {
success = true;
constructor(client, cluster, worker, ipc, options) {
constructor(client, options) {
this.client = client;
this.cluster = cluster;
this.worker = worker;
this.ipc = ipc;
this.origOptions = options;
this.type = options.type;
this.args = options.args;
@ -50,7 +47,7 @@ class Command {
async acknowledge() {
if (this.type === "classic") {
await this.client.sendChannelTyping(this.channel.id);
} else {
} else if (!this.interaction.acknowledged) {
await this.interaction.acknowledge();
}
}

28
classes/imageCommand.js
View file

@ -1,5 +1,6 @@
import Command from "./command.js";
import imageDetect from "../utils/imagedetect.js";
import { runImageJob } from "../utils/image.js";
import { runningCommands } from "../utils/collections.js";
import { readFileSync } from "fs";
const { emotes } = JSON.parse(readFileSync(new URL("../config/messages.json", import.meta.url)));
@ -22,7 +23,7 @@ class ImageCommand extends Command {
// before awaiting the command result, add this command to the set of running commands
runningCommands.set(this.author.id, timestamp);
const magickParams = {
const imageParams = {
cmd: this.constructor.command,
params: {}
};
@ -36,7 +37,7 @@ class ImageCommand extends Command {
if (selection) selectedImages.delete(this.author.id);
if (image === undefined) {
runningCommands.delete(this.author.id);
return this.constructor.noImage;
return `${this.constructor.noImage} (Tip: try right-clicking/holding on a message and press Apps -> Select Image, then try again.)`;
} else if (image.type === "large") {
runningCommands.delete(this.author.id);
return "That image is too large (>= 25MB)! Try using a smaller image.";
@ -44,11 +45,12 @@ class ImageCommand extends Command {
runningCommands.delete(this.author.id);
return "I've been rate-limited by Tenor. Please try uploading your GIF elsewhere.";
}
magickParams.path = image.path;
magickParams.params.type = image.type;
magickParams.url = image.url; // technically not required but can be useful for text filtering
magickParams.name = image.name;
if (this.constructor.requiresGIF) magickParams.onlyGIF = true;
imageParams.path = image.path;
imageParams.params.type = image.type;
imageParams.url = image.url; // technically not required but can be useful for text filtering
imageParams.name = image.name;
imageParams.id = (this.interaction ?? this.message).id;
if (this.constructor.requiresGIF) imageParams.onlyGIF = true;
} catch (e) {
runningCommands.delete(this.author.id);
throw e;
@ -57,31 +59,31 @@ class ImageCommand extends Command {
if (this.constructor.requiresText) {
const text = this.options.text ?? this.args.join(" ").trim();
if (text.length === 0 || !await this.criteria(text, magickParams.url)) {
if (text.length === 0 || !await this.criteria(text, imageParams.url)) {
runningCommands.delete(this.author.id);
return this.constructor.noText;
}
}
if (typeof this.params === "function") {
Object.assign(magickParams.params, this.params(magickParams.url, magickParams.name));
Object.assign(imageParams.params, this.params(imageParams.url, imageParams.name));
} else if (typeof this.params === "object") {
Object.assign(magickParams.params, this.params);
Object.assign(imageParams.params, this.params);
}
let status;
if (magickParams.params.type === "image/gif" && this.type === "classic") {
if (imageParams.params.type === "image/gif" && this.type === "classic") {
status = await this.processMessage(this.message);
}
try {
const { buffer, type } = await this.ipc.serviceCommand("image", { type: "run", obj: magickParams }, true, 9000000);
const { arrayBuffer, type } = await runImageJob(imageParams);
if (type === "nogif" && this.constructor.requiresGIF) {
return "That isn't a GIF!";
}
this.success = true;
return {
file: Buffer.from(buffer.data),
file: Buffer.from(arrayBuffer),
name: `${this.constructor.command}.${type}`
};
} catch (e) {

4
classes/musicCommand.js
View file

@ -2,8 +2,8 @@ import Command from "./command.js";
import { players, queues } from "../utils/soundplayer.js";
class MusicCommand extends Command {
constructor(client, cluster, worker, ipc, options) {
super(client, cluster, worker, ipc, options);
constructor(client, options) {
super(client, options);
if (this.channel.guild) {
this.connection = players.get(this.channel.guild.id);
this.queue = queues.get(this.channel.guild.id);

11
commands/general/avatar.js
View file

@ -7,11 +7,11 @@ class AvatarCommand extends Command {
const self = await this.client.getRESTUser(this.author.id);
if (this.type === "classic" && this.message.mentions[0]) {
return this.message.mentions[0].dynamicAvatarURL(null, 512);
} else if (await this.ipc.fetchUser(member)) {
let user = await this.ipc.fetchUser(member);
if (!user) user = await this.client.getRESTUser(member);
} else if (member) {
const user = await this.client.getRESTUser(member);
if (user) {
return user?.avatar ? this.client._formatImage(`/avatars/${user.id}/${user.avatar}`, null, 512) : `https://cdn.discordapp.com/embed/avatars/${user.discriminator % 5}.png`; // hacky "solution"
} else if (mentionRegex.test(member)) {
} else if (mentionRegex.text(member)) {
const id = member.match(mentionRegex)[1];
if (id < 21154535154122752n) {
this.success = false;
@ -23,6 +23,9 @@ class AvatarCommand extends Command {
} catch {
return self.dynamicAvatarURL(null, 512);
}
} else {
return self.dynamicAvatarURL(null, 512);
}
} else if (this.args.join(" ") !== "" && this.channel.guild) {
const searched = await this.channel.guild.searchMembers(this.args.join(" "));
if (searched.length === 0) return self.dynamicAvatarURL(null, 512);

8
commands/general/banner.js
View file

@ -7,10 +7,11 @@ class BannerCommand extends Command {
const self = await this.client.getRESTUser(this.author.id);
if (this.type === "classic" && this.message.mentions[0]) {
return this.message.mentions[0].dynamicBannerURL(null, 512) ?? "This user doesn't have a banner!";
} else if (await this.ipc.fetchUser(member)) {
} else if (member) {
const user = await this.client.getRESTUser(member);
if (user) {
return user.dynamicBannerURL(null, 512) ?? "This user doesn't have a banner!";
} else if (mentionRegex.test(member)) {
} else if (mentionRegex.text(member)) {
const id = member.match(mentionRegex)[1];
if (id < 21154535154122752n) {
this.success = false;
@ -22,6 +23,9 @@ class BannerCommand extends Command {
} catch {
return self.dynamicBannerURL(null, 512) ?? "You don't have a banner!";
}
} else {
return "This user doesn't have a banner!";
}
} else if (this.args.join(" ") !== "" && this.channel.guild) {
const searched = await this.channel.guild.searchMembers(this.args.join(" "));
if (searched.length === 0) return self.dynamicBannerURL(null, 512) ?? "This user doesn't have a banner!";

42
commands/general/broadcast.js
View file

@ -1,33 +1,39 @@
import Command from "../../classes/command.js";
import { endBroadcast, startBroadcast } from "../../utils/misc.js";
class BroadcastCommand extends Command {
// yet another very hacky command
run() {
return new Promise((resolve) => {
async run() {
const owners = process.env.OWNER.split(",");
if (!owners.includes(this.author.id)) {
this.success = false;
resolve("Only the bot owner can broadcast messages!");
return;
return "Only the bot owner can broadcast messages!";
}
const message = this.options.message ?? this.args.join(" ");
if (message?.trim()) {
this.ipc.centralStore.set("broadcast", message);
this.ipc.broadcast("playbroadcast", message);
this.ipc.register("broadcastSuccess", () => {
this.ipc.unregister("broadcastSuccess");
resolve("Successfully broadcasted message.");
});
} else {
this.ipc.centralStore.delete("broadcast");
this.ipc.broadcast("broadcastend");
this.ipc.register("broadcastEnd", () => {
this.ipc.unregister("broadcastEnd");
resolve("Successfully ended broadcast.");
});
startBroadcast(this.client, message);
if (process.env.PM2_USAGE) {
process.send({
type: "process:msg",
data: {
type: "broadcastStart",
message
}
});
}
return "Started broadcast.";
} else {
endBroadcast(this.client);
if (process.env.PM2_USAGE) {
process.send({
type: "process:msg",
data: {
type: "broadcastEnd"
}
});
}
return "Ended broadcast.";
}
}
static flags = [{
name: "message",

16
commands/general/imagereload.js
View file

@ -1,4 +1,5 @@
import Command from "../../classes/command.js";
import { reloadImageConnections } from "../../utils/image.js";
class ImageReloadCommand extends Command {
async run() {
@ -7,9 +8,18 @@ class ImageReloadCommand extends Command {
this.success = false;
return "Only the bot owner can reload the image servers!";
}
const amount = await this.ipc.serviceCommand("image", { type: "reload" }, true);
if (amount > 0) {
return `Successfully connected to ${amount} image servers.`;
await this.acknowledge();
const length = await reloadImageConnections();
if (!length) {
if (process.env.PM2_USAGE) {
process.send({
type: "process:msg",
data: {
type: "imagereload"
}
});
}
return `Successfully connected to ${length} image server(s).`;
} else {
return "I couldn't connect to any image servers!";
}

13
commands/general/imagestats.js
View file

@ -1,9 +1,9 @@
import Command from "../../classes/command.js";
import { connections } from "../../utils/image.js";
class ImageStatsCommand extends Command {
async run() {
await this.acknowledge();
const servers = await this.ipc.serviceCommand("image", { type: "stats" }, true);
const embed = {
embeds: [{
"author": {
@ -11,14 +11,17 @@ class ImageStatsCommand extends Command {
"icon_url": this.client.user.avatarURL
},
"color": 16711680,
"description": `The bot is currently connected to ${servers.length} image server(s).`,
"description": `The bot is currently connected to ${connections.size} image server(s).`,
"fields": []
}]
};
for (let i = 0; i < servers.length; i++) {
let i = 0;
for (const connection of connections.values()) {
const count = await connection.getCount();
if (!count) continue;
embed.embeds[0].fields.push({
name: `Server ${i + 1}`,
value: `Running Jobs: ${Math.min(servers[i].runningJobs, servers[i].max)}\nQueued: ${Math.max(0, servers[i].runningJobs - servers[i].max)}\nMax Jobs: ${servers[i].max}`
name: `Server ${i++}`,
value: `Running Jobs: ${count}`
});
}
return embed;

9
commands/general/info.js
View file

@ -5,13 +5,14 @@ const { version } = JSON.parse(readFileSync(new URL("../../package.json", import
import Command from "../../classes/command.js";
import { exec as baseExec } from "child_process";
import { promisify } from "util";
import { getServers } from "../../utils/misc.js";
const exec = promisify(baseExec);
class InfoCommand extends Command {
async run() {
let owner = await this.ipc.fetchUser(process.env.OWNER.split(",")[0]);
if (!owner) owner = await this.client.getRESTUser(process.env.OWNER.split(",")[0]);
const stats = await this.ipc.getStats();
const owner = await this.client.getRESTUser(process.env.OWNER.split(",")[0]);
const servers = await getServers();
await this.acknowledge();
return {
embeds: [{
color: 16711680,
@ -30,7 +31,7 @@ class InfoCommand extends Command {
},
{
name: "💬 Total Servers:",
value: stats?.guilds ? stats.guilds : `${this.client.guilds.size} (for this cluster only)`
value: servers ? servers : `${this.client.guilds.size} (for this process only)`
},
{
name: "✅ Official Server:",

38
commands/general/reload.js
View file

@ -1,28 +1,30 @@
import Command from "../../classes/command.js";
import { load } from "../../utils/handler.js";
import { checkStatus } from "../../utils/soundplayer.js";
import { paths } from "../../utils/collections.js";
class ReloadCommand extends Command {
// quite possibly one of the hackiest commands in the bot
run() {
return new Promise((resolve) => {
async run() {
const owners = process.env.OWNER.split(",");
if (!owners.includes(this.author.id)) return resolve("Only the bot owner can reload commands!");
if (!owners.includes(this.author.id)) return "Only the bot owner can reload commands!";
const commandName = this.options.cmd ?? this.args.join(" ");
if (!commandName || !commandName.trim()) return resolve("You need to provide a command to reload!");
this.acknowledge().then(() => {
this.ipc.broadcast("reload", commandName);
this.ipc.register("reloadSuccess", () => {
this.ipc.unregister("reloadSuccess");
this.ipc.unregister("reloadFail");
resolve(`The command \`${commandName}\` has been reloaded.`);
});
this.ipc.register("reloadFail", (message) => {
this.ipc.unregister("reloadSuccess");
this.ipc.unregister("reloadFail");
resolve(message.result);
});
});
if (!commandName || !commandName.trim()) return "You need to provide a command to reload!";
await this.acknowledge();
const path = paths.get(commandName);
if (!path) return "I couldn't find that command!";
const result = await load(this.client, path, await checkStatus(), true);
if (result !== commandName) return "I couldn't reload that command!";
if (process.env.PM2_USAGE) {
process.send({
type: "process:msg",
data: {
type: "reload",
message: commandName
}
});
}
return `The command \`${commandName}\` has been reloaded.`;
}
static flags = [{
name: "cmd",

3
commands/general/restart.js
View file

@ -10,8 +10,7 @@ class RestartCommand extends Command {
await this.client.createMessage(this.channel.id, Object.assign({
content: "esmBot is restarting."
}, this.reference));
this.ipc.restartAllClusters(true);
//this.ipc.broadcast("restart");
process.exit(1);
}
static description = "Restarts me";

34
commands/general/soundreload.js
View file

@ -1,30 +1,30 @@
import Command from "../../classes/command.js";
import { checkStatus, reload } from "../../utils/soundplayer.js";
class SoundReloadCommand extends Command {
// another very hacky command
run() {
return new Promise((resolve) => {
async run() {
const owners = process.env.OWNER.split(",");
if (!owners.includes(this.author.id)) {
this.success = false;
return "Only the bot owner can reload Lavalink!";
}
this.acknowledge().then(() => {
this.ipc.broadcast("soundreload");
this.ipc.register("soundReloadSuccess", (msg) => {
this.ipc.unregister("soundReloadSuccess");
this.ipc.unregister("soundReloadFail");
resolve(`Successfully connected to ${msg.length} Lavalink node(s).`);
});
this.ipc.register("soundReloadFail", () => {
this.ipc.unregister("soundReloadSuccess");
this.ipc.unregister("soundReloadFail");
resolve("I couldn't connect to any Lavalink nodes!");
});
});
await this.acknowledge();
const soundStatus = await checkStatus();
if (!soundStatus) {
const length = reload();
if (process.env.PM2_USAGE) {
process.send({
type: "process:msg",
data: {
type: "soundreload"
}
});
}
return `Successfully connected to ${length} Lavalink node(s).`;
} else {
return "I couldn't connect to any Lavalink nodes!";
}
}
static description = "Attempts to reconnect to all available Lavalink nodes";
static aliases = ["lava", "lavalink", "lavaconnect", "soundconnect"];

26
commands/general/stats.js
View file

@ -7,6 +7,8 @@ import Command from "../../classes/command.js";
import { VERSION } from "eris";
import { exec as baseExec } from "child_process";
import { promisify } from "util";
import pm2 from "pm2";
import { getServers } from "../../utils/misc.js";
const exec = promisify(baseExec);
class StatsCommand extends Command {
@ -14,7 +16,7 @@ class StatsCommand extends Command {
const uptime = process.uptime() * 1000;
const connUptime = this.client.uptime;
const owner = await this.client.getRESTUser(process.env.OWNER.split(",")[0]);
const stats = await this.ipc.getStats();
const servers = await getServers();
return {
embeds: [{
"author": {
@ -28,13 +30,13 @@ class StatsCommand extends Command {
"value": `v${version}${process.env.NODE_ENV === "development" ? `-dev (${(await exec("git rev-parse HEAD", { cwd: dirname(fileURLToPath(import.meta.url)) })).stdout.substring(0, 7)})` : ""}`
},
{
"name": "Cluster Memory Usage",
"value": stats?.clusters[this.cluster] ? `${stats.clusters[this.cluster].ram.toFixed(2)} MB` : `${(process.memoryUsage().heapUsed / 1024 / 1024).toFixed(2)} MB`,
"name": "Process Memory Usage",
"value": `${(process.memoryUsage().heapUsed / 1024 / 1024).toFixed(2)} MB`,
"inline": true
},
{
"name": "Total Memory Usage",
"value": stats?.totalRam ? `${stats.totalRam.toFixed(2)} MB` : "Unknown",
"value": process.env.PM2_USAGE ? `${((await this.list()).reduce((prev, cur) => prev + cur.monit.memory, 0) / 1024 / 1024).toFixed(2)} MB` : "Unknown",
"inline": true
},
{
@ -65,14 +67,9 @@ class StatsCommand extends Command {
"value": this.channel.guild ? this.client.guildShardMap[this.channel.guild.id] : "N/A",
"inline": true
},
{
"name": "Cluster",
"value": this.cluster,
"inline": true
},
{
"name": "Servers",
"value": stats?.guilds ? stats.guilds : `${this.client.guilds.size} (for this cluster only)`,
"value": servers ? servers : `${this.client.guilds.size} (for this process only)`,
"inline": true
}
]
@ -80,6 +77,15 @@ class StatsCommand extends Command {
};
}
list() {
return new Promise((resolve, reject) => {
pm2.list((err, list) => {
if (err) return reject(err);
resolve(list.filter((v) => v.name === "esmBot"));
});
});
}
static description = "Gets some statistics about me";
static aliases = ["status", "stat"];
}

2
commands/general/userinfo.js
View file

@ -2,7 +2,7 @@ import Command from "../../classes/command.js";
class UserInfoCommand extends Command {
async run() {
const getUser = this.message.mentions.length >= 1 ? this.message.mentions[0] : (this.args.length !== 0 ? await this.ipc.fetchUser(this.args[0]) : this.author);
const getUser = this.message.mentions.length >= 1 ? this.message.mentions[0] : (this.args.length !== 0 ? this.client.users.get(this.args[0]) : this.author);
let user;
if (getUser) {
user = getUser;

2
commands/music/host.js
View file

@ -12,7 +12,7 @@ class HostCommand extends MusicCommand {
if (input?.trim()) {
let user;
if (this.type === "classic") {
const getUser = this.message.mentions.length >= 1 ? this.message.mentions[0] : (await this.ipc.fetchUser(input));
const getUser = this.message.mentions.length >= 1 ? this.message.mentions[0] : this.client.users.get(input);
if (getUser) {
user = getUser;
} else if (input.match(/^<?[@#]?[&!]?\d+>?$/) && input >= 21154535154122752n) {

2
commands/music/music.js
View file

@ -15,7 +15,7 @@ class MusicAIOCommand extends Command {
if (aliases.has(cmd)) cmd = aliases.get(cmd);
if (commands.has(cmd) && info.get(cmd).category === "music") {
const command = commands.get(cmd);
const inst = new command(this.client, this.cluster, this.worker, this.ipc, this.origOptions);
const inst = new command(this.client, this.origOptions);
const result = await inst.run();
this.success = inst.success;
return result;

2
commands/tags/tags.js
View file

@ -44,7 +44,7 @@ class TagsCommand extends Command {
if (!tagName || !tagName.trim()) return "You need to provide the name of the tag you want to check the owner of!";
const getResult = await database.getTag(this.channel.guild.id, tagName);
if (!getResult) return "This tag doesn't exist!";
const user = await this.ipc.fetchUser(getResult.author);
const user = this.client.users.get(getResult.author);
this.success = true;
if (!user) {
try {

4
docs/config.md
View file

@ -22,9 +22,7 @@ These variables that are not necessarily required for the bot to run, but can gr
- `TMP_DOMAIN`: The root domain/directory that the images larger than 8MB are stored at. Example: `https://projectlounge.pw/tmp`
- `THRESHOLD`: A filesize threshold that the bot will start deleting old files in `TEMPDIR` at.
- `METRICS`: The HTTP port to serve [Prometheus](https://prometheus.io/)-compatible metrics on.
- `API_TYPE`: Set this to "none" if you want to process all images locally. Alternatively, set it to "ws" to use an image API server specified in the `image` block of `config/servers.json`, or "azure" to use the Azure Functions-based API.
- `AZURE_URL`: Your Azure webhook URL. Only applies if `API` is set to "azure".
- `AZURE_PASS`: An optional password used for Azure requests. Only applies if `API` is set to "azure".
- `API_TYPE`: Set this to "none" if you want to process all images locally. Alternatively, set it to "ws" to use an image API server specified in the `image` block of `config/servers.json`.
- `ADMIN_SERVER`: A Discord server/guild ID to limit owner-only commands such as eval to.
## JSON

3
docs/custom-commands.md
View file

@ -45,9 +45,6 @@ The default command name is the same as the filename that you save it as, exclud
The parameters available to your command consist of the following:
- `this.client`: An instance of an Eris [`Client`](https://abal.moe/Eris/docs/Client), useful for getting info or performing lower-level communication with the Discord API.
- `this.cluster`: The ID of the eris-fleet cluster that the command is being run from. This should be a number greater than or equal to 0.
- `this.worker`: The ID of the current eris-fleet worker. This should be a number greater than or equal to 0.
- `this.ipc`: An eris-fleet [`IPC`](https://danclay.github.io/eris-fleet/classes/IPC.html) instance, useful for communication between worker processes.
- `this.origOptions`: The raw options object provided to the command by the command handler.
- `this.type`: The type of message that activated the command. Can be "classic" (a regular message) or "application" (slash/context menu commands).
- `this.channel`: An Eris [`TextChannel`](https://abal.moe/Eris/docs/TextChannel) object of the channel that the command was run in, useful for getting info about a server and how to respond to a message.

17
ecosystem.config.cjs Normal file
View file

@ -0,0 +1,17 @@
module.exports = {
apps: [{
name: "esmBot-manager",
script: "ext.js",
autorestart: true,
exp_backoff_restart_delay: 1000,
watch: false,
exec_mode: "fork"
}, {
name: "esmBot",
script: "app.js",
autorestart: true,
exp_backoff_restart_delay: 1000,
watch: false,
exec_mode: "cluster"
}]
};

2
events/debug.js
View file

@ -1,5 +1,5 @@
import { debug } from "../utils/logger.js";
export default async (client, cluster, worker, ipc, message) => {
export default async (client, message) => {
debug(message);
};

5
events/error.js Normal file
View file

@ -0,0 +1,5 @@
import { error } from "../utils/logger.js";
export default async (client, message) => {
error(message);
};

2
events/guildCreate.js
View file

@ -2,7 +2,7 @@ import db from "../utils/database.js";
import { log } from "../utils/logger.js";
// run when the bot is added to a guild
export default async (client, cluster, worker, ipc, guild) => {
export default async (client, guild) => {
log(`[GUILD JOIN] ${guild.name} (${guild.id}) added the bot.`);
const guildDB = await db.getGuild(guild.id);
if (!guildDB) await db.addGuild(guild);

2
events/guildDelete.js
View file

@ -1,6 +1,6 @@
import { log } from "../utils/logger.js";
// run when the bot is removed from a guild
export default async (client, cluster, worker, ipc, guild) => {
export default async (client, guild) => {
log(`[GUILD LEAVE] ${guild.name} (${guild.id}) removed the bot.`);
};

6
events/interactionCreate.js
View file

@ -5,7 +5,7 @@ import { clean } from "../utils/misc.js";
import { upload } from "../utils/tempimages.js";
// run when a slash command is executed
export default async (client, cluster, worker, ipc, interaction) => {
export default async (client, interaction) => {
if (interaction?.type !== 2) return;
// check if command exists and if it's enabled
@ -23,7 +23,7 @@ export default async (client, cluster, worker, ipc, interaction) => {
try {
await database.addCount(command);
// eslint-disable-next-line no-unused-vars
const commandClass = new cmd(client, cluster, worker, ipc, { type: "application", interaction });
const commandClass = new cmd(client, { type: "application", interaction });
const result = await commandClass.run();
const replyMethod = interaction.acknowledged ? "editOriginalMessage" : "createMessage";
if (typeof result === "string") {
@ -39,7 +39,7 @@ export default async (client, cluster, worker, ipc, interaction) => {
const fileSize = 8388119;
if (result.file.length > fileSize) {
if (process.env.TEMPDIR && process.env.TEMPDIR !== "") {
await upload(client, ipc, result, interaction, true);
await upload(client, result, interaction, true);
} else {
await interaction[replyMethod]({
content: "The resulting image was more than 8MB in size, so I can't upload it.",

6
events/messageCreate.js
View file

@ -6,7 +6,7 @@ import { clean } from "../utils/misc.js";
import { upload } from "../utils/tempimages.js";
// run when someone sends a message
export default async (client, cluster, worker, ipc, message) => {
export default async (client, message) => {
// ignore other bots
if (message.author.bot) return;
@ -104,7 +104,7 @@ export default async (client, cluster, worker, ipc, message) => {
await database.addCount(aliases.get(command) ?? command);
const startTime = new Date();
// eslint-disable-next-line no-unused-vars
const commandClass = new cmd(client, cluster, worker, ipc, { type: "classic", message, args: parsed._, content: message.content.substring(prefix.length).trim().replace(command, "").trim(), specialArgs: (({ _, ...o }) => o)(parsed) }); // we also provide the message content as a parameter for cases where we need more accuracy
const commandClass = new cmd(client, { type: "classic", message, args: parsed._, content: message.content.substring(prefix.length).trim().replace(command, "").trim(), specialArgs: (({ _, ...o }) => o)(parsed) }); // we also provide the message content as a parameter for cases where we need more accuracy
const result = await commandClass.run();
const endTime = new Date();
if ((endTime - startTime) >= 180000) reference.allowedMentions.repliedUser = true;
@ -129,7 +129,7 @@ export default async (client, cluster, worker, ipc, message) => {
}
if (result.file.length > fileSize) {
if (process.env.TEMPDIR && process.env.TEMPDIR !== "") {
await upload(client, ipc, result, message);
await upload(client, result, message);
} else {
await client.createMessage(message.channel.id, "The resulting image was more than 8MB in size, so I can't upload it.");
}

2
events/voiceChannelLeave.js
View file

@ -4,7 +4,7 @@ import { random } from "../utils/misc.js";
const isWaiting = new Map();
export default async (client, cluster, worker, ipc, member, oldChannel) => {
export default async (client, member, oldChannel) => {
if (!oldChannel) return;
const connection = players.get(oldChannel.guild.id);
if (oldChannel.id === connection?.voiceChannel.id) {

4
events/voiceChannelSwitch.js
View file

@ -1,5 +1,5 @@
import leaveHandler from "./voiceChannelLeave.js";
export default async (client, cluster, worker, ipc, member, newChannel, oldChannel) => {
await leaveHandler(client, cluster, worker, ipc, member, oldChannel);
export default async (client, member, newChannel, oldChannel) => {
await leaveHandler(client, member, oldChannel);
};

5
events/warn.js Normal file
View file

@ -0,0 +1,5 @@
import { warn } from "../utils/logger.js";
export default async (client, message) => {
warn(message);
};

2
package.json
View file

@ -32,7 +32,6 @@
"dotenv": "^16.0.2",
"emoji-regex": "^10.1.0",
"eris": "github:esmBot/eris#dev",
"eris-fleet": "github:esmBot/eris-fleet#a19920f",
"file-type": "^17.1.6",
"format-duration": "^2.0.0",
"jsqr": "^1.4.0",
@ -57,6 +56,7 @@
"better-sqlite3": "^7.6.2",
"bufferutil": "^4.0.6",
"erlpack": "github:abalabahaha/erlpack",
"pm2": "^5.2.0",
"postgres": "^3.2.4",
"uuid": "^8.3.2",
"ws": "^8.8.1",

1017
pnpm-lock.yaml
View file

File diff suppressed because it is too large Load diff

178
shard.js
View file

@ -1,178 +0,0 @@
// shard base
import { BaseClusterWorker } from "eris-fleet";
// path stuff
import { readdir } from "fs/promises";
import { readFileSync } from "fs";
import { resolve, dirname } from "path";
import { fileURLToPath } from "url";
// fancy loggings
import { log, error } from "./utils/logger.js";
// initialize command loader
import { load, send } from "./utils/handler.js";
// lavalink stuff
import { checkStatus, connect, reload, status, connected } from "./utils/soundplayer.js";
// database stuff
import database from "./utils/database.js";
// command collections
import { paths } from "./utils/collections.js";
// playing messages
const { messages } = JSON.parse(readFileSync(new URL("./config/messages.json", import.meta.url)));
// command config
const { types } = JSON.parse(readFileSync(new URL("./config/commands.json", import.meta.url)));
// other stuff
import { random } from "./utils/misc.js";
// generate help page
import { generateList, createPage } from "./utils/help.js";
// whether a broadcast is currently in effect
let broadcast = false;
class Shard extends BaseClusterWorker {
constructor(bot) {
super(bot);
console.info = (str) => this.ipc.sendToAdmiral("info", str);
this.playingSuffix = types.classic ? ` | @${this.bot.user.username} help` : "";
this.init();
}
async init() {
if (!types.classic && !types.application) {
error("Both classic and application commands are disabled! Please enable at least one command type in config/commands.json.");
this.ipc.totalShutdown(true);
return;
}
// register commands and their info
const soundStatus = await checkStatus();
log("info", "Attempting to load commands...");
for await (const commandFile of this.getFiles(resolve(dirname(fileURLToPath(import.meta.url)), "./commands/"))) {
log("log", `Loading command from ${commandFile}...`);
try {
await load(this.bot, commandFile, soundStatus);
} catch (e) {
error(`Failed to register command from ${commandFile}: ${e}`);
}
}
if (types.application) {
try {
await send(this.bot);
} catch (e) {
log("error", e);
log("error", "Failed to send command data to Discord, slash/message commands may be unavailable.");
}
}
log("info", "Finished loading commands.");
await database.setup(this.ipc);
// register events
log("info", "Attempting to load events...");
for await (const file of this.getFiles(resolve(dirname(fileURLToPath(import.meta.url)), "./events/"))) {
log("log", `Loading event from ${file}...`);
const eventArray = file.split("/");
const eventName = eventArray[eventArray.length - 1].split(".")[0];
if (eventName === "interactionCreate" && !types.application) {
log("warn", `Skipped loading event from ${file} because application commands are disabled`);
continue;
}
const { default: event } = await import(file);
this.bot.on(eventName, event.bind(null, this.bot, this.clusterID, this.workerID, this.ipc));
}
log("info", "Finished loading events.");
// generate docs
if (process.env.OUTPUT && process.env.OUTPUT !== "") {
generateList();
if (this.clusterID === 0) {
await createPage(process.env.OUTPUT);
log("info", "The help docs have been generated.");
}
}
this.ipc.register("reload", async (message) => {
const path = paths.get(message);
if (!path) return this.ipc.broadcast("reloadFail", { result: "I couldn't find that command!" });
try {
const result = await load(this.bot, path, await checkStatus(), true);
if (result !== message) return this.ipc.broadcast("reloadFail", { result });
return this.ipc.broadcast("reloadSuccess");
} catch (result) {
return this.ipc.broadcast("reloadFail", { result });
}
});
this.ipc.register("soundreload", async () => {
const soundStatus = await checkStatus();
if (!soundStatus) {
const length = reload();
return this.ipc.broadcast("soundReloadSuccess", { length });
} else {
return this.ipc.broadcast("soundReloadFail");
}
});
this.ipc.register("playbroadcast", (message) => {
this.bot.editStatus("dnd", {
name: message + this.playingSuffix,
});
broadcast = true;
return this.ipc.broadcast("broadcastSuccess");
});
this.ipc.register("broadcastend", () => {
this.bot.editStatus("dnd", {
name: random(messages) + this.playingSuffix,
});
broadcast = false;
return this.ipc.broadcast("broadcastEnd");
});
// connect to lavalink
if (!status && !connected) connect(this.bot);
const broadcastMessage = await this.ipc.centralStore.get("broadcast");
if (broadcastMessage) {
broadcast = true;
this.bot.editStatus("dnd", {
name: broadcastMessage + this.playingSuffix,
});
}
this.activityChanger();
log("info", `Started worker ${this.workerID}.`);
}
// set activity (a.k.a. the gamer code)
activityChanger() {
if (!broadcast) {
this.bot.editStatus("dnd", {
name: random(messages) + this.playingSuffix,
});
}
setTimeout(this.activityChanger.bind(this), 900000);
}
async* getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
for (const dirent of dirents) {
const name = dir + (dir.charAt(dir.length - 1) !== "/" ? "/" : "") + dirent.name;
if (dirent.isDirectory()) {
yield* this.getFiles(name);
} else if (dirent.name.endsWith(".js")) {
yield name;
}
}
}
shutdown(done) {
log("warn", "Shutting down...");
this.bot.editStatus("dnd", {
name: "Restarting/shutting down..."
});
database.stop();
done();
}
}
export default Shard;

4
utils/help.js
View file

@ -37,11 +37,7 @@ This page was last generated on \`${new Date().toString()}\`.
\`[]\` means an argument is required, \`{}\` means an argument is optional.
Default prefix is \`&\`.
**Want to help support esmBot's development? Consider donating on Patreon!** https://patreon.com/TheEssem
> Tip: You can get much more info about a command by using \`help [command]\` in the bot itself.
`;
template += "\n## Table of Contents\n";

113
utils/image.js
View file

@ -1,14 +1,22 @@
import { request } from "undici";
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
import { Worker } from "worker_threads";
import { createRequire } from "module";
import { fileTypeFromBuffer, fileTypeFromFile } from "file-type";
import * as logger from "./logger.js";
import ImageConnection from "./imageConnection.js";
// only requiring this to work around an issue regarding worker threads
const nodeRequire = createRequire(import.meta.url);
if (!process.env.API_TYPE || process.env.API_TYPE === "none") {
nodeRequire(`../build/${process.env.DEBUG && process.env.DEBUG === "true" ? "Debug" : "Release"}/image.node`);
}
const formats = ["image/jpeg", "image/png", "image/webp", "image/gif", "video/mp4", "video/webm", "video/quicktime"];
export const jobs = {};
export const connections = new Map();
export const servers = JSON.parse(fs.readFileSync(new URL("../config/servers.json", import.meta.url), { encoding: "utf8" })).image;
export let servers = process.env.API_TYPE === "ws" ? JSON.parse(fs.readFileSync(new URL("../config/servers.json", import.meta.url), { encoding: "utf8" })).image : null;
export async function getType(image, extraReturnTypes) {
if (!image.startsWith("http")) {
@ -65,3 +73,98 @@ export async function getType(image, extraReturnTypes) {
}
return type;
}
function connect(server, auth) {
const connection = new ImageConnection(server, auth);
connections.set(server, connection);
}
function disconnect() {
for (const connection of connections.values()) {
connection.close();
}
connections.clear();
}
async function repopulate() {
const data = await fs.promises.readFile(new URL("../config/servers.json", import.meta.url), { encoding: "utf8" });
servers = JSON.parse(data).image;
}
export async function reloadImageConnections() {
disconnect();
await repopulate();
let amount = 0;
for (const server of servers) {
try {
connect(server.server, server.auth);
amount += 1;
} catch (e) {
logger.error(e);
}
}
return amount;
}
function chooseServer(ideal) {
if (ideal.length === 0) throw "No available servers";
const sorted = ideal.sort((a, b) => {
return a.load - b.load;
});
return sorted[0];
}
async function getIdeal(object) {
const idealServers = [];
for (const [address, connection] of connections) {
if (connection.conn.readyState !== 0 && connection.conn.readyState !== 1) {
continue;
}
if (object.params.type && !connection.formats[object.cmd]?.includes(object.params.type)) continue;
idealServers.push({
addr: address,
load: await connection.getCount()
});
}
const server = chooseServer(idealServers);
return connections.get(server.addr);
}
function waitForWorker(worker) {
return new Promise((resolve, reject) => {
worker.once("message", (data) => {
resolve({
buffer: Buffer.from([...data.buffer]),
type: data.fileExtension
});
});
worker.once("error", reject);
});
}
export async function runImageJob(params) {
if (process.env.API_TYPE === "ws") {
for (let i = 0; i < 3; i++) {
const currentServer = await getIdeal(params);
try {
await currentServer.queue(BigInt(params.id), params);
await currentServer.wait(BigInt(params.id));
const output = await currentServer.getOutput(params.id);
return output;
} catch (e) {
if (i < 2 && e === "Request ended prematurely due to a closed connection") {
continue;
} else {
if (e === "No available servers" && i >= 2) throw "Request ended prematurely due to a closed connection";
throw e;
}
}
}
} else {
// Called from command (not using image API)
const worker = new Worker(path.join(path.dirname(fileURLToPath(import.meta.url)), "./image-runner.js"), {
workerData: params
});
return await waitForWorker(worker);
}
}

40
utils/imageConnection.js
View file

@ -22,8 +22,6 @@ class ImageConnection {
this.auth = auth;
this.tag = 0;
this.disconnected = false;
this.njobs = 0;
this.max = 0;
this.formats = {};
this.wsproto = null;
if (tls) {
@ -43,17 +41,15 @@ class ImageConnection {
} else {
httpproto = "http";
}
this.httpurl = `${httpproto}://${host}/image`;
this.httpurl = `${httpproto}://${host}`;
this.conn.on("message", (msg) => this.onMessage(msg));
this.conn.once("error", (err) => this.onError(err));
this.conn.once("close", () => this.onClose());
}
onMessage(msg) {
async onMessage(msg) {
const op = msg.readUint8(0);
if (op === Rinit) {
this.max = msg.readUint16LE(3);
this.njobs = msg.readUint16LE(5);
this.formats = JSON.parse(msg.toString("utf8", 7));
return;
}
@ -64,10 +60,7 @@ class ImageConnection {
return;
}
this.requests.delete(tag);
if (op === Rqueue) this.njobs++;
if (op === Rcancel || op === Rwait) this.njobs--;
if (op === Rerror) {
this.njobs--;
promise.reject(new Error(msg.slice(3, msg.length).toString()));
return;
}
@ -82,9 +75,7 @@ class ImageConnection {
for (const [tag, obj] of this.requests.entries()) {
obj.reject("Request ended prematurely due to a closed connection");
this.requests.delete(tag);
if (obj.op === Twait || obj.op === Tcancel) this.njobs--;
}
//this.requests.clear();
if (!this.disconnected) {
logger.warn(`Lost connection to ${this.host}, attempting to reconnect in 5 seconds...`);
await setTimeout(5000);
@ -107,25 +98,25 @@ class ImageConnection {
queue(jobid, jobobj) {
const str = JSON.stringify(jobobj);
const buf = Buffer.alloc(4);
buf.writeUint32LE(jobid);
const buf = Buffer.alloc(8);
buf.writeBigUint64LE(jobid);
return this.do(Tqueue, jobid, Buffer.concat([buf, Buffer.from(str)]));
}
wait(jobid) {
const buf = Buffer.alloc(4);
buf.writeUint32LE(jobid);
const buf = Buffer.alloc(8);
buf.writeBigUint64LE(jobid);
return this.do(Twait, jobid, buf);
}
cancel(jobid) {
const buf = Buffer.alloc(4);
buf.writeUint32LE(jobid);
const buf = Buffer.alloc(8);
buf.writeBigUint64LE(jobid);
return this.do(Tcancel, jobid, buf);
}
async getOutput(jobid) {
const req = await request(`${this.httpurl}?id=${jobid}`, {
const req = await request(`${this.httpurl}/image?id=${jobid}`, {
headers: {
authentication: this.auth || undefined
}
@ -149,7 +140,18 @@ class ImageConnection {
type = contentType;
break;
}
return { buffer: Buffer.from(await req.body.arrayBuffer()), type };
return { arrayBuffer: await req.body.arrayBuffer(), type };
}
async getCount() {
const req = await request(`${this.httpurl}/count`, {
headers: {
authentication: this.auth || undefined
}
});
if (req.statusCode !== 200) return;
const res = parseInt(await req.body.text());
return res;
}
async do(op, id, data) {

39
utils/logger.js
View file

@ -1,4 +1,41 @@
export function log(type, content) { return content ? console[type](content) : console.info(type); }
import winston from "winston";
import "winston-daily-rotate-file";
export const logger = winston.createLogger({
levels: {
error: 0,
warn: 1,
info: 2,
main: 3,
debug: 4
},
transports: [
new winston.transports.Console({ format: winston.format.colorize({ all: true }), stderrLevels: ["error", "warn"] }),
new winston.transports.DailyRotateFile({ filename: "logs/error-%DATE%.log", level: "error", zippedArchive: true, maxSize: 4194304, maxFiles: 8 }),
new winston.transports.DailyRotateFile({ filename: "logs/main-%DATE%.log", zippedArchive: true, maxSize: 4194304, maxFiles: 8 })
],
level: process.env.DEBUG_LOG ? "debug" : "main",
format: winston.format.combine(
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
winston.format.printf((info) => {
const {
timestamp, level, message, ...args
} = info;
return `[${timestamp}]: [${level.toUpperCase()}] - ${message} ${Object.keys(args).length ? JSON.stringify(args, null, 2) : ""}`;
}),
)
});
winston.addColors({
info: "green",
main: "gray",
debug: "magenta",
warn: "yellow",
error: "red"
});
export function log(type, content) { return content ? logger.log(type === "log" ? "main" : type, content) : logger.info(type); }
export function error(...args) { return log("error", ...args); }

65
utils/misc.js
View file

@ -1,7 +1,14 @@
import util from "util";
import fs from "fs";
import pm2 from "pm2";
import { config } from "dotenv";
// playing messages
const { messages } = JSON.parse(fs.readFileSync(new URL("../config/messages.json", import.meta.url)));
const { types } = JSON.parse(fs.readFileSync(new URL("../config/commands.json", import.meta.url)));
let broadcast = false;
// random(array) to select a random entry in array
export function random(array) {
if (!array || array.length < 1) return null;
@ -42,3 +49,61 @@ export function clean(text) {
export function textEncode(string) {
return string.replaceAll("&", "&amp;").replaceAll(">", "&gt;").replaceAll("<", "&lt;").replaceAll("\"", "&quot;").replaceAll("'", "&apos;").replaceAll("\\n", "\n").replaceAll("\\:", ":");
}
// set activity (a.k.a. the gamer code)
export function activityChanger(bot) {
if (!broadcast) {
bot.editStatus("dnd", {
name: random(messages) + (types.classic ? ` | @${bot.user.username} help` : ""),
});
}
setTimeout(() => activityChanger(bot), 900000);
}
export function checkBroadcast(bot) {
/*if () {
startBroadcast(bot, message);
}*/
}
export function startBroadcast(bot, message) {
bot.editStatus("dnd", {
name: message + (types.classic ? ` | @${bot.user.username} help` : ""),
});
broadcast = true;
}
export function endBroadcast(bot) {
bot.editStatus("dnd", {
name: random(messages) + (types.classic ? ` | @${bot.user.username} help` : ""),
});
broadcast = false;
}
export function getServers() {
return new Promise((resolve, reject) => {
if (process.env.PM2_USAGE) {
pm2.launchBus((err, pm2Bus) => {
const listener = (packet) => {
if (packet.data?.type === "countResponse") {
resolve(packet.data.serverCount);
pm2Bus.off("process:msg");
}
};
pm2Bus.on("process:msg", listener);
});
pm2.sendDataToProcessId(0, {
id: 0,
type: "process:msg",
data: {
type: "getCount"
},
topic: true
}, (err) => {
if (err) reject(err);
});
} else {
resolve(0);
}
});
}

125
utils/pm2/ext.js Normal file
View file

@ -0,0 +1,125 @@
import pm2 from "pm2";
import { Api } from "@top-gg/sdk";
import winston from "winston";
// load config from .env file
import { resolve, dirname } from "path";
import { fileURLToPath } from "url";
import { config } from "dotenv";
config({ path: resolve(dirname(fileURLToPath(import.meta.url)), "../../.env") });
const dbl = process.env.NODE_ENV === "production" && process.env.DBL ? new Api(process.env.DBL) : null;
const logger = winston.createLogger({
levels: {
error: 0,
warn: 1,
info: 2,
main: 3,
debug: 4
},
transports: [
new winston.transports.Console({ format: winston.format.colorize({ all: true }), stderrLevels: ["error", "warn"] })
],
level: process.env.DEBUG_LOG ? "debug" : "main",
format: winston.format.combine(
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
winston.format.printf((info) => {
const {
timestamp, level, message, ...args
} = info;
return `[${timestamp}]: [${level.toUpperCase()}] - ${message} ${Object.keys(args).length ? JSON.stringify(args, null, 2) : ""}`;
}),
)
});
winston.addColors({
info: "green",
main: "gray",
debug: "magenta",
warn: "yellow",
error: "red"
});
let serverCount = 0;
let shardCount = 0;
let clusterCount = 0;
let responseCount = 0;
let timeout;
process.on("message", (packet) => {
if (packet.data?.type === "getCount") {
process.send({
type: "process:msg",
data: {
type: "countResponse",
serverCount
}
});
}
});
function updateStats() {
return new Promise((resolve, reject) => {
pm2.list((err, list) => {
if (err) reject(err);
const clusters = list.filter((v) => v.name === "esmBot");
clusterCount = clusters.length;
const listener = (packet) => {
if (packet.data?.type === "serverCounts") {
clearTimeout(timeout);
serverCount += packet.data.guilds;
shardCount += packet.data.shards;
responseCount += 1;
if (responseCount >= clusterCount) {
resolve();
process.removeListener("message", listener);
} else {
timeout = setTimeout(() => {
reject();
process.removeListener("message", listener);
}, 5000);
}
}
};
timeout = setTimeout(() => {
reject();
process.removeListener("message", listener);
}, 5000);
process.on("message", listener);
process.send({
type: "process:msg",
data: {
type: "serverCounts"
}
});
});
});
}
async function dblPost() {
logger.main("Posting stats to Top.gg...");
serverCount = 0;
shardCount = 0;
clusterCount = 0;
responseCount = 0;
try {
//await updateStats();
await dbl.postStats({
serverCount,
shardCount
});
logger.main("Stats posted.");
} catch (e) {
logger.error(e);
}
}
setInterval(updateStats, 300000);
if (dbl) setInterval(dblPost, 1800000);
setTimeout(updateStats, 10000);
logger.info("Started esmBot management process.");

268
utils/services/image.js
View file

@ -1,268 +0,0 @@
import { BaseServiceWorker } from "eris-fleet";
import * as logger from "../logger.js";
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
import { Worker } from "worker_threads";
import { createRequire } from "module";
import { createServer } from "http";
import { request } from "undici";
import EventEmitter from "events";
// only requiring this to work around an issue regarding worker threads
const nodeRequire = createRequire(import.meta.url);
if (!process.env.API_TYPE || process.env.API_TYPE === "none") {
nodeRequire(`../../build/${process.env.DEBUG && process.env.DEBUG === "true" ? "Debug" : "Release"}/image.node`);
}
import ImageConnection from "../imageConnection.js";
class ImageWorker extends BaseServiceWorker {
constructor(setup) {
super(setup);
console.info = (str) => this.ipc.sendToAdmiral("info", str);
if (process.env.API_TYPE === "ws") {
this.connections = new Map();
this.servers = JSON.parse(fs.readFileSync(new URL("../../config/servers.json", import.meta.url), { encoding: "utf8" })).image;
this.nextID = 0;
} else if (process.env.API_TYPE === "azure") {
this.jobs = new Map();
this.webhook = createServer();
this.port = parseInt(process.env.WEBHOOK_PORT) || 3763;
}
this.begin().then(() => this.serviceReady());
}
async begin() {
// connect to image api if enabled
if (process.env.API_TYPE === "ws") {
for (const server of this.servers) {
try {
await this.connect(server.server, server.auth);
} catch (e) {
logger.error(e);
}
}
} else if (process.env.API_TYPE === "azure") {
this.webhook.on("request", async (req, res) => {
if (req.method !== "POST") {
res.statusCode = 405;
return res.end("405 Method Not Allowed");
}
if (process.env.AZURE_PASS && req.headers.authorization !== process.env.AZURE_PASS) {
res.statusCode = 401;
return res.end("401 Unauthorized");
}
const reqUrl = new URL(req.url, `http://${req.headers.host}`);
if (reqUrl.pathname === "/callback") {
try {
const chunks = [];
req.on("data", (data) => {
chunks.push(data);
});
req.once("end", () => {
if (this.jobs.has(req.headers["x-azure-id"])) {
try {
const error = JSON.parse(Buffer.concat(chunks).toString());
if (error.error) this.jobs.get(req.headers["x-azure-id"]).emit("error", new Error(error.message));
} catch {
// no-op
}
const contentType = req.headers["content-type"];
let type;
switch (contentType) {
case "image/gif":
type = "gif";
break;
case "image/png":
type = "png";
break;
case "image/jpeg":
type = "jpg";
break;
case "image/webp":
type = "webp";
break;
default:
type = contentType;
break;
}
this.jobs.get(req.headers["x-azure-id"]).emit("image", { buffer: Buffer.concat(chunks), type });
return res.end("OK");
} else {
res.statusCode = 409;
return res.end("409 Conflict");
}
});
} catch (e) {
logger.error("An error occurred while processing a webhook request: ", e);
res.statusCode = 500;
return res.end("500 Internal Server Error");
}
} else {
res.statusCode = 404;
return res.end("404 Not Found");
}
});
this.webhook.on("error", (e) => {
logger.error("An error occurred on the Azure webhook: ", e);
});
this.webhook.listen(this.port, () => {
logger.log(`Azure HTTP webhook listening on port ${this.port}`);
});
}
}
async repopulate() {
const data = await fs.promises.readFile(new URL("../../config/servers.json", import.meta.url), { encoding: "utf8" });
this.servers = JSON.parse(data).image;
return;
}
async getRunning() {
const statuses = [];
if (process.env.API_TYPE === "ws") {
for (const [address, connection] of this.connections) {
if (connection.conn.readyState !== 0 && connection.conn.readyState !== 1) {
continue;
}
statuses.push({
address,
runningJobs: connection.njobs,
max: connection.max
});
}
}
return statuses;
}
async chooseServer(ideal) {
if (ideal.length === 0) throw "No available servers";
const sorted = ideal.sort((a, b) => {
return a.load - b.load;
}).filter((e, i, array) => {
return !(e.load < array[0].load);
});
return sorted[0];
}
async getIdeal(object) {
const idealServers = [];
for (const [address, connection] of this.connections) {
if (connection.conn.readyState !== 0 && connection.conn.readyState !== 1) {
continue;
}
if (object.params.type && !connection.formats[object.cmd]?.includes(object.params.type)) continue;
idealServers.push({
addr: address,
load: connection.njobs / connection.max
});
}
const server = await this.chooseServer(idealServers);
return this.connections.get(server.addr);
}
async connect(server, auth) {
const connection = new ImageConnection(server, auth);
this.connections.set(server, connection);
}
async disconnect() {
for (const connection of this.connections.values()) {
connection.close();
}
this.connections.clear();
return;
}
waitForWorker(worker) {
return new Promise((resolve, reject) => {
worker.once("message", (data) => {
resolve({
buffer: Buffer.from([...data.buffer]),
type: data.fileExtension
});
});
worker.once("error", reject);
});
}
waitForAzure(event) {
return new Promise((resolve, reject) => {
event.once("image", (data) => {
resolve(data);
});
event.once("error", reject);
});
}
async run(object) {
if (process.env.API_TYPE === "ws") {
let num = this.nextID++;
if (num > 4294967295) num = this.nextID = 0;
for (let i = 0; i < 3; i++) {
const currentServer = await this.getIdeal(object);
try {
await currentServer.queue(num, object);
await currentServer.wait(num);
const output = await currentServer.getOutput(num);
return output;
} catch (e) {
if (i < 2 && e === "Request ended prematurely due to a closed connection") {
continue;
} else {
if (e === "No available servers" && i >= 2) throw "Request ended prematurely due to a closed connection";
throw e;
}
}
}
} else if (process.env.API_TYPE === "azure") {
object.callback = `${process.env.AZURE_CALLBACK_URL}:${this.port}/callback`;
const response = await request(`${process.env.AZURE_URL}/api/orchestrators/ImageOrchestrator`, { method: "POST", body: JSON.stringify(object) }).then(r => r.body.json());
const event = new EventEmitter();
this.jobs.set(response.id, event);
return await this.waitForAzure(event);
} else {
// Called from command (not using image API)
const worker = new Worker(path.join(path.dirname(fileURLToPath(import.meta.url)), "../image-runner.js"), {
workerData: object
});
return await this.waitForWorker(worker);
}
}
async handleCommand(data) {
try {
if (data.type === "run") {
const result = await this.run(data.obj);
return result;
} else if (data.type === "reload") {
await this.disconnect();
await this.repopulate();
let amount = 0;
for (const server of this.servers) {
try {
await this.connect(server.server, server.auth);
amount += 1;
} catch (e) {
logger.error(e);
}
}
return amount;
} else if (data.type === "stats") {
return await this.getRunning();
}
} catch (err) {
return { err: typeof err === "string" ? err : err.message };
}
}
shutdown(done) {
done();
}
}
export default ImageWorker;

40
utils/tempimages.js
View file

@ -1,7 +1,9 @@
import * as logger from "../utils/logger.js";
import { readdir, lstat, rm, writeFile } from "fs/promises";
import { readdir, lstat, rm, writeFile, stat } from "fs/promises";
export async function upload(client, ipc, result, context, interaction = false) {
let dirSizeCache;
export async function upload(client, result, context, interaction = false) {
const filename = `${Math.random().toString(36).substring(2, 15)}.${result.name.split(".")[1]}`;
await writeFile(`${process.env.TEMPDIR}/${filename}`, result.file);
const imageURL = `${process.env.TMP_DOMAIN || "https://tmp.projectlounge.pw"}/${filename}`;
@ -34,13 +36,13 @@ export async function upload(client, ipc, result, context, interaction = false)
}));
}
if (process.env.THRESHOLD) {
const size = await ipc.centralStore.get("dirSizeCache") + result.file.length;
await ipc.centralStore.set("dirSizeCache", size);
await removeOldImages(ipc, size);
const size = dirSizeCache + result.file.length;
dirSizeCache = size;
await removeOldImages(size);
}
}
export async function removeOldImages(ipc, size) {
async function removeOldImages(size) {
if (size > process.env.THRESHOLD) {
const files = (await readdir(process.env.TEMPDIR)).map((file) => {
return lstat(`${process.env.TEMPDIR}/${file}`).then((stats) => {
@ -67,6 +69,30 @@ export async function removeOldImages(ipc, size) {
const newSize = oldestFiles.reduce((a, b) => {
return a + b.size;
}, 0);
await ipc.centralStore.set("dirSizeCache", newSize);
dirSizeCache = newSize;
}
}
export async function parseThreshold() {
const matched = process.env.THRESHOLD.match(/(\d+)([KMGT])/);
const sizes = {
K: 1024,
M: 1048576,
G: 1073741824,
T: 1099511627776
};
if (matched && matched[1] && matched[2]) {
process.env.THRESHOLD = matched[1] * sizes[matched[2]];
} else {
logger.error("Invalid THRESHOLD config.");
process.env.THRESHOLD = undefined;
}
const dirstat = (await readdir(process.env.TEMPDIR)).map((file) => {
return stat(`${process.env.TEMPDIR}/${file}`).then((stats) => stats.size);
});
const size = await Promise.all(dirstat);
const reduced = size.reduce((a, b) => {
return a + b;
}, 0);
dirSizeCache = reduced;
}