2022-08-11 16:46:56 +00:00
|
|
|
import { request } from "undici";
|
2021-08-19 14:19:14 +00:00
|
|
|
import fs from "fs";
|
2022-09-21 05:05:03 +00:00
|
|
|
import path from "path";
|
|
|
|
import { fileURLToPath } from "url";
|
|
|
|
import { Worker } from "worker_threads";
|
|
|
|
import { createRequire } from "module";
|
2022-02-02 17:01:33 +00:00
|
|
|
import { fileTypeFromBuffer, fileTypeFromFile } from "file-type";
|
2022-09-21 05:05:03 +00:00
|
|
|
import * as logger from "./logger.js";
|
|
|
|
import ImageConnection from "./imageConnection.js";
|
2020-10-18 21:53:35 +00:00
|
|
|
|
2022-09-21 05:05:03 +00:00
|
|
|
// only requiring this to work around an issue regarding worker threads
|
|
|
|
const nodeRequire = createRequire(import.meta.url);
|
|
|
|
if (!process.env.API_TYPE || process.env.API_TYPE === "none") {
|
|
|
|
nodeRequire(`../build/${process.env.DEBUG && process.env.DEBUG === "true" ? "Debug" : "Release"}/image.node`);
|
|
|
|
}
|
2021-01-18 20:11:28 +00:00
|
|
|
|
2022-09-21 05:05:03 +00:00
|
|
|
const formats = ["image/jpeg", "image/png", "image/webp", "image/gif", "video/mp4", "video/webm", "video/quicktime"];
|
2021-08-19 14:19:14 +00:00
|
|
|
export const connections = new Map();
|
2022-09-21 05:05:03 +00:00
|
|
|
export let servers = process.env.API_TYPE === "ws" ? JSON.parse(fs.readFileSync(new URL("../config/servers.json", import.meta.url), { encoding: "utf8" })).image : null;
|
Class commands, improved sharding, and many other changes (#88)
* Load commands recursively
* Sort commands
* Missed a couple of spots
* missed even more spots apparently
* Ported commands in "fun" category to new class-based format, added babel eslint plugin
* Ported general commands, removed old/unneeded stuff, replaced moment with day, many more fixes I lost track of
* Missed a spot
* Removed unnecessary abort-controller package, add deprecation warning for mongo database
* Added imagereload, clarified premature end message
* Fixed docker-compose path issue, added total bot uptime to stats, more fixes for various parts
* Converted image commands into classes, fixed reload, ignore another WS event, cleaned up command handler and image runner
* Converted music/soundboard commands to class format
* Cleanup unnecessary logs
* awful tag command class port
* I literally somehow just learned that you can leave out the constructor in classes
* Pass client directly to commands/events, cleaned up command handler
* Migrated bot to eris-sharder, fixed some error handling stuff
* Remove unused modules
* Fixed type returning
* Switched back to Eris stable
* Some fixes and cleanup
* might wanna correct this
* Implement image command ratelimiting
* Added Bot token prefix, added imagestats, added running endpoint to API
2021-04-12 16:16:12 +00:00
|
|
|
|
2021-08-19 14:19:14 +00:00
|
|
|
export async function getType(image, extraReturnTypes) {
|
2020-11-26 17:48:19 +00:00
|
|
|
if (!image.startsWith("http")) {
|
2022-02-02 17:01:33 +00:00
|
|
|
const imageType = await fileTypeFromFile(image);
|
2020-11-26 17:48:19 +00:00
|
|
|
if (imageType && formats.includes(imageType.mime)) {
|
|
|
|
return imageType.mime;
|
|
|
|
}
|
|
|
|
return undefined;
|
|
|
|
}
|
|
|
|
let type;
|
2022-08-11 16:46:56 +00:00
|
|
|
const controller = new AbortController();
|
2020-11-26 17:48:19 +00:00
|
|
|
const timeout = setTimeout(() => {
|
|
|
|
controller.abort();
|
2022-08-11 16:46:56 +00:00
|
|
|
}, 3000);
|
2020-11-26 17:48:19 +00:00
|
|
|
try {
|
2022-08-11 16:46:56 +00:00
|
|
|
const imageRequest = await request(image, {
|
|
|
|
signal: controller.signal,
|
|
|
|
method: "HEAD"
|
2021-05-22 15:10:42 +00:00
|
|
|
});
|
2021-01-04 16:29:18 +00:00
|
|
|
clearTimeout(timeout);
|
2022-08-11 16:46:56 +00:00
|
|
|
const size = imageRequest.headers["content-range"] ? imageRequest.headers["content-range"].split("/")[1] : imageRequest.headers["content-length"];
|
2021-06-28 22:59:05 +00:00
|
|
|
if (parseInt(size) > 26214400 && extraReturnTypes) { // 25 MB
|
2021-05-07 03:01:30 +00:00
|
|
|
type = "large";
|
|
|
|
return type;
|
|
|
|
}
|
2022-08-11 16:46:56 +00:00
|
|
|
const typeHeader = imageRequest.headers["content-type"];
|
|
|
|
if (typeHeader) {
|
|
|
|
type = typeHeader;
|
|
|
|
} else {
|
|
|
|
const timeout = setTimeout(() => {
|
|
|
|
controller.abort();
|
|
|
|
}, 3000);
|
|
|
|
const bufRequest = await request(image, {
|
|
|
|
signal: controller.signal,
|
|
|
|
headers: {
|
|
|
|
range: "bytes=0-1023"
|
|
|
|
}
|
|
|
|
});
|
|
|
|
clearTimeout(timeout);
|
|
|
|
const imageBuffer = await bufRequest.body.arrayBuffer();
|
|
|
|
const imageType = await fileTypeFromBuffer(imageBuffer);
|
|
|
|
if (imageType && formats.includes(imageType.mime)) {
|
|
|
|
type = imageType.mime;
|
|
|
|
}
|
2020-11-26 17:48:19 +00:00
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
if (error.name === "AbortError") {
|
|
|
|
throw Error("Timed out");
|
|
|
|
} else {
|
|
|
|
throw error;
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
clearTimeout(timeout);
|
|
|
|
}
|
|
|
|
return type;
|
2021-08-19 14:19:14 +00:00
|
|
|
}
|
2022-09-21 05:05:03 +00:00
|
|
|
|
|
|
|
function connect(server, auth) {
|
|
|
|
const connection = new ImageConnection(server, auth);
|
|
|
|
connections.set(server, connection);
|
|
|
|
}
|
|
|
|
|
|
|
|
function disconnect() {
|
|
|
|
for (const connection of connections.values()) {
|
|
|
|
connection.close();
|
|
|
|
}
|
|
|
|
connections.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
async function repopulate() {
|
|
|
|
const data = await fs.promises.readFile(new URL("../config/servers.json", import.meta.url), { encoding: "utf8" });
|
|
|
|
servers = JSON.parse(data).image;
|
|
|
|
}
|
|
|
|
|
|
|
|
export async function reloadImageConnections() {
|
|
|
|
disconnect();
|
|
|
|
await repopulate();
|
|
|
|
let amount = 0;
|
|
|
|
for (const server of servers) {
|
|
|
|
try {
|
|
|
|
connect(server.server, server.auth);
|
|
|
|
amount += 1;
|
|
|
|
} catch (e) {
|
|
|
|
logger.error(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return amount;
|
|
|
|
}
|
|
|
|
|
|
|
|
function chooseServer(ideal) {
|
|
|
|
if (ideal.length === 0) throw "No available servers";
|
|
|
|
const sorted = ideal.sort((a, b) => {
|
|
|
|
return a.load - b.load;
|
|
|
|
});
|
|
|
|
return sorted[0];
|
|
|
|
}
|
|
|
|
|
|
|
|
async function getIdeal(object) {
|
|
|
|
const idealServers = [];
|
|
|
|
for (const [address, connection] of connections) {
|
|
|
|
if (connection.conn.readyState !== 0 && connection.conn.readyState !== 1) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (object.params.type && !connection.formats[object.cmd]?.includes(object.params.type)) continue;
|
|
|
|
idealServers.push({
|
|
|
|
addr: address,
|
|
|
|
load: await connection.getCount()
|
|
|
|
});
|
|
|
|
}
|
|
|
|
const server = chooseServer(idealServers);
|
|
|
|
return connections.get(server.addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
function waitForWorker(worker) {
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
worker.once("message", (data) => {
|
|
|
|
resolve({
|
|
|
|
buffer: Buffer.from([...data.buffer]),
|
|
|
|
type: data.fileExtension
|
|
|
|
});
|
|
|
|
});
|
|
|
|
worker.once("error", reject);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
export async function runImageJob(params) {
|
|
|
|
if (process.env.API_TYPE === "ws") {
|
|
|
|
for (let i = 0; i < 3; i++) {
|
|
|
|
const currentServer = await getIdeal(params);
|
|
|
|
try {
|
|
|
|
await currentServer.queue(BigInt(params.id), params);
|
|
|
|
await currentServer.wait(BigInt(params.id));
|
|
|
|
const output = await currentServer.getOutput(params.id);
|
|
|
|
return output;
|
|
|
|
} catch (e) {
|
|
|
|
if (i < 2 && e === "Request ended prematurely due to a closed connection") {
|
|
|
|
continue;
|
|
|
|
} else {
|
|
|
|
if (e === "No available servers" && i >= 2) throw "Request ended prematurely due to a closed connection";
|
|
|
|
throw e;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Called from command (not using image API)
|
|
|
|
const worker = new Worker(path.join(path.dirname(fileURLToPath(import.meta.url)), "./image-runner.js"), {
|
|
|
|
workerData: params
|
|
|
|
});
|
|
|
|
return await waitForWorker(worker);
|
|
|
|
}
|
|
|
|
}
|