The image API no longer uses a unique thread for each job (due to image processing using its own thread anyway), added dummy DB driver
This commit is contained in:
parent
7b64c4ca2a
commit
55da0db479
4 changed files with 151 additions and 134 deletions
254
api/index.js
254
api/index.js
|
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
require("dotenv").config();
|
require("dotenv").config();
|
||||||
const os = require("os");
|
const os = require("os");
|
||||||
const { Worker, isMainThread, parentPort } = require("worker_threads");
|
|
||||||
const magick = require("../utils/image.js");
|
const magick = require("../utils/image.js");
|
||||||
const execPromise = require("util").promisify(require("child_process").exec);
|
const execPromise = require("util").promisify(require("child_process").exec);
|
||||||
const net = require("net");
|
const net = require("net");
|
||||||
|
|
@ -10,8 +9,8 @@ const dgram = require("dgram"); // for UDP servers
|
||||||
const socket = dgram.createSocket("udp4"); // Our universal UDP socket, this might cause issues and we may have to use a seperate socket for each connection
|
const socket = dgram.createSocket("udp4"); // Our universal UDP socket, this might cause issues and we may have to use a seperate socket for each connection
|
||||||
|
|
||||||
const start = process.hrtime();
|
const start = process.hrtime();
|
||||||
const log = (msg, thread) => {
|
const log = (msg) => {
|
||||||
console.log(`[${process.hrtime(start)[1] / 1000000}${(thread)?`:${thread}`:""}]\t ${msg}`);
|
console.log(`[${process.hrtime(start)[1] / 1000000}]\t ${msg}`);
|
||||||
};
|
};
|
||||||
|
|
||||||
const jobs = {};
|
const jobs = {};
|
||||||
|
|
@ -19,156 +18,147 @@ const jobs = {};
|
||||||
const queue = [];
|
const queue = [];
|
||||||
// Array of UUIDs
|
// Array of UUIDs
|
||||||
|
|
||||||
if (isMainThread) {
|
const { v4: uuidv4 } = require("uuid");
|
||||||
const { v4: uuidv4 } = require("uuid");
|
let cpuLoad = 0;
|
||||||
let cpuLoad = 0;
|
|
||||||
|
|
||||||
const getAverage = () => {
|
const getAverage = () => {
|
||||||
const cpus = os.cpus();
|
const cpus = os.cpus();
|
||||||
let idle = 0;
|
let idle = 0;
|
||||||
let tick = 0;
|
let tick = 0;
|
||||||
for (const cpu of cpus) {
|
for (const cpu of cpus) {
|
||||||
for (const type in cpu.times) {
|
for (const type in cpu.times) {
|
||||||
tick += cpu.times[type];
|
tick += cpu.times[type];
|
||||||
}
|
|
||||||
idle += cpu.times.idle;
|
|
||||||
}
|
}
|
||||||
return {
|
idle += cpu.times.idle;
|
||||||
idle: idle / cpus.length,
|
}
|
||||||
tick: tick / cpus.length
|
return {
|
||||||
};
|
idle: idle / cpus.length,
|
||||||
|
tick: tick / cpus.length
|
||||||
};
|
};
|
||||||
|
};
|
||||||
|
|
||||||
let measure = getAverage();
|
let measure = getAverage();
|
||||||
setInterval(() => {
|
setInterval(() => {
|
||||||
const newMeasure = getAverage();
|
const newMeasure = getAverage();
|
||||||
const idleDiff = newMeasure.idle - measure.idle;
|
const idleDiff = newMeasure.idle - measure.idle;
|
||||||
const tickDiff = newMeasure.tick - measure.tick;
|
const tickDiff = newMeasure.tick - measure.tick;
|
||||||
cpuLoad = 100 - ~~(100 * idleDiff / tickDiff);
|
cpuLoad = 100 - ~~(100 * idleDiff / tickDiff);
|
||||||
measure = newMeasure;
|
measure = newMeasure;
|
||||||
}, 5000);
|
}, 5000);
|
||||||
|
|
||||||
const MAX_WORKERS = process.env.WORKERS !== "" ? parseInt(process.env.WORKERS) : os.cpus().length * 4; // Completely arbitrary, should usually be some multiple of your amount of cores
|
const MAX_JOBS = process.env.JOBS !== "" && process.env.JOBS !== undefined ? parseInt(process.env.JOBS) : os.cpus().length * 4; // Completely arbitrary, should usually be some multiple of your amount of cores
|
||||||
let workingWorkers = 0;
|
let jobAmount = 0;
|
||||||
|
|
||||||
const acceptJob = (uuid) => {
|
const acceptJob = async (uuid) => {
|
||||||
workingWorkers++;
|
jobAmount++;
|
||||||
queue.shift();
|
queue.shift();
|
||||||
const worker = new Worker(__filename);
|
try {
|
||||||
log(`Spawned worker ${uuid}`);
|
await runJob({
|
||||||
worker.once("error", err => {
|
|
||||||
console.error(`Error on worker ${uuid}:`, err);
|
|
||||||
socket.send(Buffer.concat([Buffer.from([0x2]), Buffer.from(uuid), Buffer.from(err.toString())]), jobs[uuid].port, jobs[uuid].addr);
|
|
||||||
});
|
|
||||||
worker.once("exit", (code) => {
|
|
||||||
workingWorkers--;
|
|
||||||
if (queue.length > 0) {
|
|
||||||
acceptJob(queue[0]);
|
|
||||||
delete jobs[uuid];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (code === 0) {
|
|
||||||
log(`Worker ${uuid} successfully exited`);
|
|
||||||
} else {
|
|
||||||
console.error(`Worker ${uuid} stopped with exit code ${code}`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
worker.postMessage({
|
|
||||||
uuid: uuid,
|
uuid: uuid,
|
||||||
msg: jobs[uuid].msg,
|
msg: jobs[uuid].msg,
|
||||||
addr: jobs[uuid].addr,
|
addr: jobs[uuid].addr,
|
||||||
port: jobs[uuid].port,
|
port: jobs[uuid].port
|
||||||
threadNum: workingWorkers
|
|
||||||
});
|
});
|
||||||
};
|
jobAmount--;
|
||||||
|
if (queue.length > 0) {
|
||||||
|
acceptJob(queue[0]);
|
||||||
|
}
|
||||||
|
delete jobs[uuid];
|
||||||
|
log(`Job ${uuid} has finished`);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`Error on job ${uuid}:`, err);
|
||||||
|
socket.send(Buffer.concat([Buffer.from([0x2]), Buffer.from(uuid), Buffer.from(err.toString())]), jobs[uuid].port, jobs[uuid].addr);
|
||||||
|
jobAmount--;
|
||||||
|
if (queue.length > 0) {
|
||||||
|
acceptJob(queue[0]);
|
||||||
|
}
|
||||||
|
delete jobs[uuid];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const server = dgram.createSocket("udp4"); //Create a UDP server for listening to requests, we dont need tcp
|
const server = dgram.createSocket("udp4"); //Create a UDP server for listening to requests, we dont need tcp
|
||||||
server.on("message", (msg, rinfo) => {
|
server.on("message", (msg, rinfo) => {
|
||||||
const opcode = msg.readUint8(0);
|
const opcode = msg.readUint8(0);
|
||||||
const req = msg.toString().slice(1,msg.length);
|
const req = msg.toString().slice(1,msg.length);
|
||||||
// 0x0 == Cancel job
|
// 0x0 == Cancel job
|
||||||
// 0x1 == Queue job
|
// 0x1 == Queue job
|
||||||
// 0x2 == Get CPU usage
|
// 0x2 == Get CPU usage
|
||||||
if (opcode == 0x0) {
|
if (opcode == 0x0) {
|
||||||
queue.shift();
|
delete queue[queue.indexOf(req) - 1];
|
||||||
delete jobs[req];
|
delete jobs[req];
|
||||||
} else if (opcode == 0x1) {
|
} else if (opcode == 0x1) {
|
||||||
const job = { addr: rinfo.address, port: rinfo.port, msg: req };
|
const job = { addr: rinfo.address, port: rinfo.port, msg: req };
|
||||||
const uuid = uuidv4();
|
const uuid = uuidv4();
|
||||||
jobs[uuid] = job;
|
jobs[uuid] = job;
|
||||||
queue.push(uuid);
|
queue.push(uuid);
|
||||||
|
|
||||||
if (workingWorkers < MAX_WORKERS) {
|
if (jobAmount < MAX_JOBS) {
|
||||||
log(`Got request for job ${job.msg} with id ${uuid}`, job.threadNum);
|
log(`Got request for job ${job.msg} with id ${uuid}`, job.threadNum);
|
||||||
acceptJob(uuid);
|
acceptJob(uuid);
|
||||||
} else {
|
|
||||||
log(`Got request for job ${job.msg} with id ${uuid}, queued in position ${queue.indexOf(uuid)}`, job.threadNum);
|
|
||||||
}
|
|
||||||
|
|
||||||
const newBuffer = Buffer.concat([Buffer.from([0x0]), Buffer.from(uuid)]);
|
|
||||||
socket.send(newBuffer, rinfo.port, rinfo.address);
|
|
||||||
} else if (opcode == 0x2) {
|
|
||||||
socket.send(Buffer.concat([Buffer.from([0x3]), Buffer.from(cpuLoad.toString())]), rinfo.port, rinfo.address);
|
|
||||||
} else {
|
} else {
|
||||||
log("Could not parse message");
|
log(`Got request for job ${job.msg} with id ${uuid}, queued in position ${queue.indexOf(uuid)}`, job.threadNum);
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
|
||||||
server.on("listening", () => {
|
const newBuffer = Buffer.concat([Buffer.from([0x0]), Buffer.from(uuid)]);
|
||||||
const address = server.address();
|
socket.send(newBuffer, rinfo.port, rinfo.address);
|
||||||
log(`server listening ${address.address}:${address.port}`);
|
} else if (opcode == 0x2) {
|
||||||
});
|
socket.send(Buffer.concat([Buffer.from([0x3]), Buffer.from(cpuLoad.toString())]), rinfo.port, rinfo.address);
|
||||||
|
} else {
|
||||||
|
log("Could not parse message");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
server.bind(8080); // ATTENTION: Always going to be bound to 0.0.0.0 !!!
|
server.on("listening", () => {
|
||||||
} else {
|
const address = server.address();
|
||||||
parentPort.once("message", async (job) => {
|
log(`server listening ${address.address}:${address.port}`);
|
||||||
log(`${job.uuid} worker started`, job.threadNum);
|
});
|
||||||
|
|
||||||
const object = JSON.parse(job.msg);
|
server.bind(8080); // ATTENTION: Always going to be bound to 0.0.0.0 !!!
|
||||||
let type;
|
|
||||||
if (object.path) {
|
const runJob = async (job) => {
|
||||||
type = object.type;
|
log(`Job ${job.uuid} started`, job.threadNum);
|
||||||
if (!object.type) {
|
|
||||||
type = await magick.getType(object.path);
|
const object = JSON.parse(job.msg);
|
||||||
}
|
let type;
|
||||||
if (!type) {
|
if (object.path) {
|
||||||
throw new TypeError("Unknown image type");
|
type = object.type;
|
||||||
}
|
if (!object.type) {
|
||||||
object.type = type.split("/")[1];
|
type = await magick.getType(object.path);
|
||||||
if (object.type !== "gif" && object.onlyGIF) throw new TypeError(`Expected a GIF, got ${object.type}`);
|
|
||||||
object.delay = object.delay ? object.delay : 0;
|
|
||||||
}
|
}
|
||||||
|
if (!type) {
|
||||||
|
throw new TypeError("Unknown image type");
|
||||||
|
}
|
||||||
|
object.type = type.split("/")[1];
|
||||||
|
if (object.type !== "gif" && object.onlyGIF) throw new TypeError(`Expected a GIF, got ${object.type}`);
|
||||||
|
object.delay = object.delay ? object.delay : 0;
|
||||||
|
}
|
||||||
|
|
||||||
if (object.type === "gif" && !object.delay) {
|
if (object.type === "gif" && !object.delay) {
|
||||||
const delay = (await execPromise(`ffprobe -v 0 -of csv=p=0 -select_streams v:0 -show_entries stream=r_frame_rate ${object.path}`)).stdout.replace("\n", "");
|
const delay = (await execPromise(`ffprobe -v 0 -of csv=p=0 -select_streams v:0 -show_entries stream=r_frame_rate ${object.path}`)).stdout.replace("\n", "");
|
||||||
object.delay = (100 / delay.split("/")[0]) * delay.split("/")[1];
|
object.delay = (100 / delay.split("/")[0]) * delay.split("/")[1];
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = await magick.run(object, true);
|
const data = await magick.run(object, true);
|
||||||
|
|
||||||
log(`${job.uuid} is done`, job.threadNum);
|
log(`Sending result of job ${job.uuid} back to the bot`, job.threadNum);
|
||||||
const server = net.createServer(function(tcpSocket) {
|
const server = net.createServer(function(tcpSocket) {
|
||||||
tcpSocket.write(Buffer.concat([Buffer.from(type ? type : "image/png"), Buffer.from("\n"), data]), (err) => {
|
tcpSocket.write(Buffer.concat([Buffer.from(type ? type : "image/png"), Buffer.from("\n"), data]), (err) => {
|
||||||
if (err) console.error(err);
|
if (err) console.error(err);
|
||||||
tcpSocket.end(() => {
|
tcpSocket.end(() => {
|
||||||
server.close();
|
server.close();
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
server.listen(job.port, job.addr);
|
|
||||||
// handle address in use errors
|
|
||||||
server.on("error", (e) => {
|
|
||||||
if (e.code === "EADDRINUSE") {
|
|
||||||
console.log("Address in use, retrying...");
|
|
||||||
setTimeout(() => {
|
|
||||||
server.close();
|
|
||||||
server.listen(job.port, job.addr);
|
|
||||||
}, 500);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
socket.send(Buffer.concat([Buffer.from([0x1]), Buffer.from(job.uuid), Buffer.from(job.port.toString())]), job.port, job.addr, () => {
|
|
||||||
socket.close();
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
}
|
server.listen(job.port, job.addr);
|
||||||
|
// handle address in use errors
|
||||||
|
server.on("error", (e) => {
|
||||||
|
if (e.code === "EADDRINUSE") {
|
||||||
|
log("Address in use, retrying...", job.threadNum);
|
||||||
|
setTimeout(() => {
|
||||||
|
server.close();
|
||||||
|
server.listen(job.port, job.addr);
|
||||||
|
}, 500);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
socket.send(Buffer.concat([Buffer.from([0x1]), Buffer.from(job.uuid), Buffer.from(job.port.toString())]), job.port, job.addr);
|
||||||
|
};
|
||||||
|
|
@ -116,5 +116,6 @@
|
||||||
"Open Fortress",
|
"Open Fortress",
|
||||||
"btw I use arch",
|
"btw I use arch",
|
||||||
"Friday Night Funkin'",
|
"Friday Night Funkin'",
|
||||||
"fgsfds"
|
"fgsfds",
|
||||||
|
"Doilus Stage"
|
||||||
]
|
]
|
||||||
26
utils/database/dummy.js
Normal file
26
utils/database/dummy.js
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
// dummy (no-op) database handler
|
||||||
|
const misc = require("../misc.js");
|
||||||
|
|
||||||
|
exports.setup = async () => {};
|
||||||
|
exports.stop = async () => {};
|
||||||
|
exports.fixGuild = async () => {};
|
||||||
|
exports.addCount = async () => {};
|
||||||
|
exports.getCounts = async () => {
|
||||||
|
return {};
|
||||||
|
};
|
||||||
|
exports.disableChannel = async () => {};
|
||||||
|
exports.enableChannel = async () => {};
|
||||||
|
exports.toggleTags = async () => {};
|
||||||
|
exports.setTag = async () => {};
|
||||||
|
exports.removeTag = async () => {};
|
||||||
|
exports.setPrefix = async () => {};
|
||||||
|
exports.addGuild = async (guild) => {
|
||||||
|
return {
|
||||||
|
id: guild.id,
|
||||||
|
tags: misc.tagDefaults,
|
||||||
|
prefix: process.env.PREFIX,
|
||||||
|
disabled: [],
|
||||||
|
tagsDisabled: false,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
exports.getGuild = exports.addGuild;
|
||||||
|
|
@ -66,7 +66,7 @@ const checkImages = async (message) => {
|
||||||
if (message.embeds[0].type === "gifv") {
|
if (message.embeds[0].type === "gifv") {
|
||||||
type = await getImage(message.embeds[0].video.url, message.embeds[0].url, true);
|
type = await getImage(message.embeds[0].video.url, message.embeds[0].url, true);
|
||||||
// then we check for other image types
|
// then we check for other image types
|
||||||
} else if (message.embeds[0].type === "video" || message.embeds[0].type === "image") {
|
} else if ((message.embeds[0].type === "video" || message.embeds[0].type === "image") && message.embeds[0].thumbnail) {
|
||||||
type = await getImage(message.embeds[0].thumbnail.proxy_url, message.embeds[0].thumbnail.url);
|
type = await getImage(message.embeds[0].thumbnail.proxy_url, message.embeds[0].thumbnail.url);
|
||||||
// finally we check both possible image fields for "generic" embeds
|
// finally we check both possible image fields for "generic" embeds
|
||||||
} else if (message.embeds[0].type === "rich") {
|
} else if (message.embeds[0].type === "rich") {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue