mrmBot-Matrix/utils/image.js

285 lines
9.3 KiB
JavaScript
Raw Normal View History

const magick = require("../build/Release/image.node");
const { Worker } = require("worker_threads");
const fetch = require("node-fetch");
const fs = require("fs");
const net = require("net");
const fileType = require("file-type");
const path = require("path");
const { EventEmitter } = require("events");
const logger = require("./logger.js");
const formats = ["image/jpeg", "image/png", "image/webp", "image/gif", "video/mp4", "video/webm", "video/mov"];
const jobs = {};
exports.connections = [];
const statuses = {};
exports.servers = JSON.parse(fs.readFileSync("./servers.json", { encoding: "utf8" })).image;
2020-12-26 02:27:45 +00:00
const chooseServer = async (ideal) => {
if (ideal.length === 0) throw "No available servers";
const sorted = ideal.sort((a, b) => {
return b.load - a.load;
2020-12-26 02:27:45 +00:00
});
return sorted[0];
};
exports.repopulate = async () => {
const data = await fs.promises.readFile("./servers.json", { encoding: "utf8" });
this.servers = JSON.parse(data).image;
return;
};
exports.getStatus = () => {
return new Promise((resolve, reject) => {
let serversLeft = this.connections.length;
const statuses = [];
const timeout = setTimeout(() => {
resolve(statuses);
}, 5000);
for (const connection of this.connections) {
if (!connection.remoteAddress) continue;
fetch(`http://${connection.remoteAddress}:8081/running`).then(statusRequest => statusRequest.json()).then((status) => {
serversLeft--;
statuses.push(status);
if (!serversLeft) {
clearTimeout(timeout);
resolve(statuses);
}
return;
}).catch(e => reject(e));
}
});
};
exports.connect = (server) => {
2020-12-26 02:27:45 +00:00
return new Promise((resolve, reject) => {
const connection = net.createConnection(8080, server);
2020-12-26 02:27:45 +00:00
const timeout = setTimeout(() => {
const connectionIndex = this.connections.indexOf(connection);
if (connectionIndex < 0) delete this.connections[connectionIndex];
reject(`Failed to connect to ${server}`);
2020-12-26 02:27:45 +00:00
}, 5000);
connection.once("connect", () => {
clearTimeout(timeout);
});
connection.on("data", async (msg) => {
2020-12-26 02:27:45 +00:00
const opcode = msg.readUint8(0);
const req = msg.slice(37, msg.length);
const uuid = msg.slice(1, 37).toString();
if (opcode === 0x00) { // Job queued
if (jobs[req]) {
jobs[req].event.emit("uuid", uuid);
}
} else if (opcode === 0x01) { // Job completed successfully
// the image API sends all job responses over the same socket; make sure this is ours
if (jobs[uuid]) {
const imageReq = await fetch(`http://${connection.remoteAddress}:8081/image?id=${uuid}`);
const image = await imageReq.buffer();
// The response data is given as the file extension/ImageMagick type of the image (e.g. "png"), followed
// by a newline, followed by the image data.
jobs[uuid].event.emit("image", image, imageReq.headers.get("ext"));
}
} else if (opcode === 0x02) { // Job errored
if (jobs[uuid]) {
jobs[uuid].event.emit("error", new Error(req));
}
} else if (opcode === 0x03) {
// we use the uuid part here because queue info requests don't respond with one
statuses[`${connection.remoteAddress}:${connection.remotePort}`] = parseInt(uuid);
}
});
connection.on("error", (e) => {
logger.error(e.toString());
});
connection.once("close", () => {
for (const uuid of Object.keys(jobs)) {
if (jobs[uuid].addr === connection.remoteAddress) jobs[uuid].event.emit("error", "Job ended prematurely due to a closed connection; please run your image job again");
}
});
this.connections.push(connection);
resolve();
});
};
exports.disconnect = async () => {
for (const connection of this.connections) {
connection.destroy();
}
for (const uuid of Object.keys(jobs)) {
jobs[uuid].event.emit("error", "Job ended prematurely (not really an error; just run your image job again)");
delete jobs[uuid];
}
this.connections = [];
return;
};
const getIdeal = () => {
2021-03-16 03:29:48 +00:00
return new Promise((resolve, reject) => {
let serversLeft = this.connections.length;
const idealServers = [];
const timeout = setTimeout(async () => {
try {
const server = await chooseServer(idealServers);
resolve(this.connections.find(val => val.remoteAddress === server.addr));
} catch (e) {
reject(e);
}
}, 5000);
for (const connection of this.connections) {
if (!connection.remoteAddress || connection.destroyed) {
serversLeft--;
continue;
}
fetch(`http://${connection.remoteAddress}:8081/status`).then(statusRequest => statusRequest.text()).then((status) => {
2020-12-26 02:27:45 +00:00
serversLeft--;
idealServers.push({
addr: connection.remoteAddress,
load: parseInt(status)
2020-12-26 02:27:45 +00:00
});
return;
}).then(async () => {
2020-12-26 02:27:45 +00:00
if (!serversLeft) {
clearTimeout(timeout);
const server = await chooseServer(idealServers);
resolve(this.connections.find(val => val.remoteAddress === server.addr));
2020-12-26 02:27:45 +00:00
}
}).catch(e => reject(e));
}
if (!serversLeft) {
clearTimeout(timeout);
chooseServer(idealServers).then(server => {
resolve(this.connections.find(val => val.remoteAddress === server.addr));
2021-03-16 03:29:48 +00:00
}).catch(e => reject(e));
}
});
};
const start = (object, num) => {
2021-03-16 03:29:48 +00:00
return getIdeal().then(async (currentServer) => {
const data = Buffer.concat([Buffer.from([0x01 /* queue job */]), Buffer.from(num.length.toString()), Buffer.from(num), Buffer.from(JSON.stringify(object))]);
return new Promise((resolve, reject) => {
currentServer.write(data, (err) => {
if (err) {
if (err.code === "EPIPE") {
logger.log(`Lost connection to ${currentServer.remoteAddress}, attempting to reconnect...`);
2021-03-16 03:29:48 +00:00
currentServer.connect(8080, currentServer.remoteAddress, () => {
const res = start(object, num);
2021-03-16 03:29:48 +00:00
reject(res); // this is done to differentiate the result from a step
});
} else {
reject(err);
}
2021-03-16 03:29:48 +00:00
} else {
resolve(currentServer.remoteAddress);
}
});
2021-03-16 03:29:48 +00:00
});
}).then((addr) => {
2021-03-16 03:29:48 +00:00
const event = new EventEmitter();
return new Promise((resolve) => {
event.once("uuid", (uuid) => resolve({ event, uuid, addr }));
jobs[num] = { event, addr };
2021-03-16 03:29:48 +00:00
});
}, (result) => {
throw result;
2021-03-16 03:29:48 +00:00
}).then(data => {
delete jobs[num];
jobs[data.uuid] = { event: data.event, addr: data.addr };
2021-03-16 03:29:48 +00:00
return { uuid: data.uuid, event: data.event };
2020-12-26 02:27:45 +00:00
});
};
exports.check = (cmd) => {
return magick[cmd] ? true : false;
};
exports.getType = async (image) => {
if (!image.startsWith("http")) {
const imageType = await fileType.fromFile(image);
if (imageType && formats.includes(imageType.mime)) {
return imageType.mime;
}
return undefined;
}
let type;
const controller = new AbortController(); // eslint-disable-line no-undef
const timeout = setTimeout(() => {
controller.abort();
}, 25000);
try {
const imageRequest = await fetch(image, { signal: controller.signal, headers: {
"Range": "bytes=0-1023"
}});
clearTimeout(timeout);
const size = imageRequest.headers.has("Content-Range") ? imageRequest.headers.get("Content-Range").split("/")[1] : imageRequest.headers.get("Content-Length");
if (parseInt(size) > 20971520) {
2021-05-07 03:01:30 +00:00
type = "large";
return type;
}
const imageBuffer = await imageRequest.buffer();
const imageType = await fileType.fromBuffer(imageBuffer);
if (imageType && formats.includes(imageType.mime)) {
type = imageType.mime;
}
} catch (error) {
if (error.name === "AbortError") {
throw Error("Timed out");
} else {
throw error;
}
} finally {
clearTimeout(timeout);
}
return type;
};
exports.run = object => {
2021-03-16 03:29:48 +00:00
return new Promise((resolve, reject) => {
if (process.env.API === "true") {
2021-03-16 03:29:48 +00:00
// Connect to best image server
const num = Math.floor(Math.random() * 100000).toString().slice(0, 5);
const timeout = setTimeout(() => {
if (jobs[num]) delete jobs[num];
reject("the image request timed out after 25 seconds. Try uploading your image elsewhere.");
2021-03-16 03:29:48 +00:00
}, 25000);
start(object, num).catch(err => { // incredibly hacky code incoming
clearTimeout(timeout);
2021-03-16 03:29:48 +00:00
if (err instanceof Error) return reject(err);
return err;
}).then((data) => {
clearTimeout(timeout);
if (!data.event) reject("Not connected to image server");
2021-03-16 03:29:48 +00:00
data.event.once("image", (image, type) => {
delete jobs[data.uuid];
const payload = {
2021-03-16 03:29:48 +00:00
// Take just the image data
buffer: image,
type: type
};
resolve(payload);
});
2021-03-16 03:29:48 +00:00
data.event.once("error", (err) => {
reject(err);
});
return;
2021-03-16 03:29:48 +00:00
}).catch(err => reject(err));
} else {
// Called from command (not using image API)
const worker = new Worker(path.join(__dirname, "image-runner.js"), {
workerData: object
});
worker.once("message", (data) => {
resolve({
buffer: Buffer.from([...data.buffer]),
type: data.fileExtension
});
});
worker.once("error", reject);
}
});
};