Switched image API from raw TCP to websockets, removed selective image reading code
This commit is contained in:
		
							parent
							
								
									c2e13c26b6
								
							
						
					
					
						commit
						a0a32c31b9
					
				
					 7 changed files with 133 additions and 138 deletions
				
			
		
							
								
								
									
										141
									
								
								api/index.js
									
										
									
									
									
								
							
							
						
						
									
										141
									
								
								api/index.js
									
										
									
									
									
								
							| 
						 | 
					@ -1,12 +1,10 @@
 | 
				
			||||||
// code originally provided by tzlil
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
require("dotenv").config();
 | 
					require("dotenv").config();
 | 
				
			||||||
const os = require("os");
 | 
					const os = require("os");
 | 
				
			||||||
//const { run } = require("../utils/image-runner.js");
 | 
					//const { run } = require("../utils/image-runner.js");
 | 
				
			||||||
const { Worker } = require("worker_threads");
 | 
					const { Worker } = require("worker_threads");
 | 
				
			||||||
const path = require("path");
 | 
					const path = require("path");
 | 
				
			||||||
const net = require("net");
 | 
					 | 
				
			||||||
const http = require("http");
 | 
					const http = require("http");
 | 
				
			||||||
 | 
					const WebSocket = require("ws");
 | 
				
			||||||
 | 
					
 | 
				
			||||||
const start = process.hrtime();
 | 
					const start = process.hrtime();
 | 
				
			||||||
const log = (msg, jobNum) => {
 | 
					const log = (msg, jobNum) => {
 | 
				
			||||||
| 
						 | 
					@ -23,7 +21,7 @@ class JobCache extends Map {
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
const jobs = new JobCache();
 | 
					const jobs = new JobCache();
 | 
				
			||||||
// Should look like UUID : { addr : "someaddr", port: someport, msg: "request" }
 | 
					// Should look like UUID : { msg: "request", num: <job number> }
 | 
				
			||||||
const queue = [];
 | 
					const queue = [];
 | 
				
			||||||
// Array of UUIDs
 | 
					// Array of UUIDs
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -45,7 +43,7 @@ const acceptJob = (uuid, sock) => {
 | 
				
			||||||
  }).catch((err) => {
 | 
					  }).catch((err) => {
 | 
				
			||||||
    console.error(`Error on job ${uuid}:`, err);
 | 
					    console.error(`Error on job ${uuid}:`, err);
 | 
				
			||||||
    jobs.delete(uuid);
 | 
					    jobs.delete(uuid);
 | 
				
			||||||
    sock.write(Buffer.concat([Buffer.from([0x2]), Buffer.from(uuid), Buffer.from(err.message)]));
 | 
					    sock.send(Buffer.concat([Buffer.from([0x2]), Buffer.from(uuid), Buffer.from(err.message)]));
 | 
				
			||||||
  }).finally(() => {
 | 
					  }).finally(() => {
 | 
				
			||||||
    jobAmount--;
 | 
					    jobAmount--;
 | 
				
			||||||
    if (queue.length > 0) {
 | 
					    if (queue.length > 0) {
 | 
				
			||||||
| 
						 | 
					@ -54,16 +52,68 @@ const acceptJob = (uuid, sock) => {
 | 
				
			||||||
  });
 | 
					  });
 | 
				
			||||||
};
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
const httpServer = http.createServer((req, res) => {
 | 
					const wss = new WebSocket.Server({ clientTracking: true, noServer: true });
 | 
				
			||||||
  if (req.method !== "GET") {
 | 
					
 | 
				
			||||||
 | 
					wss.on("connection", (ws, request) => {
 | 
				
			||||||
 | 
					  log(`WS client ${request.socket.remoteAddress}:${request.socket.remotePort} has connected`);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  ws.on("error", (err) => {
 | 
				
			||||||
 | 
					    console.error(err);
 | 
				
			||||||
 | 
					  });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  ws.on("message", (msg) => {
 | 
				
			||||||
 | 
					    const opcode = msg.readUint8(0);
 | 
				
			||||||
 | 
					    const req = msg.toString().slice(1,msg.length);
 | 
				
			||||||
 | 
					    console.log(req);
 | 
				
			||||||
 | 
					    // 0x00 == Cancel job
 | 
				
			||||||
 | 
					    // 0x01 == Queue job
 | 
				
			||||||
 | 
					    if (opcode == 0x00) {
 | 
				
			||||||
 | 
					      delete queue[queue.indexOf(req) - 1];
 | 
				
			||||||
 | 
					      jobs.delete(req);
 | 
				
			||||||
 | 
					    } else if (opcode == 0x01) {
 | 
				
			||||||
 | 
					      const length = parseInt(req.slice(0, 1));
 | 
				
			||||||
 | 
					      const num = req.slice(1, length + 1);
 | 
				
			||||||
 | 
					      const obj = req.slice(length + 1);
 | 
				
			||||||
 | 
					      const job = { msg: obj, num: jobAmount };
 | 
				
			||||||
 | 
					      const uuid = uuidv4();
 | 
				
			||||||
 | 
					      jobs.set(uuid, job);
 | 
				
			||||||
 | 
					      queue.push(uuid);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      const newBuffer = Buffer.concat([Buffer.from([0x00]), Buffer.from(uuid), Buffer.from(num)]);
 | 
				
			||||||
 | 
					      ws.send(newBuffer);
 | 
				
			||||||
 | 
					  
 | 
				
			||||||
 | 
					      if (jobAmount < MAX_JOBS) {
 | 
				
			||||||
 | 
					        log(`Got WS request for job ${job.msg} with id ${uuid}`, job.num);
 | 
				
			||||||
 | 
					        acceptJob(uuid, ws);
 | 
				
			||||||
 | 
					      } else {
 | 
				
			||||||
 | 
					        log(`Got WS request for job ${job.msg} with id ${uuid}, queued in position ${queue.indexOf(uuid)}`, job.num);
 | 
				
			||||||
 | 
					      }
 | 
				
			||||||
 | 
					    } else {
 | 
				
			||||||
 | 
					      log("Could not parse WS message");
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					  });
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  ws.on("close", () => {
 | 
				
			||||||
 | 
					    log(`WS client ${request.socket.remoteAddress}:${request.socket.remotePort} has disconnected`);
 | 
				
			||||||
 | 
					  });
 | 
				
			||||||
 | 
					});
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					wss.on("error", (err) => {
 | 
				
			||||||
 | 
					  console.error("A WS error occurred: ", err);
 | 
				
			||||||
 | 
					});
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					const httpServer = http.createServer();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					httpServer.on("request", async (req, res) => {
 | 
				
			||||||
 | 
					  if (req.method !== "GET" && req.method !== "POST") {
 | 
				
			||||||
    res.statusCode = 405;
 | 
					    res.statusCode = 405;
 | 
				
			||||||
    return res.end("405 Method Not Allowed");
 | 
					    return res.end("405 Method Not Allowed");
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
  const reqUrl = new URL(req.url, `http://${req.headers.host}`);
 | 
					  const reqUrl = new URL(req.url, `http://${req.headers.host}`);
 | 
				
			||||||
  if (reqUrl.pathname === "/status") {
 | 
					  if (reqUrl.pathname === "/status" && req.method === "GET") {
 | 
				
			||||||
    log(`Sending server status to ${req.socket.remoteAddress}:${req.socket.remotePort} via HTTP`);
 | 
					    log(`Sending server status to ${req.socket.remoteAddress}:${req.socket.remotePort} via HTTP`);
 | 
				
			||||||
    return res.end(Buffer.from((MAX_JOBS - jobAmount).toString()));
 | 
					    return res.end(Buffer.from((MAX_JOBS - jobAmount).toString()));
 | 
				
			||||||
  } else if (reqUrl.pathname === "/running") {
 | 
					  } else if (reqUrl.pathname === "/running" && req.method === "GET") {
 | 
				
			||||||
    log(`Sending currently running jobs to ${req.socket.remoteAddress}:${req.socket.remotePort} via HTTP`);
 | 
					    log(`Sending currently running jobs to ${req.socket.remoteAddress}:${req.socket.remotePort} via HTTP`);
 | 
				
			||||||
    const keys = jobs.keys();
 | 
					    const keys = jobs.keys();
 | 
				
			||||||
    const newObject = { queued: queue.length, runningJobs: jobAmount, max: MAX_JOBS };
 | 
					    const newObject = { queued: queue.length, runningJobs: jobAmount, max: MAX_JOBS };
 | 
				
			||||||
| 
						 | 
					@ -79,7 +129,7 @@ const httpServer = http.createServer((req, res) => {
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
    return res.end(JSON.stringify(newObject));
 | 
					    return res.end(JSON.stringify(newObject));
 | 
				
			||||||
  } else if (reqUrl.pathname === "/image") {
 | 
					  } else if (reqUrl.pathname === "/image" && req.method === "GET") {
 | 
				
			||||||
    if (!reqUrl.searchParams.has("id")) {
 | 
					    if (!reqUrl.searchParams.has("id")) {
 | 
				
			||||||
      res.statusCode = 400;
 | 
					      res.statusCode = 400;
 | 
				
			||||||
      return res.end("400 Bad Request");
 | 
					      return res.end("400 Bad Request");
 | 
				
			||||||
| 
						 | 
					@ -102,64 +152,24 @@ const httpServer = http.createServer((req, res) => {
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
});
 | 
					});
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					httpServer.on("upgrade", (req, sock, head) => {
 | 
				
			||||||
 | 
					  const reqUrl = new URL(req.url, `http://${req.headers.host}`);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					  if (reqUrl.pathname === "/sock") {
 | 
				
			||||||
 | 
					    wss.handleUpgrade(req, sock, head, (ws) => {
 | 
				
			||||||
 | 
					      wss.emit("connection", ws, req);
 | 
				
			||||||
 | 
					    });
 | 
				
			||||||
 | 
					  } else {
 | 
				
			||||||
 | 
					    sock.destroy();
 | 
				
			||||||
 | 
					  }
 | 
				
			||||||
 | 
					});
 | 
				
			||||||
 | 
					
 | 
				
			||||||
httpServer.on("error", (e) => {
 | 
					httpServer.on("error", (e) => {
 | 
				
			||||||
  console.error("An HTTP error occurred: ", e);
 | 
					  console.error("An HTTP error occurred: ", e);
 | 
				
			||||||
});
 | 
					});
 | 
				
			||||||
 | 
					
 | 
				
			||||||
httpServer.listen(8081, () => {
 | 
					httpServer.listen(8080, () => {
 | 
				
			||||||
  log("HTTP listening on port 8081");
 | 
					  log("HTTP listening on port 8080");
 | 
				
			||||||
});
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
const server = net.createServer((sock) => { // Create a TCP socket/server to listen to requests
 | 
					 | 
				
			||||||
  log(`TCP client ${sock.remoteAddress}:${sock.remotePort} has connected`);
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
  sock.on("error", (e) => {
 | 
					 | 
				
			||||||
    console.error(e);
 | 
					 | 
				
			||||||
  });
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
  sock.on("data", (msg) => {
 | 
					 | 
				
			||||||
    const opcode = msg.readUint8(0);
 | 
					 | 
				
			||||||
    const req = msg.toString().slice(1,msg.length);
 | 
					 | 
				
			||||||
    console.log(req);
 | 
					 | 
				
			||||||
    // 0x00 == Cancel job
 | 
					 | 
				
			||||||
    // 0x01 == Queue job
 | 
					 | 
				
			||||||
    if (opcode == 0x00) {
 | 
					 | 
				
			||||||
      delete queue[queue.indexOf(req) - 1];
 | 
					 | 
				
			||||||
      jobs.delete(req);
 | 
					 | 
				
			||||||
    } else if (opcode == 0x01) {
 | 
					 | 
				
			||||||
      const length = parseInt(req.slice(0, 1));
 | 
					 | 
				
			||||||
      const num = req.slice(1, length + 1);
 | 
					 | 
				
			||||||
      const obj = req.slice(length + 1);
 | 
					 | 
				
			||||||
      const job = { addr: sock.remoteAddress, port: sock.remotePort, msg: obj, num: jobAmount };
 | 
					 | 
				
			||||||
      const uuid = uuidv4();
 | 
					 | 
				
			||||||
      jobs.set(uuid, job);
 | 
					 | 
				
			||||||
      queue.push(uuid);
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
      const newBuffer = Buffer.concat([Buffer.from([0x00]), Buffer.from(uuid), Buffer.from(num)]);
 | 
					 | 
				
			||||||
      sock.write(newBuffer);
 | 
					 | 
				
			||||||
  
 | 
					 | 
				
			||||||
      if (jobAmount < MAX_JOBS) {
 | 
					 | 
				
			||||||
        log(`Got TCP request for job ${job.msg} with id ${uuid}`, job.num);
 | 
					 | 
				
			||||||
        acceptJob(uuid, sock);
 | 
					 | 
				
			||||||
      } else {
 | 
					 | 
				
			||||||
        log(`Got TCP request for job ${job.msg} with id ${uuid}, queued in position ${queue.indexOf(uuid)}`, job.num);
 | 
					 | 
				
			||||||
      }
 | 
					 | 
				
			||||||
    } else {
 | 
					 | 
				
			||||||
      log("Could not parse TCP message");
 | 
					 | 
				
			||||||
    }
 | 
					 | 
				
			||||||
  });
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
  sock.on("end", () => {
 | 
					 | 
				
			||||||
    log(`TCP client ${sock.remoteAddress}:${sock.remotePort} has disconnected`);
 | 
					 | 
				
			||||||
  });
 | 
					 | 
				
			||||||
});
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
server.on("error", (e) => {
 | 
					 | 
				
			||||||
  console.error("A TCP error occurred: ", e);
 | 
					 | 
				
			||||||
});
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
server.listen(8080, () => {
 | 
					 | 
				
			||||||
  log("TCP listening on port 8080");
 | 
					 | 
				
			||||||
});
 | 
					});
 | 
				
			||||||
 | 
					
 | 
				
			||||||
const runJob = (job, sock) => {
 | 
					const runJob = (job, sock) => {
 | 
				
			||||||
| 
						 | 
					@ -187,8 +197,7 @@ const runJob = (job, sock) => {
 | 
				
			||||||
      jobObject.data = data.buffer;
 | 
					      jobObject.data = data.buffer;
 | 
				
			||||||
      jobObject.ext = data.fileExtension;
 | 
					      jobObject.ext = data.fileExtension;
 | 
				
			||||||
      jobs.set(job.uuid, jobObject);
 | 
					      jobs.set(job.uuid, jobObject);
 | 
				
			||||||
      sock.write(Buffer.concat([Buffer.from([0x1]), Buffer.from(job.uuid)]), (e) => {
 | 
					      sock.send(Buffer.concat([Buffer.from([0x1]), Buffer.from(job.uuid)]), () => {
 | 
				
			||||||
        if (e) return reject(e);
 | 
					 | 
				
			||||||
        return resolve();
 | 
					        return resolve();
 | 
				
			||||||
      });
 | 
					      });
 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -108,7 +108,7 @@ class ImageCommand extends Command {
 | 
				
			||||||
        name: `${this.constructor.command}.${type}`
 | 
					        name: `${this.constructor.command}.${type}`
 | 
				
			||||||
      };
 | 
					      };
 | 
				
			||||||
    } catch (e) {
 | 
					    } catch (e) {
 | 
				
			||||||
      if (e.toString().includes("Not connected to image server")) return "I'm still trying to connect to the image servers. Please wait a little bit.";
 | 
					      if (e.toString().includes("Not connected to image server")) return "I can't seem to contact the image servers, they might be down or still trying to start up. Please wait a little bit.";
 | 
				
			||||||
      throw e;
 | 
					      throw e;
 | 
				
			||||||
    } finally {
 | 
					    } finally {
 | 
				
			||||||
      if (status && await this.client.getMessage(status.channel.id, status.id).catch(() => undefined)) await status.delete();
 | 
					      if (status && await this.client.getMessage(status.channel.id, status.id).catch(() => undefined)) await status.delete();
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -2,7 +2,7 @@ const fs = require("fs");
 | 
				
			||||||
const database = require("../utils/database.js");
 | 
					const database = require("../utils/database.js");
 | 
				
			||||||
const logger = require("../utils/logger.js");
 | 
					const logger = require("../utils/logger.js");
 | 
				
			||||||
const collections = require("../utils/collections.js");
 | 
					const collections = require("../utils/collections.js");
 | 
				
			||||||
const commands = [...collections.aliases.keys(), ...collections.commands.keys()];
 | 
					//const commands = [...collections.aliases.keys(), ...collections.commands.keys()];
 | 
				
			||||||
 | 
					
 | 
				
			||||||
// run when someone sends a message
 | 
					// run when someone sends a message
 | 
				
			||||||
module.exports = async (client, cluster, ipc, message) => {
 | 
					module.exports = async (client, cluster, ipc, message) => {
 | 
				
			||||||
| 
						 | 
					@ -12,16 +12,6 @@ module.exports = async (client, cluster, ipc, message) => {
 | 
				
			||||||
  // don't run command if bot can't send messages
 | 
					  // don't run command if bot can't send messages
 | 
				
			||||||
  if (message.channel.guild && !message.channel.permissionsOf(client.user.id).has("sendMessages")) return;
 | 
					  if (message.channel.guild && !message.channel.permissionsOf(client.user.id).has("sendMessages")) return;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  // this is here to prevent reading the database if a message is unrelated
 | 
					 | 
				
			||||||
  let valid = false;
 | 
					 | 
				
			||||||
  for (const key of commands) {
 | 
					 | 
				
			||||||
    if (message.content.toLowerCase().includes(key)) {
 | 
					 | 
				
			||||||
      valid = true;
 | 
					 | 
				
			||||||
      break;
 | 
					 | 
				
			||||||
    }
 | 
					 | 
				
			||||||
  }
 | 
					 | 
				
			||||||
  if (!valid) return;
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
  let prefixCandidate;
 | 
					  let prefixCandidate;
 | 
				
			||||||
  if (message.channel.guild) {
 | 
					  if (message.channel.guild) {
 | 
				
			||||||
    if (collections.prefixCache.has(message.channel.guild.id)) {
 | 
					    if (collections.prefixCache.has(message.channel.guild.id)) {
 | 
				
			||||||
| 
						 | 
					@ -96,8 +86,8 @@ module.exports = async (client, cluster, ipc, message) => {
 | 
				
			||||||
  try {
 | 
					  try {
 | 
				
			||||||
    await database.addCount(collections.aliases.has(command) ? collections.aliases.get(command) : command);
 | 
					    await database.addCount(collections.aliases.has(command) ? collections.aliases.get(command) : command);
 | 
				
			||||||
    const startTime = new Date();
 | 
					    const startTime = new Date();
 | 
				
			||||||
    const commandClass = new cmd(client, cluster, ipc, message, args, message.content.substring(prefix.length).trim().replace(command, "").trim());
 | 
					    const commandClass = new cmd(client, cluster, ipc, message, args, message.content.substring(prefix.length).trim().replace(command, "").trim()); // we also provide the message content as a parameter for cases where we need more accuracy
 | 
				
			||||||
    const result = await commandClass.run(); // we also provide the message content as a parameter for cases where we need more accuracy
 | 
					    const result = await commandClass.run();
 | 
				
			||||||
    const endTime = new Date();
 | 
					    const endTime = new Date();
 | 
				
			||||||
    if ((endTime - startTime) >= 180000) reference.allowedMentions.repliedUser = true;
 | 
					    if ((endTime - startTime) >= 180000) reference.allowedMentions.repliedUser = true;
 | 
				
			||||||
    if (typeof result === "string") {
 | 
					    if (typeof result === "string") {
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -139,6 +139,7 @@
 | 
				
			||||||
    "WhatsApp",
 | 
					    "WhatsApp",
 | 
				
			||||||
    "Half Life 3",
 | 
					    "Half Life 3",
 | 
				
			||||||
    "Tower Defense Simulator",
 | 
					    "Tower Defense Simulator",
 | 
				
			||||||
    "roblox on linux 2021"
 | 
					    "roblox on linux 2021",
 | 
				
			||||||
 | 
					    "trans rights"
 | 
				
			||||||
  ]
 | 
					  ]
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
							
								
								
									
										16
									
								
								package-lock.json
									
										
									
										generated
									
									
									
								
							
							
						
						
									
										16
									
								
								package-lock.json
									
										
									
										generated
									
									
									
								
							| 
						 | 
					@ -22,7 +22,8 @@
 | 
				
			||||||
        "node-emoji": "^1.10.0",
 | 
					        "node-emoji": "^1.10.0",
 | 
				
			||||||
        "node-fetch": "^2.6.1",
 | 
					        "node-fetch": "^2.6.1",
 | 
				
			||||||
        "qrcode": "^1.4.4",
 | 
					        "qrcode": "^1.4.4",
 | 
				
			||||||
        "sharp": "^0.28.2"
 | 
					        "sharp": "^0.28.2",
 | 
				
			||||||
 | 
					        "ws": "*"
 | 
				
			||||||
      },
 | 
					      },
 | 
				
			||||||
      "devDependencies": {
 | 
					      "devDependencies": {
 | 
				
			||||||
        "@babel/eslint-parser": "^7.13.8",
 | 
					        "@babel/eslint-parser": "^7.13.8",
 | 
				
			||||||
| 
						 | 
					@ -39,6 +40,7 @@
 | 
				
			||||||
        "mongoose": "^5.11.8",
 | 
					        "mongoose": "^5.11.8",
 | 
				
			||||||
        "pg": "^8.5.1",
 | 
					        "pg": "^8.5.1",
 | 
				
			||||||
        "uuid": "^8.3.1",
 | 
					        "uuid": "^8.3.1",
 | 
				
			||||||
 | 
					        "ws": "^7.5.0",
 | 
				
			||||||
        "zlib-sync": "^0.1.6"
 | 
					        "zlib-sync": "^0.1.6"
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
    },
 | 
					    },
 | 
				
			||||||
| 
						 | 
					@ -3519,9 +3521,9 @@
 | 
				
			||||||
      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
 | 
					      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
 | 
				
			||||||
    },
 | 
					    },
 | 
				
			||||||
    "node_modules/ws": {
 | 
					    "node_modules/ws": {
 | 
				
			||||||
      "version": "7.4.6",
 | 
					      "version": "7.5.0",
 | 
				
			||||||
      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz",
 | 
					      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.0.tgz",
 | 
				
			||||||
      "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==",
 | 
					      "integrity": "sha512-6ezXvzOZupqKj4jUqbQ9tXuJNo+BR2gU8fFRk3XCP3e0G6WT414u5ELe6Y0vtp7kmSJ3F7YWObSNr1ESsgi4vw==",
 | 
				
			||||||
      "engines": {
 | 
					      "engines": {
 | 
				
			||||||
        "node": ">=8.3.0"
 | 
					        "node": ">=8.3.0"
 | 
				
			||||||
      },
 | 
					      },
 | 
				
			||||||
| 
						 | 
					@ -6334,9 +6336,9 @@
 | 
				
			||||||
      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
 | 
					      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
 | 
				
			||||||
    },
 | 
					    },
 | 
				
			||||||
    "ws": {
 | 
					    "ws": {
 | 
				
			||||||
      "version": "7.4.6",
 | 
					      "version": "7.5.0",
 | 
				
			||||||
      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz",
 | 
					      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.0.tgz",
 | 
				
			||||||
      "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==",
 | 
					      "integrity": "sha512-6ezXvzOZupqKj4jUqbQ9tXuJNo+BR2gU8fFRk3XCP3e0G6WT414u5ELe6Y0vtp7kmSJ3F7YWObSNr1ESsgi4vw==",
 | 
				
			||||||
      "requires": {}
 | 
					      "requires": {}
 | 
				
			||||||
    },
 | 
					    },
 | 
				
			||||||
    "xtend": {
 | 
					    "xtend": {
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -51,6 +51,7 @@
 | 
				
			||||||
    "mongoose": "^5.11.8",
 | 
					    "mongoose": "^5.11.8",
 | 
				
			||||||
    "pg": "^8.5.1",
 | 
					    "pg": "^8.5.1",
 | 
				
			||||||
    "uuid": "^8.3.1",
 | 
					    "uuid": "^8.3.1",
 | 
				
			||||||
 | 
					    "ws": "^7.5.0",
 | 
				
			||||||
    "zlib-sync": "^0.1.6"
 | 
					    "zlib-sync": "^0.1.6"
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -2,7 +2,7 @@ const magick = require("../build/Release/image.node");
 | 
				
			||||||
const { Worker } = require("worker_threads");
 | 
					const { Worker } = require("worker_threads");
 | 
				
			||||||
const fetch = require("node-fetch");
 | 
					const fetch = require("node-fetch");
 | 
				
			||||||
const fs = require("fs");
 | 
					const fs = require("fs");
 | 
				
			||||||
const net = require("net");
 | 
					const WebSocket = require("ws");
 | 
				
			||||||
const fileType = require("file-type");
 | 
					const fileType = require("file-type");
 | 
				
			||||||
const path = require("path");
 | 
					const path = require("path");
 | 
				
			||||||
const { EventEmitter } = require("events");
 | 
					const { EventEmitter } = require("events");
 | 
				
			||||||
| 
						 | 
					@ -12,7 +12,7 @@ const formats = ["image/jpeg", "image/png", "image/webp", "image/gif", "video/mp
 | 
				
			||||||
 | 
					
 | 
				
			||||||
exports.jobs = {};
 | 
					exports.jobs = {};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
exports.connections = [];
 | 
					exports.connections = new Map();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
exports.servers = JSON.parse(fs.readFileSync("./servers.json", { encoding: "utf8" })).image;
 | 
					exports.servers = JSON.parse(fs.readFileSync("./servers.json", { encoding: "utf8" })).image;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -32,17 +32,18 @@ exports.repopulate = async () => {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
exports.getStatus = () => {
 | 
					exports.getStatus = () => {
 | 
				
			||||||
  return new Promise((resolve, reject) => {
 | 
					  return new Promise((resolve, reject) => {
 | 
				
			||||||
    let serversLeft = this.connections.length;
 | 
					    let serversLeft = this.connections.size;
 | 
				
			||||||
    const statuses = [];
 | 
					    const statuses = [];
 | 
				
			||||||
    const timeout = setTimeout(() => {
 | 
					    const timeout = setTimeout(() => {
 | 
				
			||||||
      resolve(statuses);
 | 
					      resolve(statuses);
 | 
				
			||||||
    }, 5000);
 | 
					    }, 5000);
 | 
				
			||||||
    for (const connection of this.connections) {
 | 
					    for (const address of this.connections.keys()) {
 | 
				
			||||||
      if (!connection.remoteAddress) {
 | 
					      const connection = this.connections.get(address);
 | 
				
			||||||
 | 
					      if (connection.readyState !== 1) {
 | 
				
			||||||
        serversLeft--;
 | 
					        serversLeft--;
 | 
				
			||||||
        continue;
 | 
					        continue;
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
      fetch(`http://${connection.remoteAddress}:8081/running`).then(statusRequest => statusRequest.json()).then((status) => {
 | 
					      fetch(`http://${address}:8080/running`).then(statusRequest => statusRequest.json()).then((status) => {
 | 
				
			||||||
        serversLeft--;
 | 
					        serversLeft--;
 | 
				
			||||||
        statuses.push(status);
 | 
					        statuses.push(status);
 | 
				
			||||||
        if (!serversLeft) {
 | 
					        if (!serversLeft) {
 | 
				
			||||||
| 
						 | 
					@ -67,16 +68,15 @@ exports.getStatus = () => {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
exports.connect = (server) => {
 | 
					exports.connect = (server) => {
 | 
				
			||||||
  return new Promise((resolve, reject) => {
 | 
					  return new Promise((resolve, reject) => {
 | 
				
			||||||
    const connection = net.createConnection(8080, server);
 | 
					    const connection = new WebSocket(`ws://${server}:8080/sock`);
 | 
				
			||||||
    const timeout = setTimeout(() => {
 | 
					    const timeout = setTimeout(() => {
 | 
				
			||||||
      const connectionIndex = this.connections.indexOf(connection);
 | 
					      this.connections.delete(server);
 | 
				
			||||||
      if (connectionIndex < 0) delete this.connections[connectionIndex];
 | 
					 | 
				
			||||||
      reject(`Failed to connect to ${server}`);
 | 
					      reject(`Failed to connect to ${server}`);
 | 
				
			||||||
    }, 5000);
 | 
					    }, 5000);
 | 
				
			||||||
    connection.once("connect", () => {
 | 
					    connection.once("open", () => {
 | 
				
			||||||
      clearTimeout(timeout);
 | 
					      clearTimeout(timeout);
 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
    connection.on("data", async (msg) => {
 | 
					    connection.on("message", async (msg) => {
 | 
				
			||||||
      const opcode = msg.readUint8(0);
 | 
					      const opcode = msg.readUint8(0);
 | 
				
			||||||
      const req = msg.slice(37, msg.length);
 | 
					      const req = msg.slice(37, msg.length);
 | 
				
			||||||
      const uuid = msg.slice(1, 37).toString();
 | 
					      const uuid = msg.slice(1, 37).toString();
 | 
				
			||||||
| 
						 | 
					@ -87,7 +87,7 @@ exports.connect = (server) => {
 | 
				
			||||||
      } else if (opcode === 0x01) { // Job completed successfully
 | 
					      } else if (opcode === 0x01) { // Job completed successfully
 | 
				
			||||||
        // the image API sends all job responses over the same socket; make sure this is ours
 | 
					        // the image API sends all job responses over the same socket; make sure this is ours
 | 
				
			||||||
        if (this.jobs[uuid]) {
 | 
					        if (this.jobs[uuid]) {
 | 
				
			||||||
          const imageReq = await fetch(`http://${connection.remoteAddress}:8081/image?id=${uuid}`);
 | 
					          const imageReq = await fetch(`http://${server}:8080/image?id=${uuid}`);
 | 
				
			||||||
          const image = await imageReq.buffer();
 | 
					          const image = await imageReq.buffer();
 | 
				
			||||||
          // The response data is given as the file extension/ImageMagick type of the image (e.g. "png"), followed
 | 
					          // The response data is given as the file extension/ImageMagick type of the image (e.g. "png"), followed
 | 
				
			||||||
          // by a newline, followed by the image data.
 | 
					          // by a newline, followed by the image data.
 | 
				
			||||||
| 
						 | 
					@ -103,49 +103,61 @@ exports.connect = (server) => {
 | 
				
			||||||
    connection.on("error", (e) => {
 | 
					    connection.on("error", (e) => {
 | 
				
			||||||
      logger.error(e.toString());
 | 
					      logger.error(e.toString());
 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
    connection.once("end", () => {
 | 
					    connection.once("close", () => {
 | 
				
			||||||
      for (const uuid of Object.keys(this.jobs)) {
 | 
					      for (const uuid of Object.keys(this.jobs)) {
 | 
				
			||||||
        if (this.jobs[uuid].addr === connection.remoteAddress) this.jobs[uuid].event.emit("error", "Job ended prematurely due to a closed connection; please run your image job again");
 | 
					        if (this.jobs[uuid].addr === server) this.jobs[uuid].event.emit("error", "Job ended prematurely due to a closed connection; please run your image job again");
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
 | 
					      logger.log(`Lost connection to ${server}, attempting to reconnect...`);
 | 
				
			||||||
 | 
					      this.connections.delete(server);
 | 
				
			||||||
 | 
					      //this.connect(server);
 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
    this.connections.push(connection);
 | 
					    this.connections.set(server, connection);
 | 
				
			||||||
    resolve();
 | 
					    resolve();
 | 
				
			||||||
  });
 | 
					  });
 | 
				
			||||||
};
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
exports.disconnect = async () => {
 | 
					exports.disconnect = async () => {
 | 
				
			||||||
  for (const connection of this.connections) {
 | 
					  for (const connection of this.connections.values()) {
 | 
				
			||||||
    connection.destroy();
 | 
					    connection.close();
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
  for (const uuid of Object.keys(this.jobs)) {
 | 
					  for (const uuid of Object.keys(this.jobs)) {
 | 
				
			||||||
    this.jobs[uuid].event.emit("error", "Job ended prematurely (not really an error; just run your image job again)");
 | 
					    this.jobs[uuid].event.emit("error", "Job ended prematurely (not really an error; just run your image job again)");
 | 
				
			||||||
    delete this.jobs[uuid];
 | 
					    delete this.jobs[uuid];
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
  this.connections = [];
 | 
					  this.connections.clear();
 | 
				
			||||||
  return;
 | 
					  return;
 | 
				
			||||||
};
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
const getIdeal = () => {
 | 
					const getIdeal = () => {
 | 
				
			||||||
  return new Promise((resolve, reject) => {
 | 
					  return new Promise((resolve, reject) => {
 | 
				
			||||||
    let serversLeft = this.connections.length;
 | 
					    let serversLeft = this.connections.size;
 | 
				
			||||||
 | 
					    if (serversLeft === 0) {
 | 
				
			||||||
 | 
					      for (const server of this.servers) {
 | 
				
			||||||
 | 
					        this.connect(server).catch(e => {
 | 
				
			||||||
 | 
					          logger.error(e);
 | 
				
			||||||
 | 
					        });
 | 
				
			||||||
 | 
					      }
 | 
				
			||||||
 | 
					      serversLeft = this.connections.size;
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
    const idealServers = [];
 | 
					    const idealServers = [];
 | 
				
			||||||
    const timeout = setTimeout(async () => {
 | 
					    const timeout = setTimeout(async () => {
 | 
				
			||||||
      try {
 | 
					      try {
 | 
				
			||||||
        const server = await chooseServer(idealServers);
 | 
					        const server = await chooseServer(idealServers);
 | 
				
			||||||
        resolve(this.connections.find(val => val.remoteAddress === server.addr));
 | 
					        resolve({ addr: server.addr, sock: this.connections.get(server.addr) });
 | 
				
			||||||
      } catch (e) {
 | 
					      } catch (e) {
 | 
				
			||||||
        reject(e);
 | 
					        reject(e);
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
    }, 5000);
 | 
					    }, 5000);
 | 
				
			||||||
    for (const connection of this.connections) {
 | 
					    for (const address of this.connections.keys()) {
 | 
				
			||||||
      if (!connection.remoteAddress) {
 | 
					      const connection = this.connections.get(address);
 | 
				
			||||||
 | 
					      if (connection.readyState !== 1) {
 | 
				
			||||||
        serversLeft--;
 | 
					        serversLeft--;
 | 
				
			||||||
        continue;
 | 
					        continue;
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
      fetch(`http://${connection.remoteAddress}:8081/status`).then(statusRequest => statusRequest.text()).then((status) => {
 | 
					      fetch(`http://${address}:8080/status`).then(statusRequest => statusRequest.text()).then((status) => {
 | 
				
			||||||
        serversLeft--;
 | 
					        serversLeft--;
 | 
				
			||||||
        idealServers.push({
 | 
					        idealServers.push({
 | 
				
			||||||
          addr: connection.remoteAddress,
 | 
					          addr: address,
 | 
				
			||||||
          load: parseInt(status)
 | 
					          load: parseInt(status)
 | 
				
			||||||
        });
 | 
					        });
 | 
				
			||||||
        return;
 | 
					        return;
 | 
				
			||||||
| 
						 | 
					@ -153,7 +165,7 @@ const getIdeal = () => {
 | 
				
			||||||
        if (!serversLeft) {
 | 
					        if (!serversLeft) {
 | 
				
			||||||
          clearTimeout(timeout);
 | 
					          clearTimeout(timeout);
 | 
				
			||||||
          const server = await chooseServer(idealServers);
 | 
					          const server = await chooseServer(idealServers);
 | 
				
			||||||
          resolve(this.connections.find(val => val.remoteAddress === server.addr));
 | 
					          resolve({ addr: server.addr, sock: this.connections.get(server.addr) });
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
      }).catch(e => {
 | 
					      }).catch(e => {
 | 
				
			||||||
        if (e.code === "ECONNREFUSED") {
 | 
					        if (e.code === "ECONNREFUSED") {
 | 
				
			||||||
| 
						 | 
					@ -166,7 +178,7 @@ const getIdeal = () => {
 | 
				
			||||||
    if (!serversLeft) {
 | 
					    if (!serversLeft) {
 | 
				
			||||||
      clearTimeout(timeout);
 | 
					      clearTimeout(timeout);
 | 
				
			||||||
      chooseServer(idealServers).then(server => {
 | 
					      chooseServer(idealServers).then(server => {
 | 
				
			||||||
        resolve(this.connections.find(val => val.remoteAddress === server.addr));
 | 
					        resolve({ addr: server.addr, sock: this.connections.get(server.addr) });
 | 
				
			||||||
      }).catch(e => reject(e));
 | 
					      }).catch(e => reject(e));
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
  });
 | 
					  });
 | 
				
			||||||
| 
						 | 
					@ -175,30 +187,10 @@ const getIdeal = () => {
 | 
				
			||||||
const start = (object, num) => {
 | 
					const start = (object, num) => {
 | 
				
			||||||
  return getIdeal().then(async (currentServer) => {
 | 
					  return getIdeal().then(async (currentServer) => {
 | 
				
			||||||
    const data = Buffer.concat([Buffer.from([0x01 /* queue job */]), Buffer.from(num.length.toString()), Buffer.from(num), Buffer.from(JSON.stringify(object))]);
 | 
					    const data = Buffer.concat([Buffer.from([0x01 /* queue job */]), Buffer.from(num.length.toString()), Buffer.from(num), Buffer.from(JSON.stringify(object))]);
 | 
				
			||||||
    return new Promise((resolve, reject) => {
 | 
					    return new Promise((resolve) => {
 | 
				
			||||||
      if (currentServer.destroyed) {
 | 
					      currentServer.sock.send(data, () => {
 | 
				
			||||||
        logger.log(`Lost connection to ${currentServer.remoteAddress}, attempting to reconnect...`);
 | 
					        resolve(currentServer.addr);
 | 
				
			||||||
        currentServer.connect(8080, currentServer.remoteAddress, () => {
 | 
					 | 
				
			||||||
          const res = start(object, num);
 | 
					 | 
				
			||||||
          reject(res); // this is done to differentiate the result from a step
 | 
					 | 
				
			||||||
      });
 | 
					      });
 | 
				
			||||||
      } else {
 | 
					 | 
				
			||||||
        currentServer.write(data, (err) => {
 | 
					 | 
				
			||||||
          if (err) {
 | 
					 | 
				
			||||||
            if (err.code === "EPIPE") {
 | 
					 | 
				
			||||||
              logger.log(`Lost connection to ${currentServer.remoteAddress}, attempting to reconnect...`);
 | 
					 | 
				
			||||||
              currentServer.connect(8080, currentServer.remoteAddress, () => {
 | 
					 | 
				
			||||||
                const res = start(object, num);
 | 
					 | 
				
			||||||
                reject(res); // this is done to differentiate the result from a step
 | 
					 | 
				
			||||||
              });
 | 
					 | 
				
			||||||
            } else {
 | 
					 | 
				
			||||||
              reject(err);
 | 
					 | 
				
			||||||
            }
 | 
					 | 
				
			||||||
          } else {
 | 
					 | 
				
			||||||
            resolve(currentServer.remoteAddress);
 | 
					 | 
				
			||||||
          }
 | 
					 | 
				
			||||||
        });
 | 
					 | 
				
			||||||
      }
 | 
					 | 
				
			||||||
    });
 | 
					    });
 | 
				
			||||||
  }).catch((result) => {
 | 
					  }).catch((result) => {
 | 
				
			||||||
    throw result;
 | 
					    throw result;
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue