murm/mrmBot-Matrix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Rework image API again, replaced many calls to replace with replaceAll

Browse source
This commit is contained in:
TheEssem 2021-01-18 14:11:28 -06:00
parent b2b8fd643a
commit 62346cbae4
15 changed files with 271 additions and 185 deletions
Download patch file Download diff file Expand all files Collapse all files

140
api/index.js
View file

@ -4,8 +4,7 @@ require("dotenv").config();
const os = require("os"); const os = require("os");
const { run } = require("../utils/image-runner.js"); const { run } = require("../utils/image-runner.js");
const net = require("net"); const net = require("net");
const dgram = require("dgram"); // for UDP servers const http = require("http");
const socket = dgram.createSocket("udp4"); // Our universal UDP socket, this might cause issues and we may have to use a seperate socket for each connection
const start = process.hrtime(); const start = process.hrtime();
const log = (msg, jobNum) => { const log = (msg, jobNum) => {
@ -22,74 +21,123 @@ const { v4: uuidv4 } = require("uuid");
const MAX_JOBS = process.env.JOBS !== "" && process.env.JOBS !== undefined ? parseInt(process.env.JOBS) : os.cpus().length * 4; // Completely arbitrary, should usually be some multiple of your amount of cores const MAX_JOBS = process.env.JOBS !== "" && process.env.JOBS !== undefined ? parseInt(process.env.JOBS) : os.cpus().length * 4; // Completely arbitrary, should usually be some multiple of your amount of cores
let jobAmount = 0; let jobAmount = 0;
const acceptJob = async (uuid) => { const acceptJob = async (uuid, sock) => {
jobAmount++; jobAmount++;
queue.shift(); queue.shift();
try { try {
await runJob({ await runJob({
uuid: uuid, uuid: uuid,
msg: jobs[uuid].msg, msg: jobs[uuid].msg,
addr: jobs[uuid].addr,
port: jobs[uuid].port,
num: jobs[uuid].num num: jobs[uuid].num
}); }, sock);
jobAmount--; jobAmount--;
if (queue.length > 0) { if (queue.length > 0) {
acceptJob(queue[0]); acceptJob(queue[0]);
} }
delete jobs[uuid];
log(`Job ${uuid} has finished`); log(`Job ${uuid} has finished`);
} catch (err) { } catch (err) {
console.error(`Error on job ${uuid}:`, err); console.error(`Error on job ${uuid}:`, err);
socket.send(Buffer.concat([Buffer.from([0x2]), Buffer.from(uuid), Buffer.from(err.toString())]), jobs[uuid].port, jobs[uuid].addr);
jobAmount--; jobAmount--;
if (queue.length > 0) { if (queue.length > 0) {
acceptJob(queue[0]); acceptJob(queue[0]);
} }
delete jobs[uuid]; delete jobs[uuid];
sock.write(Buffer.concat([Buffer.from([0x2]), Buffer.from(uuid), Buffer.from(err.toString())]));
} }
}; };
const server = dgram.createSocket("udp4"); //Create a UDP server for listening to requests, we dont need tcp const httpServer = http.createServer((req, res) => {
server.on("message", (msg, rinfo) => { if (req.method !== "GET") {
res.statusCode = 405;
return res.end("405 Method Not Allowed");
}
const reqUrl = new URL(req.url, `http://${req.headers.host}`);
if (reqUrl.pathname === "/status") {
log(`Sending server status to ${req.socket.remoteAddress}:${req.socket.remotePort} via HTTP`);
return res.end(Buffer.from((MAX_JOBS - jobAmount).toString()));
} else if (reqUrl.pathname === "/image") {
if (!reqUrl.searchParams.has("id")) {
res.statusCode = 400;
return res.end("400 Bad Request");
}
const id = reqUrl.searchParams.get("id");
if (!jobs[id]) {
res.statusCode = 410;
return res.end("410 Gone");
}
log(`Sending image data for job ${id} to ${req.socket.remoteAddress}:${req.socket.remotePort} via HTTP`);
res.setHeader("ext", jobs[id].ext);
return res.end(jobs[id].data, (err) => {
if (err) console.error(err);
delete jobs[id];
});
} else {
res.statusCode = 404;
return res.end("404 Not Found");
}
});
httpServer.on("error", (e) => {
console.error("An HTTP error occurred: ", e);
});
httpServer.listen(8081, () => {
log("HTTP listening on port 8081");
});
const server = net.createServer((sock) => { // Create a TCP socket/server to listen to requests
log(`TCP client ${sock.remoteAddress}:${sock.remotePort} has connected`);
sock.on("error", (e) => {
console.error(e);
});
sock.on("data", (msg) => {
const opcode = msg.readUint8(0); const opcode = msg.readUint8(0);
const req = msg.toString().slice(1,msg.length); const req = msg.toString().slice(1,msg.length);
// 0x0 == Cancel job console.log(req);
// 0x1 == Queue job // 0x00 == Cancel job
// 0x2 == Get CPU usage // 0x01 == Queue job
if (opcode == 0x0) { if (opcode == 0x00) {
delete queue[queue.indexOf(req) - 1]; delete queue[queue.indexOf(req) - 1];
delete jobs[req]; delete jobs[req];
} else if (opcode == 0x1) { } else if (opcode == 0x01) {
const job = { addr: rinfo.address, port: rinfo.port, msg: req, num: jobAmount }; const length = parseInt(req.slice(0, 1));
const num = req.slice(1, length + 1);
const obj = req.slice(length + 1);
const job = { addr: sock.remoteAddress, port: sock.remotePort, msg: obj, num: jobAmount };
const uuid = uuidv4(); const uuid = uuidv4();
jobs[uuid] = job; jobs[uuid] = job;
queue.push(uuid); queue.push(uuid);
const newBuffer = Buffer.concat([Buffer.from([0x00]), Buffer.from(uuid), Buffer.from(num)]);
sock.write(newBuffer);
if (jobAmount < MAX_JOBS) { if (jobAmount < MAX_JOBS) {
log(`Got request for job ${job.msg} with id ${uuid}`, job.num); log(`Got TCP request for job ${job.msg} with id ${uuid}`, job.num);
acceptJob(uuid); acceptJob(uuid, sock);
} else { } else {
log(`Got request for job ${job.msg} with id ${uuid}, queued in position ${queue.indexOf(uuid)}`, job.num); log(`Got TCP request for job ${job.msg} with id ${uuid}, queued in position ${queue.indexOf(uuid)}`, job.num);
} }
} else {
log("Could not parse TCP message");
}
});
const newBuffer = Buffer.concat([Buffer.from([0x0]), Buffer.from(uuid)]); sock.on("end", () => {
socket.send(newBuffer, rinfo.port, rinfo.address); log(`TCP client ${sock.remoteAddress}:${sock.remotePort} has disconnected`);
} else if (opcode == 0x2) { });
socket.send(Buffer.concat([Buffer.from([0x3]), Buffer.from((MAX_JOBS - jobAmount).toString())]), rinfo.port, rinfo.address);
} else {
log("Could not parse message");
}
}); });
server.on("listening", () => { server.on("error", (e) => {
const address = server.address(); console.error("A TCP error occurred: ", e);
log(`server listening ${address.address}:${address.port}`);
}); });
server.bind(8080); // ATTENTION: Always going to be bound to 0.0.0.0 !!! server.listen(8080, () => {
log("TCP listening on port 8080");
});
const runJob = (job) => { const runJob = (job, sock) => {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
log(`Job ${job.uuid} starting...`, job.num); log(`Job ${job.uuid} starting...`, job.num);
@ -100,29 +148,17 @@ const runJob = (job) => {
} }
log(`Job ${job.uuid} started`, job.num); log(`Job ${job.uuid} started`, job.num);
const {buffer, fileExtension} = await run(object); try {
const { buffer, fileExtension } = await run(object);
log(`Sending result of job ${job.uuid} back to the bot`, job.num); log(`Sending result of job ${job.uuid} back to the bot`, job.num);
const server = net.createServer(function(tcpSocket) { jobs[job.uuid].data = buffer;
tcpSocket.write(Buffer.concat([Buffer.from(fileExtension), Buffer.from("\n"), buffer]), (err) => { jobs[job.uuid].ext = fileExtension;
if (err) console.error(err); sock.write(Buffer.concat([Buffer.from([0x1]), Buffer.from(job.uuid)]), (e) => {
tcpSocket.end(() => { if (e) return reject(e);
server.close(); return resolve();
resolve(null);
}); });
}); } catch (e) {
}); reject(e);
server.listen(job.port, job.addr);
// handle address in use errors
server.on("error", (e) => {
if (e.code === "EADDRINUSE") {
log("Address in use, retrying...", job.num);
setTimeout(() => {
server.close();
server.listen(job.port, job.addr);
}, 500);
} }
}); });
socket.send(Buffer.concat([Buffer.from([0x1]), Buffer.from(job.uuid), Buffer.from(job.port.toString())]), job.port, job.addr);
});
}; };

12
app.js
View file

@ -14,6 +14,7 @@ const client = require("./utils/client.js");
// initialize command loader // initialize command loader
const handler = require("./utils/handler.js"); const handler = require("./utils/handler.js");
const sound = require("./utils/soundplayer.js"); const sound = require("./utils/soundplayer.js");
const image = require("./utils/image.js");
// registers stuff and connects the bot // registers stuff and connects the bot
async function init() { async function init() {
@ -41,6 +42,17 @@ async function init() {
client.on(eventName, event); client.on(eventName, event);
} }
// connect to image api if enabled
if (process.env.API === "true") {
for (const server of image.servers) {
try {
await image.connect(server);
} catch (e) {
logger.error(e);
}
}
}
// login // login
client.connect(); client.connect();

2
commands/caption.js
View file

@ -9,7 +9,7 @@ exports.run = async (message, args) => {
const { buffer, type } = await magick.run({ const { buffer, type } = await magick.run({
cmd: "caption", cmd: "caption",
path: image.path, path: image.path,
caption: newArgs.join(" ").replace(/&/g, "\\&amp;").replace(/>/g, "\\&gt;").replace(/</g, "\\&lt;").replace(/"/g, "\\&quot;").replace(/'/g, "\\&apos;").replace(/%/g, "\\%"), caption: newArgs.join(" ").replaceAll("&", "\\&amp;").replaceAll(">", "\\&gt;").replaceAll("<", "\\&lt;").replaceAll("\"", "\\&quot;").replaceAll("'", "\\&apos;").replaceAll("%", "\\%"),
type: image.type type: image.type
}); });
if (processMessage.channel.messages.get(processMessage.id)) await processMessage.delete(); if (processMessage.channel.messages.get(processMessage.id)) await processMessage.delete();

2
commands/caption2.js
View file

@ -9,7 +9,7 @@ exports.run = async (message, args) => {
const { buffer, type } = await magick.run({ const { buffer, type } = await magick.run({
cmd: "captionTwo", cmd: "captionTwo",
path: image.path, path: image.path,
caption: newArgs.length !== 0 ? newArgs.join(" ").replace(/&/g, "\\&amp;").replace(/>/g, "\\&gt;").replace(/</g, "\\&lt;").replace(/"/g, "\\&quot;").replace(/'/g, "\\&apos;").replace(/%/g, "\\%") : words.sort(() => 0.5 - Math.random()).slice(0, Math.floor(Math.random() * words.length + 1)).join(" "), caption: newArgs.length !== 0 ? newArgs.join(" ").replaceAll("&", "\\&amp;").replaceAll(">", "\\&gt;").replaceAll("<", "\\&lt;").replaceAll("\"", "\\&quot;").replaceAll("'", "\\&apos;").replaceAll("%", "\\%") : words.sort(() => 0.5 - Math.random()).slice(0, Math.floor(Math.random() * words.length + 1)).join(" "),
type: image.type type: image.type
}); });
if (processMessage.channel.messages.get(processMessage.id)) await processMessage.delete(); if (processMessage.channel.messages.get(processMessage.id)) await processMessage.delete();

4
commands/channel.js
View file

@ -11,7 +11,7 @@ exports.run = async (message, args) => {
if (args[0].toLowerCase() === "disable") { if (args[0].toLowerCase() === "disable") {
let channel; let channel;
if (args[1] && args[1].match(/^<?[@#]?[&!]?\d+>?$/) && args[1] >= 21154535154122752) { if (args[1] && args[1].match(/^<?[@#]?[&!]?\d+>?$/) && args[1] >= 21154535154122752) {
const id = args[1].replace(/@/g, "").replace(/#/g, "").replace(/!/g, "").replace(/&/g, "").replace(/</g, "").replace(/>/g, ""); const id = args[1].replaceAll("@", "").replaceAll("#", "").replaceAll("!", "").replaceAll("&", "").replaceAll("<", "").replaceAll(">", "");
if (guildDB.disabled.includes(id)) return `${message.author.mention}, I'm already disabled in this channel!`; if (guildDB.disabled.includes(id)) return `${message.author.mention}, I'm already disabled in this channel!`;
channel = message.channel.guild.channels.get(id); channel = message.channel.guild.channels.get(id);
} else { } else {
@ -24,7 +24,7 @@ exports.run = async (message, args) => {
} else if (args[0].toLowerCase() === "enable") { } else if (args[0].toLowerCase() === "enable") {
let channel; let channel;
if (args[1] && args[1].match(/^<?[@#]?[&!]?\d+>?$/) && args[1] >= 21154535154122752) { if (args[1] && args[1].match(/^<?[@#]?[&!]?\d+>?$/) && args[1] >= 21154535154122752) {
const id = args[1].replace(/@/g, "").replace(/#/g, "").replace(/!/g, "").replace(/&/g, "").replace(/</g, "").replace(/>/g, ""); const id = args[1].replaceAll("@", "").replaceAll("#", "").replaceAll("!", "").replaceAll("&", "").replaceAll("<", "").replaceAll(">", "");
if (!guildDB.disabled.includes(id)) return `${message.author.mention}, I'm not disabled in that channel!`; if (!guildDB.disabled.includes(id)) return `${message.author.mention}, I'm not disabled in that channel!`;
channel = message.channel.guild.channels.get(id); channel = message.channel.guild.channels.get(id);
} else { } else {

2
commands/flag.js
View file

@ -8,7 +8,7 @@ exports.run = async (message, args) => {
const image = await require("../utils/imagedetect.js")(message); const image = await require("../utils/imagedetect.js")(message);
if (image === undefined) return `${message.author.mention}, you need to provide an image to overlay a flag onto!`; if (image === undefined) return `${message.author.mention}, you need to provide an image to overlay a flag onto!`;
if (!args[0].match(emojiRegex)) return `${message.author.mention}, you need to provide an emoji of a flag to overlay!`; if (!args[0].match(emojiRegex)) return `${message.author.mention}, you need to provide an emoji of a flag to overlay!`;
const flag = emoji.unemojify(args[0]).replace(/:/g, "").replace("flag-", ""); const flag = emoji.unemojify(args[0]).replaceAll(":", "").replace("flag-", "");
let path = `./assets/images/region-flags/png/${flag.toUpperCase()}.png`; let path = `./assets/images/region-flags/png/${flag.toUpperCase()}.png`;
if (flag === "🏴‍☠️") path = "./assets/images/pirateflag.png"; if (flag === "🏴‍☠️") path = "./assets/images/pirateflag.png";
if (flag === "rainbow-flag") path = "./assets/images/rainbowflag.png"; if (flag === "rainbow-flag") path = "./assets/images/rainbowflag.png";

2
commands/fullwidth.js
View file

@ -1,6 +1,6 @@
exports.run = async (message, args) => { exports.run = async (message, args) => {
if (args.length === 0) return `${message.author.mention}, you need to provide some text to convert to fullwidth!`; if (args.length === 0) return `${message.author.mention}, you need to provide some text to convert to fullwidth!`;
return args.join("").replace(/[A-Za-z0-9]/g, (s) => { return String.fromCharCode(s.charCodeAt(0) + 0xFEE0); }); return args.join("").replaceAll(/[A-Za-z0-9]/g, (s) => { return String.fromCharCode(s.charCodeAt(0) + 0xFEE0); });
}; };
exports.aliases = ["aesthetic", "aesthetics", "aes"]; exports.aliases = ["aesthetic", "aesthetics", "aes"];

2
commands/homebrew.js
View file

@ -5,7 +5,7 @@ exports.run = async (message, args) => {
message.channel.sendTyping(); message.channel.sendTyping();
const { buffer } = await magick.run({ const { buffer } = await magick.run({
cmd: "homebrew", cmd: "homebrew",
caption: args.join(" ").toLowerCase().replace(/\n/g, " ") caption: args.join(" ").toLowerCase().replaceAll("\n", " ")
}); });
return { return {
file: buffer, file: buffer,

4
commands/meme.js
View file

@ -10,8 +10,8 @@ exports.run = async (message, args) => {
const { buffer, type } = await magick.run({ const { buffer, type } = await magick.run({
cmd: "meme", cmd: "meme",
path: image.path, path: image.path,
top: topText.toUpperCase().replace(/&/g, "\\&amp;").replace(/>/g, "\\&gt;").replace(/</g, "\\&lt;").replace(/"/g, "\\&quot;").replace(/'/g, "\\&apos;").replace(/%/g, "\\%"), top: topText.toUpperCase().replaceAll("&", "\\&amp;").replaceAll(">", "\\&gt;").replaceAll("<", "\\&lt;").replaceAll("\"", "\\&quot;").replaceAll("'", "\\&apos;").replaceAll("%", "\\%"),
bottom: bottomText ? bottomText.toUpperCase().replace(/&/g, "\\&amp;").replace(/>/g, "\\&gt;").replace(/</g, "\\&lt;").replace(/"/g, "\\&quot;").replace(/'/g, "\\&apos;").replace(/%/g, "\\%") : "", bottom: bottomText ? bottomText.toUpperCase().replaceAll("&", "\\&amp;").replaceAll(">", "\\&gt;").replaceAll("<", "\\&lt;").replaceAll("\"", "\\&quot;").replaceAll("'", "\\&apos;").replaceAll("%", "\\%") : "",
type: image.type type: image.type
}); });
return { return {

4
commands/motivate.js
View file

@ -10,8 +10,8 @@ exports.run = async (message, args) => {
const { buffer, type } = await magick.run({ const { buffer, type } = await magick.run({
cmd: "motivate", cmd: "motivate",
path: image.path, path: image.path,
top: topText.replace(/&/g, "\\&amp;").replace(/>/g, "\\&gt;").replace(/</g, "\\&lt;").replace(/"/g, "\\&quot;").replace(/'/g, "\\&apos;").replace(/%/g, "\\%"), top: topText.replaceAll("&", "\\&amp;").replaceAll(">", "\\&gt;").replaceAll("<", "\\&lt;").replaceAll("\"", "\\&quot;").replaceAll("'", "\\&apos;").replaceAll("%", "\\%"),
bottom: bottomText ? bottomText.replace(/&/g, "\\&amp;").replace(/>/g, "\\&gt;").replace(/</g, "\\&lt;").replace(/"/g, "\\&quot;").replace(/'/g, "\\&apos;").replace(/%/g, "\\%") : "", bottom: bottomText ? bottomText.replaceAll("&", "\\&amp;").replaceAll(">", "\\&gt;").replaceAll("<", "\\&lt;").replaceAll("\"", "\\&quot;").replaceAll("'", "\\&apos;").replaceAll("%", "\\%") : "",
type: image.type type: image.type
}); });
if (processMessage.channel.messages.get(processMessage.id)) await processMessage.delete(); if (processMessage.channel.messages.get(processMessage.id)) await processMessage.delete();

2
commands/snowflake.js
View file

@ -1,7 +1,7 @@
exports.run = async (message, args) => { exports.run = async (message, args) => {
if (!args[0]) return `${message.author.mention}, you need to provide a snowflake ID!`; if (!args[0]) return `${message.author.mention}, you need to provide a snowflake ID!`;
if (!args[0].match(/^<?[@#]?[&!]?\d+>?$/) && args[0] < 21154535154122752) return `${message.author.mention}, that's not a valid snowflake!`; if (!args[0].match(/^<?[@#]?[&!]?\d+>?$/) && args[0] < 21154535154122752) return `${message.author.mention}, that's not a valid snowflake!`;
return new Date((args[0].replace(/@/g, "").replace(/#/g, "").replace(/!/g, "").replace(/&/g, "").replace(/</g, "").replace(/>/g, "") / 4194304) + 1420070400000).toUTCString(); return new Date((args[0].replaceAll("@", "").replaceAll("#", "").replaceAll("!", "").replaceAll("&", "").replaceAll("<", "").replaceAll(">", "") / 4194304) + 1420070400000).toUTCString();
}; };
exports.aliases = ["timestamp", "snowstamp", "snow"]; exports.aliases = ["timestamp", "snowstamp", "snow"];

2
commands/sonic.js
View file

@ -4,7 +4,7 @@ const wrap = require("../utils/wrap.js");
exports.run = async (message, args) => { exports.run = async (message, args) => {
if (args.length === 0) return `${message.author.mention}, you need to provide some text to make a Sonic meme!`; if (args.length === 0) return `${message.author.mention}, you need to provide some text to make a Sonic meme!`;
message.channel.sendTyping(); message.channel.sendTyping();
const cleanedMessage = args.join(" ").replace(/&/g, "\\&amp;").replace(/>/g, "\\&gt;").replace(/</g, "\\&lt;").replace(/"/g, "\\&quot;").replace(/'/g, "\\&apos;").replace(/%/g, "\\%"); const cleanedMessage = args.join(" ").replaceAll("&", "\\&amp;").replaceAll(">", "\\&gt;").replaceAll("<", "\\&lt;").replaceAll("\"", "\\&quot;").replaceAll("'", "\\&apos;").replaceAll("%", "\\%");
const { buffer } = await magick.run({ const { buffer } = await magick.run({
cmd: "sonic", cmd: "sonic",
text: wrap(cleanedMessage, {width: 15, indent: ""}) text: wrap(cleanedMessage, {width: 15, indent: ""})

6
commands/youtube.js
View file

@ -10,11 +10,11 @@ exports.run = async (message, args) => {
const result = await request.json(); const result = await request.json();
for (const [i, value] of result.items.entries()) { for (const [i, value] of result.items.entries()) {
if (value.id.kind === "youtube#channel") { if (value.id.kind === "youtube#channel") {
messages.push(`Page ${i + 1} of ${result.items.length}\n<:youtube:637020823005167626> **${decodeEntities(value.snippet.title).replace(/\*/g, "\\*")}**\nhttps://youtube.com/channel/${value.id.channelId}`); messages.push(`Page ${i + 1} of ${result.items.length}\n<:youtube:637020823005167626> **${decodeEntities(value.snippet.title).replaceAll("*", "\\*")}**\nhttps://youtube.com/channel/${value.id.channelId}`);
} else if (value.id.kind === "youtube#playlist") { } else if (value.id.kind === "youtube#playlist") {
messages.push(`Page ${i + 1} of ${result.items.length}\n<:youtube:637020823005167626> **${decodeEntities(value.snippet.title).replace(/\*/g, "\\*")}**\nCreated by **${decodeEntities(value.snippet.channelTitle).replace(/\*/g, "\\*")}**\nhttps://youtube.com/playlist?list=${value.id.playlistId}`); messages.push(`Page ${i + 1} of ${result.items.length}\n<:youtube:637020823005167626> **${decodeEntities(value.snippet.title).replaceAll("*", "\\*")}**\nCreated by **${decodeEntities(value.snippet.channelTitle).replaceAll("*", "\\*")}**\nhttps://youtube.com/playlist?list=${value.id.playlistId}`);
} else { } else {
messages.push(`Page ${i + 1} of ${result.items.length}\n<:youtube:637020823005167626> **${decodeEntities(value.snippet.title).replace(/\*/g, "\\*")}**\nUploaded by **${decodeEntities(value.snippet.channelTitle).replace(/\*/g, "\\*")}** on **${value.snippet.publishedAt.split("T")[0]}**\nhttps://youtube.com/watch?v=${value.id.videoId}`); messages.push(`Page ${i + 1} of ${result.items.length}\n<:youtube:637020823005167626> **${decodeEntities(value.snippet.title).replaceAll("*", "\\*")}**\nUploaded by **${decodeEntities(value.snippet.channelTitle).replaceAll("*", "\\*")}** on **${value.snippet.publishedAt.split("T")[0]}**\nhttps://youtube.com/watch?v=${value.id.videoId}`);
} }
} }
return paginator(message, messages); return paginator(message, messages);

174
utils/image.js
View file

@ -3,13 +3,20 @@ const { Worker } = require("worker_threads");
const fetch = require("node-fetch"); const fetch = require("node-fetch");
const AbortController = require("abort-controller"); const AbortController = require("abort-controller");
const net = require("net"); const net = require("net");
const dgram = require("dgram");
const fileType = require("file-type"); const fileType = require("file-type");
const servers = require("../servers.json").image; exports.servers = require("../servers.json").image;
const path = require("path"); const path = require("path");
const { EventEmitter } = require("events");
const logger = require("./logger.js");
const formats = ["image/jpeg", "image/png", "image/webp", "image/gif"]; const formats = ["image/jpeg", "image/png", "image/webp", "image/gif"];
const jobs = {};
const connections = [];
const statuses = {};
const chooseServer = async (ideal) => { const chooseServer = async (ideal) => {
if (ideal.length === 0) throw "No available servers"; if (ideal.length === 0) throw "No available servers";
const sorted = ideal.sort((a, b) => { const sorted = ideal.sort((a, b) => {
@ -18,47 +25,113 @@ const chooseServer = async (ideal) => {
return sorted[0]; return sorted[0];
}; };
const getIdeal = () => { exports.connect = (server) => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const socket = dgram.createSocket("udp4"); const connection = net.createConnection(8080, server);
let serversLeft = servers.length;
const idealServers = [];
const timeout = setTimeout(() => { const timeout = setTimeout(() => {
socket.close(async () => { const connectionIndex = connections.indexOf(connection);
if (connectionIndex < 0) delete connections[connectionIndex];
reject(`Failed to connect to ${server}`);
}, 5000);
connection.once("connect", () => {
clearTimeout(timeout);
});
connection.on("data", async (msg) => {
const opcode = msg.readUint8(0);
const req = msg.slice(37, msg.length);
const uuid = msg.slice(1, 37).toString();
if (opcode === 0x00) { // Job queued
if (jobs[req]) {
jobs[req].emit("uuid", uuid);
}
} else if (opcode === 0x01) { // Job completed successfully
// the image API sends all job responses over the same socket; make sure this is ours
if (jobs[uuid]) {
const imageReq = await fetch(`http://${connection.remoteAddress}:8081/image?id=${uuid}`);
const image = await imageReq.buffer();
// The response data is given as the file extension/ImageMagick type of the image (e.g. "png"), followed
// by a newline, followed by the image data.
jobs[uuid].emit("image", image, imageReq.headers.get("ext"));
}
} else if (opcode === 0x02) { // Job errored
if (jobs[uuid]) {
jobs[uuid].emit("error", new Error(req));
}
} else if (opcode === 0x03) {
// we use the uuid part here because queue info requests don't respond with one
statuses[`${connection.remoteAddress}:${connection.remotePort}`] = parseInt(uuid);
}
});
connection.on("error", (e) => {
console.error(e);
});
connections.push(connection);
resolve();
});
};
const getIdeal = () => {
return new Promise(async (resolve, reject) => {
let serversLeft = connections.length;
const idealServers = [];
const timeout = setTimeout(async () => {
try { try {
const server = await chooseServer(idealServers); const server = await chooseServer(idealServers);
resolve(server); resolve(connections.find(val => val.remoteAddress === server.addr));
} catch (e) { } catch (e) {
reject(e); reject(e);
} }
});
}, 5000); }, 5000);
socket.on("message", async (msg, rinfo) => { for (const connection of connections) {
const opcode = msg.readUint8(0); if (!connection.remoteAddress) continue;
const res = parseInt(msg.slice(1, msg.length).toString()); try {
if (opcode === 0x3) { const statusRequest = await fetch(`http://${connection.remoteAddress}:8081/status`);
const status = await statusRequest.text();
serversLeft--; serversLeft--;
idealServers.push({ idealServers.push({
addr: rinfo.address, addr: connection.remoteAddress,
load: res load: parseInt(status)
}); });
if (!serversLeft) { if (!serversLeft) {
clearTimeout(timeout); clearTimeout(timeout);
socket.close(async () => {
try {
const server = await chooseServer(idealServers); const server = await chooseServer(idealServers);
resolve(server); resolve(connections.find(val => val.remoteAddress === server.addr));
}
} catch (e) { } catch (e) {
reject(e); reject(e);
} }
}
}); });
};
const start = (object, num) => {
return new Promise(async (resolve, reject) => {
try {
const currentServer = await getIdeal();
const data = Buffer.concat([Buffer.from([0x01 /* queue job */]), Buffer.from(num.length.toString()), Buffer.from(num), Buffer.from(JSON.stringify(object))]);
currentServer.write(data, (err) => {
if (err) {
if (err.code === "EPIPE") {
logger.log(`Lost connection to ${currentServer.remoteAddress}, attempting to reconnect...`);
currentServer.connect(8080, currentServer.remoteAddress, async () => {
const res = start(object, num);
resolve(res);
});
} else {
reject(err);
} }
} }
}); });
for (const server of servers) { const event = new EventEmitter();
socket.send(Buffer.from([0x2]), 8080, server, (err) => { event.once("uuid", (uuid) => {
if (err) reject(err); delete jobs[num];
jobs[uuid] = event;
resolve({ uuid, event });
}); });
jobs[num] = event;
} catch (e) {
reject(e);
} }
}); });
}; };
@ -105,65 +178,30 @@ exports.getType = async (image) => {
exports.run = object => { exports.run = object => {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
if (process.env.API === "true") { if (process.env.API === "true") {
try {
// Connect to best image server // Connect to best image server
const currentServer = await getIdeal(); const num = Math.floor(Math.random() * 100000).toString().slice(0, 5);
const socket = dgram.createSocket("udp4");
const data = Buffer.concat([Buffer.from([0x1 /* queue job */]), Buffer.from(JSON.stringify(object))]);
const timeout = setTimeout(() => { const timeout = setTimeout(() => {
socket.close(); if (jobs[num]) delete jobs[num];
reject("UDP timed out"); reject("Request timed out");
}, 25000); }, 25000);
const { uuid, event } = await start(object, num);
let jobID;
socket.on("message", (msg) => {
const opcode = msg.readUint8(0);
const req = msg.slice(37, msg.length);
const uuid = msg.slice(1, 36).toString();
if (opcode === 0x0) { // Job queued
clearTimeout(timeout); clearTimeout(timeout);
jobID = uuid; event.once("image", (image, type) => {
} else if (opcode === 0x1) { // Job completed successfully delete jobs[uuid];
// the image API sends all job responses over the same socket; make sure this is ours
if (jobID === uuid) {
const client = net.createConnection(req.toString(), currentServer.addr);
const array = [];
client.on("data", (rawData) => {
array.push(rawData);
});
client.once("end", () => {
const data = Buffer.concat(array);
// The response data is given as the file extension/ImageMagick type of the image (e.g. "png"), followed
// by a newline, followed by the image data.
const delimIndex = data.indexOf("\n");
socket.close();
if (delimIndex === -1) reject("Could not parse response");
const payload = { const payload = {
// Take just the image data // Take just the image data
buffer: data.slice(delimIndex + 1), buffer: image,
type: data.slice(0, delimIndex).toString() type: type
}; };
resolve(payload); resolve(payload);
}); });
client.on("error", (err) => { event.once("error", (err) => {
socket.close();
reject(err); reject(err);
}); });
} catch (e) {
reject(e);
} }
} else if (opcode === 0x2) { // Job errored
if (jobID === uuid) {
socket.close();
reject(req);
}
}
});
socket.send(data, 8080, currentServer.addr, (err) => {
if (err) {
socket.close();
reject(err);
}
});
} else { } else {
// Called from command (not using image API) // Called from command (not using image API)
const worker = new Worker(path.join(__dirname, "image-runner.js"), { const worker = new Worker(path.join(__dirname, "image-runner.js"), {

34
utils/misc.js
View file

@ -18,18 +18,18 @@ exports.clean = async (text) => {
text = util.inspect(text, { depth: 1 }); text = util.inspect(text, { depth: 1 });
text = text text = text
.replace(/`/g, `\`${String.fromCharCode(8203)}`) .replaceAll("`", `\`${String.fromCharCode(8203)}`)
.replace(/@/g, `@${String.fromCharCode(8203)}`) .replaceAll("@", `@${String.fromCharCode(8203)}`)
.replace(process.env.TOKEN, optionalReplace(process.env.TOKEN)) .replaceAll(process.env.TOKEN, optionalReplace(process.env.TOKEN))
.replace(process.env.MASHAPE, optionalReplace(process.env.MASHAPE)) .replaceAll(process.env.MASHAPE, optionalReplace(process.env.MASHAPE))
.replace(process.env.CAT, optionalReplace(process.env.CAT)) .replaceAll(process.env.CAT, optionalReplace(process.env.CAT))
.replace(process.env.GOOGLE, optionalReplace(process.env.GOOGLE)) .replaceAll(process.env.GOOGLE, optionalReplace(process.env.GOOGLE))
.replace(process.env.DBL, optionalReplace(process.env.DBL)) .replaceAll(process.env.DBL, optionalReplace(process.env.DBL))
.replace(process.env.MONGO, optionalReplace(process.env.MONGO)) .replaceAll(process.env.MONGO, optionalReplace(process.env.MONGO))
.replace(process.env.TWITTER_KEY, optionalReplace(process.env.TWITTER_KEY)) .replaceAll(process.env.TWITTER_KEY, optionalReplace(process.env.TWITTER_KEY))
.replace(process.env.CONSUMER_SECRET, optionalReplace(process.env.CONSUMER_SECRET)) .replaceAll(process.env.CONSUMER_SECRET, optionalReplace(process.env.CONSUMER_SECRET))
.replace(process.env.ACCESS_TOKEN, optionalReplace(process.env.ACCESS_TOKEN)) .replaceAll(process.env.ACCESS_TOKEN, optionalReplace(process.env.ACCESS_TOKEN))
.replace(process.env.ACCESS_SECRET, optionalReplace(process.env.ACCESS_SECRET)); .replaceAll(process.env.ACCESS_SECRET, optionalReplace(process.env.ACCESS_SECRET));
return text; return text;
}; };
@ -38,19 +38,19 @@ exports.clean = async (text) => {
exports.getTweet = async (tweets, reply = false, isDownload = false) => { exports.getTweet = async (tweets, reply = false, isDownload = false) => {
const randomTweet = this.random(reply ? (isDownload ? tweets.download : tweets.replies) : tweets.tweets); const randomTweet = this.random(reply ? (isDownload ? tweets.download : tweets.replies) : tweets.tweets);
if (randomTweet.match("{{message}}")) { if (randomTweet.match("{{message}}")) {
return randomTweet.replace(/{{message}}/gm, await this.getRandomMessage()); return randomTweet.replaceAll("{{message}}", await this.getRandomMessage());
} else { } else {
return randomTweet return randomTweet
.replace(/{{media}}/gm, () => { .replaceAll("{{media}}", () => {
return this.random(tweets.media); return this.random(tweets.media);
}) })
.replace(/{{games}}/gm, () => { .replaceAll("{{games}}", () => {
return this.random(tweets.games); return this.random(tweets.games);
}) })
.replace(/{{phrases}}/gm, () => { .replaceAll("{{phrases}}", () => {
return this.random(tweets.phrases); return this.random(tweets.phrases);
}) })
.replace(/{{characters}}/gm, () => { .replaceAll("{{characters}}", () => {
return this.random(tweets.characters); return this.random(tweets.characters);
}); });
} }