Rework image API again, replaced many calls to replace with replaceAll
This commit is contained in:
parent
b2b8fd643a
commit
62346cbae4
15 changed files with 271 additions and 185 deletions
216
utils/image.js
216
utils/image.js
|
|
@ -3,13 +3,20 @@ const { Worker } = require("worker_threads");
|
|||
const fetch = require("node-fetch");
|
||||
const AbortController = require("abort-controller");
|
||||
const net = require("net");
|
||||
const dgram = require("dgram");
|
||||
const fileType = require("file-type");
|
||||
const servers = require("../servers.json").image;
|
||||
exports.servers = require("../servers.json").image;
|
||||
const path = require("path");
|
||||
const { EventEmitter } = require("events");
|
||||
const logger = require("./logger.js");
|
||||
|
||||
const formats = ["image/jpeg", "image/png", "image/webp", "image/gif"];
|
||||
|
||||
const jobs = {};
|
||||
|
||||
const connections = [];
|
||||
|
||||
const statuses = {};
|
||||
|
||||
const chooseServer = async (ideal) => {
|
||||
if (ideal.length === 0) throw "No available servers";
|
||||
const sorted = ideal.sort((a, b) => {
|
||||
|
|
@ -18,47 +25,113 @@ const chooseServer = async (ideal) => {
|
|||
return sorted[0];
|
||||
};
|
||||
|
||||
const getIdeal = () => {
|
||||
exports.connect = (server) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = dgram.createSocket("udp4");
|
||||
let serversLeft = servers.length;
|
||||
const idealServers = [];
|
||||
const connection = net.createConnection(8080, server);
|
||||
const timeout = setTimeout(() => {
|
||||
socket.close(async () => {
|
||||
try {
|
||||
const server = await chooseServer(idealServers);
|
||||
resolve(server);
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
const connectionIndex = connections.indexOf(connection);
|
||||
if (connectionIndex < 0) delete connections[connectionIndex];
|
||||
reject(`Failed to connect to ${server}`);
|
||||
}, 5000);
|
||||
socket.on("message", async (msg, rinfo) => {
|
||||
connection.once("connect", () => {
|
||||
clearTimeout(timeout);
|
||||
});
|
||||
connection.on("data", async (msg) => {
|
||||
const opcode = msg.readUint8(0);
|
||||
const res = parseInt(msg.slice(1, msg.length).toString());
|
||||
if (opcode === 0x3) {
|
||||
const req = msg.slice(37, msg.length);
|
||||
const uuid = msg.slice(1, 37).toString();
|
||||
if (opcode === 0x00) { // Job queued
|
||||
if (jobs[req]) {
|
||||
jobs[req].emit("uuid", uuid);
|
||||
}
|
||||
} else if (opcode === 0x01) { // Job completed successfully
|
||||
// the image API sends all job responses over the same socket; make sure this is ours
|
||||
if (jobs[uuid]) {
|
||||
const imageReq = await fetch(`http://${connection.remoteAddress}:8081/image?id=${uuid}`);
|
||||
const image = await imageReq.buffer();
|
||||
// The response data is given as the file extension/ImageMagick type of the image (e.g. "png"), followed
|
||||
// by a newline, followed by the image data.
|
||||
|
||||
jobs[uuid].emit("image", image, imageReq.headers.get("ext"));
|
||||
}
|
||||
} else if (opcode === 0x02) { // Job errored
|
||||
if (jobs[uuid]) {
|
||||
jobs[uuid].emit("error", new Error(req));
|
||||
}
|
||||
} else if (opcode === 0x03) {
|
||||
// we use the uuid part here because queue info requests don't respond with one
|
||||
statuses[`${connection.remoteAddress}:${connection.remotePort}`] = parseInt(uuid);
|
||||
}
|
||||
});
|
||||
connection.on("error", (e) => {
|
||||
console.error(e);
|
||||
});
|
||||
connections.push(connection);
|
||||
resolve();
|
||||
});
|
||||
};
|
||||
|
||||
const getIdeal = () => {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
let serversLeft = connections.length;
|
||||
const idealServers = [];
|
||||
const timeout = setTimeout(async () => {
|
||||
try {
|
||||
const server = await chooseServer(idealServers);
|
||||
resolve(connections.find(val => val.remoteAddress === server.addr));
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
}, 5000);
|
||||
for (const connection of connections) {
|
||||
if (!connection.remoteAddress) continue;
|
||||
try {
|
||||
const statusRequest = await fetch(`http://${connection.remoteAddress}:8081/status`);
|
||||
const status = await statusRequest.text();
|
||||
serversLeft--;
|
||||
idealServers.push({
|
||||
addr: rinfo.address,
|
||||
load: res
|
||||
addr: connection.remoteAddress,
|
||||
load: parseInt(status)
|
||||
});
|
||||
if (!serversLeft) {
|
||||
clearTimeout(timeout);
|
||||
socket.close(async () => {
|
||||
try {
|
||||
const server = await chooseServer(idealServers);
|
||||
resolve(server);
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
const server = await chooseServer(idealServers);
|
||||
resolve(connections.find(val => val.remoteAddress === server.addr));
|
||||
}
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
for (const server of servers) {
|
||||
socket.send(Buffer.from([0x2]), 8080, server, (err) => {
|
||||
if (err) reject(err);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const start = (object, num) => {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
const currentServer = await getIdeal();
|
||||
const data = Buffer.concat([Buffer.from([0x01 /* queue job */]), Buffer.from(num.length.toString()), Buffer.from(num), Buffer.from(JSON.stringify(object))]);
|
||||
currentServer.write(data, (err) => {
|
||||
if (err) {
|
||||
if (err.code === "EPIPE") {
|
||||
logger.log(`Lost connection to ${currentServer.remoteAddress}, attempting to reconnect...`);
|
||||
currentServer.connect(8080, currentServer.remoteAddress, async () => {
|
||||
const res = start(object, num);
|
||||
resolve(res);
|
||||
});
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
}
|
||||
});
|
||||
const event = new EventEmitter();
|
||||
event.once("uuid", (uuid) => {
|
||||
delete jobs[num];
|
||||
jobs[uuid] = event;
|
||||
resolve({ uuid, event });
|
||||
});
|
||||
jobs[num] = event;
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
|
@ -105,65 +178,30 @@ exports.getType = async (image) => {
|
|||
exports.run = object => {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
if (process.env.API === "true") {
|
||||
// Connect to best image server
|
||||
const currentServer = await getIdeal();
|
||||
const socket = dgram.createSocket("udp4");
|
||||
const data = Buffer.concat([Buffer.from([0x1 /* queue job */]), Buffer.from(JSON.stringify(object))]);
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
socket.close();
|
||||
reject("UDP timed out");
|
||||
}, 25000);
|
||||
|
||||
let jobID;
|
||||
socket.on("message", (msg) => {
|
||||
const opcode = msg.readUint8(0);
|
||||
const req = msg.slice(37, msg.length);
|
||||
const uuid = msg.slice(1, 36).toString();
|
||||
if (opcode === 0x0) { // Job queued
|
||||
clearTimeout(timeout);
|
||||
jobID = uuid;
|
||||
} else if (opcode === 0x1) { // Job completed successfully
|
||||
// the image API sends all job responses over the same socket; make sure this is ours
|
||||
if (jobID === uuid) {
|
||||
const client = net.createConnection(req.toString(), currentServer.addr);
|
||||
const array = [];
|
||||
client.on("data", (rawData) => {
|
||||
array.push(rawData);
|
||||
});
|
||||
client.once("end", () => {
|
||||
const data = Buffer.concat(array);
|
||||
// The response data is given as the file extension/ImageMagick type of the image (e.g. "png"), followed
|
||||
// by a newline, followed by the image data.
|
||||
const delimIndex = data.indexOf("\n");
|
||||
socket.close();
|
||||
if (delimIndex === -1) reject("Could not parse response");
|
||||
const payload = {
|
||||
// Take just the image data
|
||||
buffer: data.slice(delimIndex + 1),
|
||||
type: data.slice(0, delimIndex).toString()
|
||||
};
|
||||
resolve(payload);
|
||||
});
|
||||
client.on("error", (err) => {
|
||||
socket.close();
|
||||
reject(err);
|
||||
});
|
||||
}
|
||||
} else if (opcode === 0x2) { // Job errored
|
||||
if (jobID === uuid) {
|
||||
socket.close();
|
||||
reject(req);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
socket.send(data, 8080, currentServer.addr, (err) => {
|
||||
if (err) {
|
||||
socket.close();
|
||||
try {
|
||||
// Connect to best image server
|
||||
const num = Math.floor(Math.random() * 100000).toString().slice(0, 5);
|
||||
const timeout = setTimeout(() => {
|
||||
if (jobs[num]) delete jobs[num];
|
||||
reject("Request timed out");
|
||||
}, 25000);
|
||||
const { uuid, event } = await start(object, num);
|
||||
clearTimeout(timeout);
|
||||
event.once("image", (image, type) => {
|
||||
delete jobs[uuid];
|
||||
const payload = {
|
||||
// Take just the image data
|
||||
buffer: image,
|
||||
type: type
|
||||
};
|
||||
resolve(payload);
|
||||
});
|
||||
event.once("error", (err) => {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
} else {
|
||||
// Called from command (not using image API)
|
||||
const worker = new Worker(path.join(__dirname, "image-runner.js"), {
|
||||
|
|
|
|||
|
|
@ -18,18 +18,18 @@ exports.clean = async (text) => {
|
|||
text = util.inspect(text, { depth: 1 });
|
||||
|
||||
text = text
|
||||
.replace(/`/g, `\`${String.fromCharCode(8203)}`)
|
||||
.replace(/@/g, `@${String.fromCharCode(8203)}`)
|
||||
.replace(process.env.TOKEN, optionalReplace(process.env.TOKEN))
|
||||
.replace(process.env.MASHAPE, optionalReplace(process.env.MASHAPE))
|
||||
.replace(process.env.CAT, optionalReplace(process.env.CAT))
|
||||
.replace(process.env.GOOGLE, optionalReplace(process.env.GOOGLE))
|
||||
.replace(process.env.DBL, optionalReplace(process.env.DBL))
|
||||
.replace(process.env.MONGO, optionalReplace(process.env.MONGO))
|
||||
.replace(process.env.TWITTER_KEY, optionalReplace(process.env.TWITTER_KEY))
|
||||
.replace(process.env.CONSUMER_SECRET, optionalReplace(process.env.CONSUMER_SECRET))
|
||||
.replace(process.env.ACCESS_TOKEN, optionalReplace(process.env.ACCESS_TOKEN))
|
||||
.replace(process.env.ACCESS_SECRET, optionalReplace(process.env.ACCESS_SECRET));
|
||||
.replaceAll("`", `\`${String.fromCharCode(8203)}`)
|
||||
.replaceAll("@", `@${String.fromCharCode(8203)}`)
|
||||
.replaceAll(process.env.TOKEN, optionalReplace(process.env.TOKEN))
|
||||
.replaceAll(process.env.MASHAPE, optionalReplace(process.env.MASHAPE))
|
||||
.replaceAll(process.env.CAT, optionalReplace(process.env.CAT))
|
||||
.replaceAll(process.env.GOOGLE, optionalReplace(process.env.GOOGLE))
|
||||
.replaceAll(process.env.DBL, optionalReplace(process.env.DBL))
|
||||
.replaceAll(process.env.MONGO, optionalReplace(process.env.MONGO))
|
||||
.replaceAll(process.env.TWITTER_KEY, optionalReplace(process.env.TWITTER_KEY))
|
||||
.replaceAll(process.env.CONSUMER_SECRET, optionalReplace(process.env.CONSUMER_SECRET))
|
||||
.replaceAll(process.env.ACCESS_TOKEN, optionalReplace(process.env.ACCESS_TOKEN))
|
||||
.replaceAll(process.env.ACCESS_SECRET, optionalReplace(process.env.ACCESS_SECRET));
|
||||
|
||||
return text;
|
||||
};
|
||||
|
|
@ -38,19 +38,19 @@ exports.clean = async (text) => {
|
|||
exports.getTweet = async (tweets, reply = false, isDownload = false) => {
|
||||
const randomTweet = this.random(reply ? (isDownload ? tweets.download : tweets.replies) : tweets.tweets);
|
||||
if (randomTweet.match("{{message}}")) {
|
||||
return randomTweet.replace(/{{message}}/gm, await this.getRandomMessage());
|
||||
return randomTweet.replaceAll("{{message}}", await this.getRandomMessage());
|
||||
} else {
|
||||
return randomTweet
|
||||
.replace(/{{media}}/gm, () => {
|
||||
.replaceAll("{{media}}", () => {
|
||||
return this.random(tweets.media);
|
||||
})
|
||||
.replace(/{{games}}/gm, () => {
|
||||
.replaceAll("{{games}}", () => {
|
||||
return this.random(tweets.games);
|
||||
})
|
||||
.replace(/{{phrases}}/gm, () => {
|
||||
.replaceAll("{{phrases}}", () => {
|
||||
return this.random(tweets.phrases);
|
||||
})
|
||||
.replace(/{{characters}}/gm, () => {
|
||||
.replaceAll("{{characters}}", () => {
|
||||
return this.random(tweets.characters);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue