Re-add image job timeout, split dockerfiles, removed api from docker-compose, more fixes

This commit is contained in:
TheEssem 2021-05-10 22:59:19 -05:00
parent f0734bba00
commit 2b810c7c86
10 changed files with 90 additions and 128 deletions

View file

@ -1,59 +1,13 @@
# syntax = edrevo/dockerfile-plus
# Docker/Kubernetes file for running the image API
FROM node:alpine
RUN apk --no-cache upgrade
RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \
zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \
libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \
vips vips-dev
INCLUDE+ Dockerfile.base
# liblqr needs to be built manually for magik to work
# and because alpine doesn't have it in their repos
RUN git clone https://github.com/carlobaldassi/liblqr \
&& cd liblqr \
&& ./configure \
&& make \
&& make install
RUN apk add --no-cache redis
RUN service redis start
# install imagemagick from source rather than using the package
# since the alpine package does not include pango support.
RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \
&& cd ImageMagick \
&& git checkout 7.0.10-45 \
&& ./configure \
--prefix=/usr \
--sysconfdir=/etc \
--mandir=/usr/share/man \
--infodir=/usr/share/info \
--enable-static \
--disable-openmp \
--with-threads \
--with-png \
--with-webp \
--with-modules \
--with-pango \
--without-hdri \
--with-lqr \
&& make \
&& make install
RUN update-ms-fonts && fc-cache -f
RUN adduser esmBot -s /bin/sh -D
WORKDIR /home/esmBot/.internal
COPY ./assets/caption.otf /usr/share/fonts/caption.otf
COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf
COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf
COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf
RUN fc-cache -fv
COPY --chown=node:node ./package.json package.json
COPY --chown=node:node ./package-lock.json package-lock.json
RUN npm install
COPY . .
RUN npm run build
USER esmBot
EXPOSE 8080 8081

53
Dockerfile.base Normal file
View file

@ -0,0 +1,53 @@
FROM node:alpine
RUN apk --no-cache upgrade
RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \
zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \
libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \
vips vips-dev
# liblqr needs to be built manually for magick to work
# and because alpine doesn't have it in their repos
RUN git clone https://github.com/carlobaldassi/liblqr \
&& cd liblqr \
&& ./configure \
&& make \
&& make install
# install imagemagick from source rather than using the package
# since the alpine package does not include pango support.
RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \
&& cd ImageMagick \
&& ./configure \
--prefix=/usr \
--sysconfdir=/etc \
--mandir=/usr/share/man \
--infodir=/usr/share/info \
--enable-static \
--disable-openmp \
--with-threads \
--with-png \
--with-webp \
--with-modules \
--with-pango \
--without-hdri \
--with-lqr \
&& make \
&& make install
RUN update-ms-fonts && fc-cache -f
RUN adduser esmBot -s /bin/sh -D
WORKDIR /home/esmBot/.internal
COPY ./assets/caption.otf /usr/share/fonts/caption.otf
COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf
COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf
COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf
RUN fc-cache -fv
COPY --chown=node:node ./package.json package.json
COPY --chown=node:node ./package-lock.json package-lock.json
RUN npm install
COPY . .
RUN npm run build

View file

@ -1,59 +1,10 @@
# syntax = edrevo/dockerfile-plus
# Docker/Kubernetes file for running the bot
FROM node:alpine
RUN apk --no-cache upgrade
RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \
zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \
libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \
vips vips-dev
INCLUDE+ Dockerfile.base
# liblqr needs to be built manually for magik to work
# and because alpine doesn't have it in their repos
RUN git clone https://github.com/carlobaldassi/liblqr \
&& cd liblqr \
&& ./configure \
&& make \
&& make install
# install imagemagick from source rather than using the package
# since the alpine package does not include pango support.
RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \
&& cd ImageMagick \
&& git checkout 7.0.10-45 \
&& ./configure \
--prefix=/usr \
--sysconfdir=/etc \
--mandir=/usr/share/man \
--infodir=/usr/share/info \
--enable-static \
--disable-openmp \
--with-threads \
--with-png \
--with-webp \
--with-modules \
--with-pango \
--without-hdri \
--with-lqr \
&& make \
&& make install
RUN update-ms-fonts && fc-cache -f
RUN adduser esmBot -s /bin/sh -D
WORKDIR /home/esmBot/.internal
COPY ./assets/caption.otf /usr/share/fonts/caption.otf
COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf
COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf
COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf
RUN fc-cache -fv
COPY --chown=node:node ./package.json package.json
COPY --chown=node:node ./package-lock.json package-lock.json
RUN npm install
COPY . .
RUN npm run build
USER esmBot
ENTRYPOINT ["node", "app.js"]

View file

@ -188,8 +188,13 @@ const runJob = (job, sock) => {
const worker = new Worker(path.join(__dirname, "../utils/image-runner.js"), {
workerData: object
});
const timeout = setTimeout(() => {
worker.terminate();
reject(new Error("Job timed out"));
}, 900000);
log(`Job ${job.uuid} started`, job.num);
worker.once("message", (data) => {
clearTimeout(timeout);
log(`Sending result of job ${job.uuid} back to the bot`, job.num);
const jobObject = jobs.get(job.uuid);
jobObject.data = data.buffer;
@ -200,7 +205,10 @@ const runJob = (job, sock) => {
return resolve();
});
});
worker.once("error", reject);
worker.once("error", (e) => {
clearTimeout(timeout);
reject(e);
});
/*run(object).then((data) => {
log(`Sending result of job ${job.uuid} back to the bot`, job.num);
const jobObject = jobs.get(job.uuid);

View file

@ -19,7 +19,6 @@ services:
links:
- lavalink
depends_on:
- api
- lavalink
- postgres
@ -27,18 +26,6 @@ services:
esmbot:
ipv4_address: 172.20.0.2
api:
container_name: api
build:
context: .
dockerfile: Dockerfile.api
image: esmbot-api
restart: unless-stopped
networks:
esmbot:
ipv4_address: 172.20.0.3
lavalink:
container_name: lavalink
image: fredboat/lavalink:dev
@ -48,7 +35,7 @@ services:
- ./assets:/opt/Lavalink/assets
networks:
esmbot:
ipv4_address: 172.20.0.4
ipv4_address: 172.20.0.3
postgres:
container_name: postgres
@ -63,7 +50,7 @@ services:
POSTGRES_DB: esmbot
networks:
esmbot:
ipv4_address: 172.20.0.5
ipv4_address: 172.20.0.4
adminer:
image: adminer
@ -74,7 +61,7 @@ services:
- 8888:8080
networks:
esmbot:
ipv4_address: 172.20.0.6
ipv4_address: 172.20.0.5
volumes:
bot-help:

View file

@ -42,6 +42,7 @@ Napi::Value Speed(const Napi::CallbackInfo &info) {
break;
}
image.animationDelay(new_delay);
image.gifDisposeMethod(Magick::BackgroundDispose);
}
if (removeFrames) {

View file

@ -39,6 +39,7 @@ Napi::Value Trump(const Napi::CallbackInfo &info) {
Magick::DstOverCompositeOp);
watermark_new.magick(type);
watermark_new.animationDelay(delay == 0 ? image.animationDelay() : delay);
watermark_new.gifDisposeMethod(Magick::BackgroundDispose);
mid.push_back(watermark_new);
}

12
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "esmbot",
"version": "1.5.1",
"version": "1.5.2",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "esmbot",
"version": "1.5.1",
"version": "1.5.2",
"license": "MIT",
"dependencies": {
"@top-gg/sdk": "^3.1.1",
@ -16,7 +16,7 @@
"duckduckgo-images-api": "^1.0.5",
"emoji-regex": "^9.2.2",
"eris": "^0.15.0",
"eris-sharder": "github:discordware/eris-sharder#eris-dev",
"eris-sharder": "github:esmBot/eris-sharder#eris-dev",
"file-type": "^16.1.0",
"jsqr": "^1.3.1",
"lavacord": "^1.1.9",
@ -1163,7 +1163,7 @@
},
"node_modules/eris-sharder": {
"version": "1.10.0",
"resolved": "git+ssh://git@github.com/discordware/eris-sharder.git#94b5330234ef908fa984284223f68edd3c718de2",
"resolved": "git+ssh://git@github.com/esmBot/eris-sharder.git#2a4418b2430d2804d28d8da54754317e235bb748",
"license": "MIT",
"dependencies": {
"asciiart-logo": "^0.2.6",
@ -4819,8 +4819,8 @@
}
},
"eris-sharder": {
"version": "git+ssh://git@github.com/discordware/eris-sharder.git#94b5330234ef908fa984284223f68edd3c718de2",
"from": "eris-sharder@github:discordware/eris-sharder#eris-dev",
"version": "git+ssh://git@github.com/esmBot/eris-sharder.git#2a4418b2430d2804d28d8da54754317e235bb748",
"from": "eris-sharder@github:esmBot/eris-sharder#eris-dev",
"requires": {
"asciiart-logo": "^0.2.6",
"colors": "^1.1.2",

View file

@ -8,8 +8,10 @@
},
"scripts": {
"build": "node-gyp configure build -j max",
"docker:build-api": "docker build -t esmbot-api -f Dockerfile.api .",
"docker:build": "DOCKER_BUILDKIT=1 docker build -t esmbot -f Dockerfile.bot .",
"docker:build-api": "DOCKER_BUILDKIT=1 docker build -t esmbot-api -f Dockerfile.api .",
"docker:build-ss": "docker build -t headless-chrome-alpine --no-cache ./utils/screenshot",
"docker:run-bot": "docker run --rm --network=host esmbot",
"docker:run-api": "docker run --rm --network=host esmbot-api",
"docker:run-ss": "docker run --rm --network=host --shm-size=128m headless-chrome-alpine",
"docker:run-lava": "docker run --rm --network host -v \"$(pwd)\"/application.yml:/opt/Lavalink/application.yml -v \"$(pwd)\"/assets:/opt/Lavalink/assets fredboat/lavalink:dev",
@ -29,7 +31,7 @@
"duckduckgo-images-api": "^1.0.5",
"emoji-regex": "^9.2.2",
"eris": "^0.15.0",
"eris-sharder": "github:discordware/eris-sharder#eris-dev",
"eris-sharder": "github:esmBot/eris-sharder#eris-dev",
"file-type": "^16.1.0",
"jsqr": "^1.3.1",
"lavacord": "^1.1.9",

View file

@ -99,7 +99,7 @@ exports.connect = (server) => {
for (const uuid of Object.keys(jobs)) {
if (jobs[uuid].addr === connection.remoteAddress) jobs[uuid].event.emit("error", new Error("Job ended prematurely due to a closed connection; please run your image job again"));
}
this.connections.filter((val) => val !== connection);
this.connections = this.connections.filter((val) => val.remoteAddress !== connection.remoteAddress);
});
this.connections.push(connection);
resolve();
@ -131,7 +131,12 @@ const getIdeal = () => {
}, 5000);
for (const connection of this.connections) {
if (!connection.remoteAddress) continue;
fetch(`http://${connection.remoteAddress}:8081/status`).then(statusRequest => statusRequest.text()).then(async (status) => {
let promise = new Promise((resolveTest) => { resolveTest(); });
if (connection.destroyed) {
this.connections = this.connections.filter((val) => val.remoteAddress !== connection.remoteAddress);
promise = this.connect(connection.remoteAddress);
}
promise.then(() => fetch(`http://${connection.remoteAddress}:8081/status`)).then(statusRequest => statusRequest.text()).then(async (status) => {
serversLeft--;
idealServers.push({
addr: connection.remoteAddress,