murm/mrmBot-Matrix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Re-add image job timeout, split dockerfiles, removed api from docker-compose, more fixes

Browse source
This commit is contained in:
TheEssem 2021-05-10 22:59:19 -05:00
parent f0734bba00
commit 2b810c7c86
10 changed files with 90 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
Dockerfile.api
View file

@ -1,59 +1,13 @@
# syntax = edrevo/dockerfile-plus
# Docker/Kubernetes file for running the image API
FROM node:alpine
RUN apk --no-cache upgrade
RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \
zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \
libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \
vips vips-dev
INCLUDE+ Dockerfile.base
# liblqr needs to be built manually for magik to work
# and because alpine doesn't have it in their repos
RUN git clone https://github.com/carlobaldassi/liblqr \
&& cd liblqr \
&& ./configure \
&& make \
&& make install
RUN apk add --no-cache redis
RUN service redis start
# install imagemagick from source rather than using the package
# since the alpine package does not include pango support.
RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \
&& cd ImageMagick \
&& git checkout 7.0.10-45 \
&& ./configure \
--prefix=/usr \
--sysconfdir=/etc \
--mandir=/usr/share/man \
--infodir=/usr/share/info \
--enable-static \
--disable-openmp \
--with-threads \
--with-png \
--with-webp \
--with-modules \
--with-pango \
--without-hdri \
--with-lqr \
&& make \
&& make install
RUN update-ms-fonts && fc-cache -f
RUN adduser esmBot -s /bin/sh -D
WORKDIR /home/esmBot/.internal
COPY ./assets/caption.otf /usr/share/fonts/caption.otf
COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf
COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf
COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf
RUN fc-cache -fv
COPY --chown=node:node ./package.json package.json
COPY --chown=node:node ./package-lock.json package-lock.json
RUN npm install
COPY . .
RUN npm run build
USER esmBot
EXPOSE 8080 8081

53
Dockerfile.base Normal file
View file

@ -0,0 +1,53 @@
FROM node:alpine
RUN apk --no-cache upgrade
RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \
zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \
libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \
vips vips-dev
# liblqr needs to be built manually for magick to work
# and because alpine doesn't have it in their repos
RUN git clone https://github.com/carlobaldassi/liblqr \
&& cd liblqr \
&& ./configure \
&& make \
&& make install
# install imagemagick from source rather than using the package
# since the alpine package does not include pango support.
RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \
&& cd ImageMagick \
&& ./configure \
--prefix=/usr \
--sysconfdir=/etc \
--mandir=/usr/share/man \
--infodir=/usr/share/info \
--enable-static \
--disable-openmp \
--with-threads \
--with-png \
--with-webp \
--with-modules \
--with-pango \
--without-hdri \
--with-lqr \
&& make \
&& make install
RUN update-ms-fonts && fc-cache -f
RUN adduser esmBot -s /bin/sh -D
WORKDIR /home/esmBot/.internal
COPY ./assets/caption.otf /usr/share/fonts/caption.otf
COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf
COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf
COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf
RUN fc-cache -fv
COPY --chown=node:node ./package.json package.json
COPY --chown=node:node ./package-lock.json package-lock.json
RUN npm install
COPY . .
RUN npm run build

53
Dockerfile.bot
View file

@ -1,59 +1,10 @@
# syntax = edrevo/dockerfile-plus
# Docker/Kubernetes file for running the bot
FROM node:alpine
RUN apk --no-cache upgrade
RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \
zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \
libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \
vips vips-dev
INCLUDE+ Dockerfile.base
# liblqr needs to be built manually for magik to work
# and because alpine doesn't have it in their repos
RUN git clone https://github.com/carlobaldassi/liblqr \
&& cd liblqr \
&& ./configure \
&& make \
&& make install
# install imagemagick from source rather than using the package
# since the alpine package does not include pango support.
RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \
&& cd ImageMagick \
&& git checkout 7.0.10-45 \
&& ./configure \
--prefix=/usr \
--sysconfdir=/etc \
--mandir=/usr/share/man \
--infodir=/usr/share/info \
--enable-static \
--disable-openmp \
--with-threads \
--with-png \
--with-webp \
--with-modules \
--with-pango \
--without-hdri \
--with-lqr \
&& make \
&& make install
RUN update-ms-fonts && fc-cache -f
RUN adduser esmBot -s /bin/sh -D
WORKDIR /home/esmBot/.internal
COPY ./assets/caption.otf /usr/share/fonts/caption.otf
COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf
COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf
COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf
RUN fc-cache -fv
COPY --chown=node:node ./package.json package.json
COPY --chown=node:node ./package-lock.json package-lock.json
RUN npm install
COPY . .
RUN npm run build
USER esmBot
ENTRYPOINT ["node", "app.js"]

10
api/index.js
View file

@ -188,8 +188,13 @@ const runJob = (job, sock) => {
const worker = new Worker(path.join(__dirname, "../utils/image-runner.js"), {
workerData: object
});
const timeout = setTimeout(() => {
worker.terminate();
reject(new Error("Job timed out"));
}, 900000);
log(`Job ${job.uuid} started`, job.num);
worker.once("message", (data) => {
clearTimeout(timeout);
log(`Sending result of job ${job.uuid} back to the bot`, job.num);
const jobObject = jobs.get(job.uuid);
jobObject.data = data.buffer;
@ -200,7 +205,10 @@ const runJob = (job, sock) => {
return resolve();
});
});
worker.once("error", reject);
worker.once("error", (e) => {
clearTimeout(timeout);
reject(e);
});
/*run(object).then((data) => {
log(`Sending result of job ${job.uuid} back to the bot`, job.num);
const jobObject = jobs.get(job.uuid);

19
docker-compose.yml
View file

@ -19,7 +19,6 @@ services:
links:
- lavalink
depends_on:
- api
- lavalink
- postgres
@ -27,18 +26,6 @@ services:
esmbot:
ipv4_address: 172.20.0.2
api:
container_name: api
build:
context: .
dockerfile: Dockerfile.api
image: esmbot-api
restart: unless-stopped
networks:
esmbot:
ipv4_address: 172.20.0.3
lavalink:
container_name: lavalink
image: fredboat/lavalink:dev
@ -48,7 +35,7 @@ services:
- ./assets:/opt/Lavalink/assets
networks:
esmbot:
ipv4_address: 172.20.0.4
ipv4_address: 172.20.0.3
postgres:
container_name: postgres
@ -63,7 +50,7 @@ services:
POSTGRES_DB: esmbot
networks:
esmbot:
ipv4_address: 172.20.0.5
ipv4_address: 172.20.0.4
adminer:
image: adminer
@ -74,7 +61,7 @@ services:
- 8888:8080
networks:
esmbot:
ipv4_address: 172.20.0.6
ipv4_address: 172.20.0.5
volumes:
bot-help:

1
natives/speed.cc
View file

@ -42,6 +42,7 @@ Napi::Value Speed(const Napi::CallbackInfo &info) {
break;
}
image.animationDelay(new_delay);
image.gifDisposeMethod(Magick::BackgroundDispose);
}
if (removeFrames) {

1
natives/trump.cc
View file

@ -39,6 +39,7 @@ Napi::Value Trump(const Napi::CallbackInfo &info) {
Magick::DstOverCompositeOp);
watermark_new.magick(type);
watermark_new.animationDelay(delay == 0 ? image.animationDelay() : delay);
watermark_new.gifDisposeMethod(Magick::BackgroundDispose);
mid.push_back(watermark_new);
}

12
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "esmbot",
"version": "1.5.1",
"version": "1.5.2",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "esmbot",
"version": "1.5.1",
"version": "1.5.2",
"license": "MIT",
"dependencies": {
"@top-gg/sdk": "^3.1.1",
@ -16,7 +16,7 @@
"duckduckgo-images-api": "^1.0.5",
"emoji-regex": "^9.2.2",
"eris": "^0.15.0",
"eris-sharder": "github:discordware/eris-sharder#eris-dev",
"eris-sharder": "github:esmBot/eris-sharder#eris-dev",
"file-type": "^16.1.0",
"jsqr": "^1.3.1",
"lavacord": "^1.1.9",
@ -1163,7 +1163,7 @@
},
"node_modules/eris-sharder": {
"version": "1.10.0",
"resolved": "git+ssh://git@github.com/discordware/eris-sharder.git#94b5330234ef908fa984284223f68edd3c718de2",
"resolved": "git+ssh://git@github.com/esmBot/eris-sharder.git#2a4418b2430d2804d28d8da54754317e235bb748",
"license": "MIT",
"dependencies": {
"asciiart-logo": "^0.2.6",
@ -4819,8 +4819,8 @@
}
},
"eris-sharder": {
"version": "git+ssh://git@github.com/discordware/eris-sharder.git#94b5330234ef908fa984284223f68edd3c718de2",
"from": "eris-sharder@github:discordware/eris-sharder#eris-dev",
"version": "git+ssh://git@github.com/esmBot/eris-sharder.git#2a4418b2430d2804d28d8da54754317e235bb748",
"from": "eris-sharder@github:esmBot/eris-sharder#eris-dev",
"requires": {
"asciiart-logo": "^0.2.6",
"colors": "^1.1.2",

6
package.json
View file

@ -8,8 +8,10 @@
},
"scripts": {
"build": "node-gyp configure build -j max",
"docker:build-api": "docker build -t esmbot-api -f Dockerfile.api .",
"docker:build": "DOCKER_BUILDKIT=1 docker build -t esmbot -f Dockerfile.bot .",
"docker:build-api": "DOCKER_BUILDKIT=1 docker build -t esmbot-api -f Dockerfile.api .",
"docker:build-ss": "docker build -t headless-chrome-alpine --no-cache ./utils/screenshot",
"docker:run-bot": "docker run --rm --network=host esmbot",
"docker:run-api": "docker run --rm --network=host esmbot-api",
"docker:run-ss": "docker run --rm --network=host --shm-size=128m headless-chrome-alpine",
"docker:run-lava": "docker run --rm --network host -v \"$(pwd)\"/application.yml:/opt/Lavalink/application.yml -v \"$(pwd)\"/assets:/opt/Lavalink/assets fredboat/lavalink:dev",
@ -29,7 +31,7 @@
"duckduckgo-images-api": "^1.0.5",
"emoji-regex": "^9.2.2",
"eris": "^0.15.0",
"eris-sharder": "github:discordware/eris-sharder#eris-dev",
"eris-sharder": "github:esmBot/eris-sharder#eris-dev",
"file-type": "^16.1.0",
"jsqr": "^1.3.1",
"lavacord": "^1.1.9",

9
utils/image.js
View file

@ -99,7 +99,7 @@ exports.connect = (server) => {
for (const uuid of Object.keys(jobs)) {
if (jobs[uuid].addr === connection.remoteAddress) jobs[uuid].event.emit("error", new Error("Job ended prematurely due to a closed connection; please run your image job again"));
}
this.connections.filter((val) => val !== connection);
this.connections = this.connections.filter((val) => val.remoteAddress !== connection.remoteAddress);
});
this.connections.push(connection);
resolve();
@ -131,7 +131,12 @@ const getIdeal = () => {
}, 5000);
for (const connection of this.connections) {
if (!connection.remoteAddress) continue;
fetch(`http://${connection.remoteAddress}:8081/status`).then(statusRequest => statusRequest.text()).then(async (status) => {
let promise = new Promise((resolveTest) => { resolveTest(); });
if (connection.destroyed) {
this.connections = this.connections.filter((val) => val.remoteAddress !== connection.remoteAddress);
promise = this.connect(connection.remoteAddress);
}
promise.then(() => fetch(`http://${connection.remoteAddress}:8081/status`)).then(statusRequest => statusRequest.text()).then(async (status) => {
serversLeft--;
idealServers.push({
addr: connection.remoteAddress,