From 2b810c7c867134b4448576c099fb3af190147132 Mon Sep 17 00:00:00 2001 From: TheEssem Date: Mon, 10 May 2021 22:59:19 -0500 Subject: [PATCH] Re-add image job timeout, split dockerfiles, removed api from docker-compose, more fixes --- Dockerfile.api | 54 ++++------------------------------------------ Dockerfile.base | 53 +++++++++++++++++++++++++++++++++++++++++++++ Dockerfile.bot | 53 ++------------------------------------------- api/index.js | 10 ++++++++- docker-compose.yml | 19 +++------------- natives/speed.cc | 1 + natives/trump.cc | 1 + package-lock.json | 12 +++++------ package.json | 6 ++++-- utils/image.js | 9 ++++++-- 10 files changed, 90 insertions(+), 128 deletions(-) create mode 100644 Dockerfile.base diff --git a/Dockerfile.api b/Dockerfile.api index 5927ef8..55dde76 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -1,59 +1,13 @@ +# syntax = edrevo/dockerfile-plus # Docker/Kubernetes file for running the image API FROM node:alpine -RUN apk --no-cache upgrade -RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \ - zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \ - libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \ - vips vips-dev +INCLUDE+ Dockerfile.base -# liblqr needs to be built manually for magik to work -# and because alpine doesn't have it in their repos -RUN git clone https://github.com/carlobaldassi/liblqr \ - && cd liblqr \ - && ./configure \ - && make \ - && make install +RUN apk add --no-cache redis +RUN service redis start -# install imagemagick from source rather than using the package -# since the alpine package does not include pango support. -RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \ - && cd ImageMagick \ - && git checkout 7.0.10-45 \ - && ./configure \ - --prefix=/usr \ - --sysconfdir=/etc \ - --mandir=/usr/share/man \ - --infodir=/usr/share/info \ - --enable-static \ - --disable-openmp \ - --with-threads \ - --with-png \ - --with-webp \ - --with-modules \ - --with-pango \ - --without-hdri \ - --with-lqr \ - && make \ - && make install - -RUN update-ms-fonts && fc-cache -f - -RUN adduser esmBot -s /bin/sh -D -WORKDIR /home/esmBot/.internal - -COPY ./assets/caption.otf /usr/share/fonts/caption.otf -COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf -COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf -COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf -RUN fc-cache -fv - -COPY --chown=node:node ./package.json package.json -COPY --chown=node:node ./package-lock.json package-lock.json -RUN npm install -COPY . . -RUN npm run build USER esmBot EXPOSE 8080 8081 diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 0000000..190a7c9 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,53 @@ +FROM node:alpine + +RUN apk --no-cache upgrade +RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \ + zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \ + libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \ + vips vips-dev + +# liblqr needs to be built manually for magick to work +# and because alpine doesn't have it in their repos +RUN git clone https://github.com/carlobaldassi/liblqr \ + && cd liblqr \ + && ./configure \ + && make \ + && make install + +# install imagemagick from source rather than using the package +# since the alpine package does not include pango support. +RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \ + && cd ImageMagick \ + && ./configure \ + --prefix=/usr \ + --sysconfdir=/etc \ + --mandir=/usr/share/man \ + --infodir=/usr/share/info \ + --enable-static \ + --disable-openmp \ + --with-threads \ + --with-png \ + --with-webp \ + --with-modules \ + --with-pango \ + --without-hdri \ + --with-lqr \ + && make \ + && make install + +RUN update-ms-fonts && fc-cache -f + +RUN adduser esmBot -s /bin/sh -D +WORKDIR /home/esmBot/.internal + +COPY ./assets/caption.otf /usr/share/fonts/caption.otf +COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf +COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf +COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf +RUN fc-cache -fv + +COPY --chown=node:node ./package.json package.json +COPY --chown=node:node ./package-lock.json package-lock.json +RUN npm install +COPY . . +RUN npm run build \ No newline at end of file diff --git a/Dockerfile.bot b/Dockerfile.bot index 3818dff..7d5f164 100644 --- a/Dockerfile.bot +++ b/Dockerfile.bot @@ -1,59 +1,10 @@ +# syntax = edrevo/dockerfile-plus # Docker/Kubernetes file for running the bot FROM node:alpine -RUN apk --no-cache upgrade -RUN apk add --no-cache git msttcorefonts-installer python3 alpine-sdk ffmpeg \ - zlib-dev libpng-dev libjpeg-turbo-dev freetype-dev fontconfig-dev \ - libtool libwebp-dev libxml2-dev pango-dev freetype fontconfig \ - vips vips-dev +INCLUDE+ Dockerfile.base -# liblqr needs to be built manually for magik to work -# and because alpine doesn't have it in their repos -RUN git clone https://github.com/carlobaldassi/liblqr \ - && cd liblqr \ - && ./configure \ - && make \ - && make install - -# install imagemagick from source rather than using the package -# since the alpine package does not include pango support. -RUN git clone https://github.com/ImageMagick/ImageMagick.git ImageMagick \ - && cd ImageMagick \ - && git checkout 7.0.10-45 \ - && ./configure \ - --prefix=/usr \ - --sysconfdir=/etc \ - --mandir=/usr/share/man \ - --infodir=/usr/share/info \ - --enable-static \ - --disable-openmp \ - --with-threads \ - --with-png \ - --with-webp \ - --with-modules \ - --with-pango \ - --without-hdri \ - --with-lqr \ - && make \ - && make install - -RUN update-ms-fonts && fc-cache -f - -RUN adduser esmBot -s /bin/sh -D -WORKDIR /home/esmBot/.internal - -COPY ./assets/caption.otf /usr/share/fonts/caption.otf -COPY ./assets/caption2.ttf /usr/share/fonts/caption2.ttf -COPY ./assets/hbc.ttf /usr/share/fonts/hbc.ttf -COPY ./assets/reddit.ttf /usr/share/fonts/reddit.ttf -RUN fc-cache -fv - -COPY --chown=node:node ./package.json package.json -COPY --chown=node:node ./package-lock.json package-lock.json -RUN npm install -COPY . . -RUN npm run build USER esmBot ENTRYPOINT ["node", "app.js"] diff --git a/api/index.js b/api/index.js index 23c210a..a6edc3b 100644 --- a/api/index.js +++ b/api/index.js @@ -188,8 +188,13 @@ const runJob = (job, sock) => { const worker = new Worker(path.join(__dirname, "../utils/image-runner.js"), { workerData: object }); + const timeout = setTimeout(() => { + worker.terminate(); + reject(new Error("Job timed out")); + }, 900000); log(`Job ${job.uuid} started`, job.num); worker.once("message", (data) => { + clearTimeout(timeout); log(`Sending result of job ${job.uuid} back to the bot`, job.num); const jobObject = jobs.get(job.uuid); jobObject.data = data.buffer; @@ -200,7 +205,10 @@ const runJob = (job, sock) => { return resolve(); }); }); - worker.once("error", reject); + worker.once("error", (e) => { + clearTimeout(timeout); + reject(e); + }); /*run(object).then((data) => { log(`Sending result of job ${job.uuid} back to the bot`, job.num); const jobObject = jobs.get(job.uuid); diff --git a/docker-compose.yml b/docker-compose.yml index 5cd4329..fd4d029 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,6 @@ services: links: - lavalink depends_on: - - api - lavalink - postgres @@ -27,18 +26,6 @@ services: esmbot: ipv4_address: 172.20.0.2 - - api: - container_name: api - build: - context: . - dockerfile: Dockerfile.api - image: esmbot-api - restart: unless-stopped - networks: - esmbot: - ipv4_address: 172.20.0.3 - lavalink: container_name: lavalink image: fredboat/lavalink:dev @@ -48,7 +35,7 @@ services: - ./assets:/opt/Lavalink/assets networks: esmbot: - ipv4_address: 172.20.0.4 + ipv4_address: 172.20.0.3 postgres: container_name: postgres @@ -63,7 +50,7 @@ services: POSTGRES_DB: esmbot networks: esmbot: - ipv4_address: 172.20.0.5 + ipv4_address: 172.20.0.4 adminer: image: adminer @@ -74,7 +61,7 @@ services: - 8888:8080 networks: esmbot: - ipv4_address: 172.20.0.6 + ipv4_address: 172.20.0.5 volumes: bot-help: diff --git a/natives/speed.cc b/natives/speed.cc index 437fd62..35c7c98 100644 --- a/natives/speed.cc +++ b/natives/speed.cc @@ -42,6 +42,7 @@ Napi::Value Speed(const Napi::CallbackInfo &info) { break; } image.animationDelay(new_delay); + image.gifDisposeMethod(Magick::BackgroundDispose); } if (removeFrames) { diff --git a/natives/trump.cc b/natives/trump.cc index 9101e52..0c82c9b 100644 --- a/natives/trump.cc +++ b/natives/trump.cc @@ -39,6 +39,7 @@ Napi::Value Trump(const Napi::CallbackInfo &info) { Magick::DstOverCompositeOp); watermark_new.magick(type); watermark_new.animationDelay(delay == 0 ? image.animationDelay() : delay); + watermark_new.gifDisposeMethod(Magick::BackgroundDispose); mid.push_back(watermark_new); } diff --git a/package-lock.json b/package-lock.json index b7b8b4a..f0ee080 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "esmbot", - "version": "1.5.1", + "version": "1.5.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "esmbot", - "version": "1.5.1", + "version": "1.5.2", "license": "MIT", "dependencies": { "@top-gg/sdk": "^3.1.1", @@ -16,7 +16,7 @@ "duckduckgo-images-api": "^1.0.5", "emoji-regex": "^9.2.2", "eris": "^0.15.0", - "eris-sharder": "github:discordware/eris-sharder#eris-dev", + "eris-sharder": "github:esmBot/eris-sharder#eris-dev", "file-type": "^16.1.0", "jsqr": "^1.3.1", "lavacord": "^1.1.9", @@ -1163,7 +1163,7 @@ }, "node_modules/eris-sharder": { "version": "1.10.0", - "resolved": "git+ssh://git@github.com/discordware/eris-sharder.git#94b5330234ef908fa984284223f68edd3c718de2", + "resolved": "git+ssh://git@github.com/esmBot/eris-sharder.git#2a4418b2430d2804d28d8da54754317e235bb748", "license": "MIT", "dependencies": { "asciiart-logo": "^0.2.6", @@ -4819,8 +4819,8 @@ } }, "eris-sharder": { - "version": "git+ssh://git@github.com/discordware/eris-sharder.git#94b5330234ef908fa984284223f68edd3c718de2", - "from": "eris-sharder@github:discordware/eris-sharder#eris-dev", + "version": "git+ssh://git@github.com/esmBot/eris-sharder.git#2a4418b2430d2804d28d8da54754317e235bb748", + "from": "eris-sharder@github:esmBot/eris-sharder#eris-dev", "requires": { "asciiart-logo": "^0.2.6", "colors": "^1.1.2", diff --git a/package.json b/package.json index acec6a3..fc96d1b 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,10 @@ }, "scripts": { "build": "node-gyp configure build -j max", - "docker:build-api": "docker build -t esmbot-api -f Dockerfile.api .", + "docker:build": "DOCKER_BUILDKIT=1 docker build -t esmbot -f Dockerfile.bot .", + "docker:build-api": "DOCKER_BUILDKIT=1 docker build -t esmbot-api -f Dockerfile.api .", "docker:build-ss": "docker build -t headless-chrome-alpine --no-cache ./utils/screenshot", + "docker:run-bot": "docker run --rm --network=host esmbot", "docker:run-api": "docker run --rm --network=host esmbot-api", "docker:run-ss": "docker run --rm --network=host --shm-size=128m headless-chrome-alpine", "docker:run-lava": "docker run --rm --network host -v \"$(pwd)\"/application.yml:/opt/Lavalink/application.yml -v \"$(pwd)\"/assets:/opt/Lavalink/assets fredboat/lavalink:dev", @@ -29,7 +31,7 @@ "duckduckgo-images-api": "^1.0.5", "emoji-regex": "^9.2.2", "eris": "^0.15.0", - "eris-sharder": "github:discordware/eris-sharder#eris-dev", + "eris-sharder": "github:esmBot/eris-sharder#eris-dev", "file-type": "^16.1.0", "jsqr": "^1.3.1", "lavacord": "^1.1.9", diff --git a/utils/image.js b/utils/image.js index 431fdd2..fdfe4cc 100644 --- a/utils/image.js +++ b/utils/image.js @@ -99,7 +99,7 @@ exports.connect = (server) => { for (const uuid of Object.keys(jobs)) { if (jobs[uuid].addr === connection.remoteAddress) jobs[uuid].event.emit("error", new Error("Job ended prematurely due to a closed connection; please run your image job again")); } - this.connections.filter((val) => val !== connection); + this.connections = this.connections.filter((val) => val.remoteAddress !== connection.remoteAddress); }); this.connections.push(connection); resolve(); @@ -131,7 +131,12 @@ const getIdeal = () => { }, 5000); for (const connection of this.connections) { if (!connection.remoteAddress) continue; - fetch(`http://${connection.remoteAddress}:8081/status`).then(statusRequest => statusRequest.text()).then(async (status) => { + let promise = new Promise((resolveTest) => { resolveTest(); }); + if (connection.destroyed) { + this.connections = this.connections.filter((val) => val.remoteAddress !== connection.remoteAddress); + promise = this.connect(connection.remoteAddress); + } + promise.then(() => fetch(`http://${connection.remoteAddress}:8081/status`)).then(statusRequest => statusRequest.text()).then(async (status) => { serversLeft--; idealServers.push({ addr: connection.remoteAddress,