From e0d25ff8878c4e9749dc921839a2277aa29f0ebb Mon Sep 17 00:00:00 2001 From: Andrew Zhao Date: Wed, 23 Dec 2020 00:52:23 -0500 Subject: [PATCH] Close http clients after using The crystal http client maintains a keepalive connection to the other server which stays alive for some time. This should be closed if the client instance is not used again to avoid hogging resources --- src/invidious.cr | 18 ++++++++++-------- src/invidious/comments.cr | 2 ++ src/invidious/helpers/proxy.cr | 6 +++++- src/invidious/helpers/utils.cr | 11 ++++++++++- src/invidious/jobs/bypass_captcha_job.cr | 2 ++ src/invidious/users.cr | 2 +- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 91798129..9e6f4797 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2123,14 +2123,13 @@ get "/api/v1/annotations/:id" do |env| file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") - client = make_client(ARCHIVE_URL) - location = client.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}") + location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) if !location.headers["Location"]? env.response.status_code = location.status_code end - response = make_client(URI.parse(location.headers["Location"])).get(location.headers["Location"]) + response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"])) if response.body.empty? env.response.status_code = 404 @@ -3481,8 +3480,12 @@ get "/videoplayback" do |env| location = URI.parse(response.headers["Location"]) env.response.headers["Access-Control-Allow-Origin"] = "*" - host = "#{location.scheme}://#{location.host}" - client = make_client(URI.parse(host), region) + new_host = "#{location.scheme}://#{location.host}" + if new_host != host + host = new_host + client.close + client = make_client(URI.parse(new_host), region) + end url = "#{location.full_path}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" else @@ -3513,7 +3516,6 @@ get "/videoplayback" do |env| end begin - client = make_client(URI.parse(host), region) client.get(url, headers) do |response| response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -3554,8 +3556,6 @@ get "/videoplayback" do |env| chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1 end - client = make_client(URI.parse(host), region) - # TODO: Record bytes written so we can restart after a chunk fails while true if !range_end && content_length @@ -3619,6 +3619,7 @@ get "/videoplayback" do |env| if ex.message != "Error reading socket: Connection reset by peer" break else + client.close client = make_client(URI.parse(host), region) end end @@ -3628,6 +3629,7 @@ get "/videoplayback" do |env| first_chunk = false end end + client.close end get "/ggpht/*" do |env| diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 9f9edca0..8849c87f 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -269,6 +269,8 @@ def fetch_reddit_comments(id, sort_by = "confidence") raise InfoException.new("Could not fetch comments") end + client.close + comments = result[1].data.as(RedditListing).children return comments, thread end diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index 4f415ba0..7a42ef41 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr @@ -108,7 +108,9 @@ def filter_proxies(proxies) proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) client.set_proxy(proxy) - client.head("/").status_code == 200 + status_ok = client.head("/").status_code == 200 + client.close + status_ok rescue ex false end @@ -132,6 +134,7 @@ def get_nova_proxies(country_code = "US") headers["Referer"] = "https://www.proxynova.com/proxy-server-list/country-#{country_code}/" response = client.get("/proxy-server-list/country-#{country_code}/", headers) + client.close document = XML.parse_html(response.body) proxies = [] of {ip: String, port: Int32, score: Float64} @@ -177,6 +180,7 @@ def get_spys_proxies(country_code = "US") } response = client.post("/free-proxy-list/#{country_code}/", headers, form: body) + client.close 20.times do if response.status_code == 200 break diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index bb9a35ea..f068b5f2 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -101,6 +101,15 @@ def make_client(url : URI, region = nil) return client end +def make_client(url : URI, region = nil, &block) + client = make_client(url, region) + begin + yield client + ensure + client.close + end +end + def decode_length_seconds(string) length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i length_seconds = [0] * (3 - length_seconds.size) + length_seconds @@ -361,7 +370,7 @@ def subscribe_pubsub(topic, key, config) "hub.secret" => key.to_s, } - return make_client(PUBSUB_URL).post("/subscribe", form: body) + return make_client(PUBSUB_URL, &.post("/subscribe", form: body)) end def parse_range(range) diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr index daba64d5..61f8eaf3 100644 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ b/src/invidious/jobs/bypass_captcha_job.cr @@ -91,6 +91,8 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob }, }.to_json).body) + captcha_client.close + raise response["error"].as_s if response["error"]? task_id = response["taskId"].as_i diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 6a3ca5c1..6bc2acd2 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -427,7 +427,7 @@ def generate_captcha(key, db) end def generate_text_captcha(key, db) - response = make_client(TEXTCAPTCHA_URL).get("/omarroth@protonmail.com.json").body + response = make_client(TEXTCAPTCHA_URL, &.get("/omarroth@protonmail.com.json").body) response = JSON.parse(response) tokens = response["a"].as_a.map do |answer|