From 6930570fa2d3c6a3ef82bb3e529b5989fc706b3c Mon Sep 17 00:00:00 2001
From: Omar Roth <omarroth@protonmail.com>
Date: Fri, 25 Oct 2019 12:58:16 -0400
Subject: [PATCH] Add HTTPClient pool

---
 shard.yml                           |  3 ++
 src/invidious.cr                    | 69 ++++++++++-------------------
 src/invidious/channels.cr           | 36 ++++++---------
 src/invidious/comments.cr           |  3 +-
 src/invidious/helpers/helpers.cr    |  1 +
 src/invidious/helpers/proxy.cr      |  4 ++
 src/invidious/helpers/signatures.cr |  5 +--
 src/invidious/helpers/utils.cr      | 47 +++++++++++++++++++-
 src/invidious/mixes.cr              |  3 +-
 src/invidious/playlists.cr          | 12 ++---
 src/invidious/search.cr             | 13 +++---
 src/invidious/trending.cr           |  9 ++--
 src/invidious/users.cr              | 11 ++---
 src/invidious/videos.cr             | 10 ++---
 14 files changed, 115 insertions(+), 111 deletions(-)

diff --git a/shard.yml b/shard.yml
index 69c1610f..04c20893 100644
--- a/shard.yml
+++ b/shard.yml
@@ -18,6 +18,9 @@ dependencies:
   kemal:
     github: kemalcr/kemal
     version: ~> 0.26.0
+  pool:
+    github: ysbaddaden/pool
+    version: ~> 0.2.3
 
 crystal: 0.31.1
 
diff --git a/src/invidious.cr b/src/invidious.cr
index 5feabe49..cd6775a7 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -46,6 +46,7 @@ PUBSUB_URL      = URI.parse("https://pubsubhubbub.appspot.com")
 REDDIT_URL      = URI.parse("https://www.reddit.com")
 TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com")
 YT_URL          = URI.parse("https://www.youtube.com")
+YT_IMG_URL      = URI.parse("https://i.ytimg.com")
 
 CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@@ -91,6 +92,9 @@ LOCALES = {
   "zh-TW" => load_locale("zh-TW"),
 }
 
+YT_POOL     = HTTPPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
+YT_IMG_POOL = HTTPPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05)
+
 config = CONFIG
 logger = Invidious::LogHandler.new
 
@@ -658,8 +662,7 @@ get "/embed/:id" do |env|
 
     next env.redirect url
   when "live_stream"
-    client = make_client(YT_URL)
-    response = client.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
+    response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
     video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
 
     env.params.query.delete_all("channel")
@@ -2250,8 +2253,7 @@ get "/modify_notifications" do |env|
     headers = HTTP::Headers.new
     headers["Cookie"] = env.request.headers["Cookie"]
 
-    client = make_client(YT_URL)
-    html = client.get("/subscription_manager?disable_polymer=1", headers)
+    html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
 
     cookies = HTTP::Cookies.from_headers(headers)
     html.cookies.each do |cookie|
@@ -2280,7 +2282,7 @@ get "/modify_notifications" do |env|
       channel_id = channel.content.lstrip("/channel/").not_nil!
       channel_req["channel_id"] = channel_id
 
-      client.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
+      YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
     end
   end
 
@@ -2558,8 +2560,7 @@ post "/data_control" do |env|
           if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
             next match["channel"]
           elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
-            client = make_client(YT_URL)
-            response = client.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
+            response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
             document = XML.parse_html(response.body)
             canonical = document.xpath_node(%q(//link[@rel="canonical"]))
 
@@ -3084,8 +3085,7 @@ get "/feed/channel/:ucid" do |env|
     next error_message
   end
 
-  client = make_client(YT_URL)
-  rss = client.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body
+  rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body
   rss = XML.parse_html(rss)
 
   videos = [] of SearchVideo
@@ -3229,8 +3229,7 @@ get "/feed/playlist/:plid" do |env|
     end
   end
 
-  client = make_client(YT_URL)
-  response = client.get("/feeds/videos.xml?playlist_id=#{plid}")
+  response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
   document = XML.parse(response.body)
 
   document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
@@ -3395,14 +3394,13 @@ end
 {"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route|
   get route do |env|
     locale = LOCALES[env.get("preferences").as(Preferences).locale]?
-    client = make_client(YT_URL)
 
     # Appears to be a bug in routing, having several routes configured
     # as `/a/:a`, `/b/:a`, `/c/:a` results in 404
     value = env.request.resource.split("/")[2]
     body = ""
     {"channel", "user", "c"}.each do |type|
-      response = client.get("/#{type}/#{value}/live?disable_polymer=1")
+      response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
       if response.status_code == 200
         body = response.body
       end
@@ -3433,10 +3431,9 @@ end
 get "/c/:user" do |env|
   locale = LOCALES[env.get("preferences").as(Preferences).locale]?
 
-  client = make_client(YT_URL)
   user = env.params.url["user"]
 
-  response = client.get("/c/#{user}")
+  response = YT_POOL.client &.get("/c/#{user}")
   document = XML.parse_html(response.body)
 
   anchor = document.xpath_node(%q(//a[contains(@class,"branded-page-header-title-link")]))
@@ -3676,7 +3673,6 @@ get "/api/v1/storyboards/:id" do |env|
   id = env.params.url["id"]
   region = env.params.query["region"]?
 
-  client = make_client(YT_URL)
   begin
     video = get_video(id, PG_DB, region: region)
   rescue ex : VideoRedirect
@@ -3764,7 +3760,6 @@ get "/api/v1/captions/:id" do |env|
   # In future this should be investigated as an alternative, since it does not require
   # getting video info.
 
-  client = make_client(YT_URL)
   begin
     video = get_video(id, PG_DB, region: region)
   rescue ex : VideoRedirect
@@ -3823,7 +3818,7 @@ get "/api/v1/captions/:id" do |env|
   # Auto-generated captions often have cues that aren't aligned properly with the video,
   # as well as some other markup that makes it cumbersome, so we try to fix that here
   if caption.name.simpleText.includes? "auto-generated"
-    caption_xml = client.get(url).body
+    caption_xml = YT_POOL.client &.get(url).body
     caption_xml = XML.parse(caption_xml)
 
     webvtt = String.build do |str|
@@ -3866,7 +3861,7 @@ get "/api/v1/captions/:id" do |env|
       end
     end
   else
-    webvtt = client.get("#{url}&format=vtt").body
+    webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
   end
 
   if title = env.params.query["title"]?
@@ -4013,9 +4008,7 @@ get "/api/v1/annotations/:id" do |env|
       cache_annotation(PG_DB, id, annotations)
     end
   when "youtube"
-    client = make_client(YT_URL)
-
-    response = client.get("/annotations_invideo?video_id=#{id}")
+    response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
 
     if response.status_code != 200
       env.response.status_code = response.status_code
@@ -5115,7 +5108,6 @@ get "/api/manifest/dash/id/:id" do |env|
   # we can opt to only add a source to a representation if it has a unique height within that representation
   unique_res = env.params.query["unique_res"]? && (env.params.query["unique_res"] == "true" || env.params.query["unique_res"] == "1")
 
-  client = make_client(YT_URL)
   begin
     video = get_video(id, PG_DB, region: region)
   rescue ex : VideoRedirect
@@ -5126,7 +5118,7 @@ get "/api/manifest/dash/id/:id" do |env|
   end
 
   if dashmpd = video.player_response["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
-    manifest = client.get(dashmpd).body
+    manifest = YT_POOL.client &.get(dashmpd).body
 
     manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
       url = baseurl.lchop("<BaseURL>")
@@ -5226,8 +5218,7 @@ get "/api/manifest/dash/id/:id" do |env|
 end
 
 get "/api/manifest/hls_variant/*" do |env|
-  client = make_client(YT_URL)
-  manifest = client.get(env.request.path)
+  manifest = YT_POOL.client &.get(env.request.path)
 
   if manifest.status_code != 200
     env.response.status_code = manifest.status_code
@@ -5252,8 +5243,7 @@ get "/api/manifest/hls_variant/*" do |env|
 end
 
 get "/api/manifest/hls_playlist/*" do |env|
-  client = make_client(YT_URL)
-  manifest = client.get(env.request.path)
+  manifest = YT_POOL.client &.get(env.request.path)
 
   if manifest.status_code != 200
     env.response.status_code = manifest.status_code
@@ -5616,10 +5606,6 @@ get "/videoplayback" do |env|
   end
 end
 
-# We need this so the below route works as expected
-get "/ggpht*" do |env|
-end
-
 get "/ggpht/*" do |env|
   host = "https://yt3.ggpht.com"
   client = make_client(URI.parse(host))
@@ -5745,12 +5731,9 @@ get "/vi/:id/:name" do |env|
   id = env.params.url["id"]
   name = env.params.url["name"]
 
-  host = "https://i.ytimg.com"
-  client = make_client(URI.parse(host))
-
   if name == "maxres.jpg"
     build_thumbnails(id, config, Kemal.config).each do |thumb|
-      if client.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200
+      if YT_IMG_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200
         name = thumb[:url] + ".jpg"
         break
       end
@@ -5766,7 +5749,7 @@ get "/vi/:id/:name" do |env|
   end
 
   begin
-    client.get(url, headers) do |response|
+    YT_IMG_POOL.client &.get(url, headers) do |response|
       env.response.status_code = response.status_code
       response.headers.each do |key, value|
         if !RESPONSE_HEADERS_BLACKLIST.includes? key
@@ -5789,9 +5772,7 @@ end
 
 # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
 get "/watch_videos" do |env|
-  client = make_client(YT_URL)
-
-  response = client.get("#{env.request.path}?#{env.request.query}")
+  response = YT_POOL.client &.get(env.request.resource)
   if url = response.headers["Location"]?
     url = URI.parse(url).full_path
     next env.redirect url
@@ -5805,11 +5786,10 @@ error 404 do |env|
     item = md["id"]
 
     # Check if item is branding URL e.g. https://youtube.com/gaming
-    client = make_client(YT_URL)
-    response = client.get("/#{item}")
+    response = YT_POOL.client &.get("/#{item}")
 
     if response.status_code == 301
-      response = client.get(response.headers["Location"])
+      response = YT_POOL.client &.get(response.headers["Location"])
     end
 
     if response.body.empty?
@@ -5837,8 +5817,7 @@ error 404 do |env|
     end
 
     # Check if item is video ID
-    client = make_client(YT_URL)
-    if item.match(/^[a-zA-Z0-9_-]{11}$/) && client.head("/watch?v=#{item}").status_code != 404
+    if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
       env.response.headers["Location"] = url
       halt env, status_code: 302
     end
diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr
index cce98310..c79be352 100644
--- a/src/invidious/channels.cr
+++ b/src/invidious/channels.cr
@@ -195,9 +195,7 @@ def get_channel(id, db, refresh = true, pull_all_videos = true)
 end
 
 def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
-  client = make_client(YT_URL)
-
-  rss = client.get("/feeds/videos.xml?channel_id=#{ucid}").body
+  rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
   rss = XML.parse_html(rss)
 
   author = rss.xpath_node(%q(//feed/title))
@@ -216,7 +214,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
   page = 1
 
   url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
-  response = client.get(url)
+  response = YT_POOL.client &.get(url)
   json = JSON.parse(response.body)
 
   if json["content_html"]? && !json["content_html"].as_s.empty?
@@ -296,7 +294,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
 
     loop do
       url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
-      response = client.get(url)
+      response = YT_POOL.client &.get(url)
       json = JSON.parse(response.body)
 
       if json["content_html"]? && !json["content_html"].as_s.empty?
@@ -375,12 +373,10 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
 end
 
 def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
-  client = make_client(YT_URL)
-
   if continuation
     url = produce_channel_playlists_url(ucid, continuation, sort_by, auto_generated)
 
-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
     json = JSON.parse(response.body)
 
     if json["load_more_widget_html"].as_s.empty?
@@ -399,7 +395,7 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
   elsif auto_generated
     url = "/channel/#{ucid}"
 
-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
     html = XML.parse_html(response.body)
 
     nodeset = html.xpath_nodes(%q(//ul[@id="browse-items-primary"]/li[contains(@class, "feed-item-container")]))
@@ -415,7 +411,7 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by)
       url += "&sort=dd"
     end
 
-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
     html = XML.parse_html(response.body)
 
     continuation = html.xpath_node(%q(//button[@data-uix-load-more-href]))
@@ -625,13 +621,12 @@ end
 
 # TODO: Add "sort_by"
 def fetch_channel_community(ucid, continuation, locale, config, kemal_config, format, thin_mode)
-  client = make_client(YT_URL)
   headers = HTTP::Headers.new
   headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
 
-  response = client.get("/channel/#{ucid}/community?gl=US&hl=en", headers)
+  response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en", headers)
   if response.status_code == 404
-    response = client.get("/user/#{ucid}/community?gl=US&hl=en", headers)
+    response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en", headers)
   end
 
   if response.status_code == 404
@@ -668,7 +663,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo
       session_token: session_token,
     }
 
-    response = client.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req)
+    response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req)
     body = JSON.parse(response.body)
 
     body = body["response"]["continuationContents"]["itemSectionContinuation"]? ||
@@ -929,11 +924,9 @@ def extract_channel_community_cursor(continuation)
 end
 
 def get_about_info(ucid, locale)
-  client = make_client(YT_URL)
-
-  about = client.get("/channel/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
+  about = YT_POOL.client &.get("/channel/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
   if about.status_code == 404
-    about = client.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
+    about = YT_POOL.client &.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en")
   end
 
   if md = about.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/)
@@ -1038,11 +1031,9 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
   count = 0
   videos = [] of SearchVideo
 
-  client = make_client(YT_URL)
-
   2.times do |i|
     url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
     json = JSON.parse(response.body)
 
     if json["content_html"]? && !json["content_html"].as_s.empty?
@@ -1067,11 +1058,10 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
 end
 
 def get_latest_videos(ucid)
-  client = make_client(YT_URL)
   videos = [] of SearchVideo
 
   url = produce_channel_videos_url(ucid, 0)
-  response = client.get(url)
+  response = YT_POOL.client &.get(url)
   json = JSON.parse(response.body)
 
   if json["content_html"]? && !json["content_html"].as_s.empty?
diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr
index 740449d3..d4fc2c6f 100644
--- a/src/invidious/comments.cr
+++ b/src/invidious/comments.cr
@@ -85,7 +85,6 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
     session_token: session_token,
   }
 
-  client = make_client(YT_URL, video.info["region"]?)
   headers = HTTP::Headers.new
 
   headers["content-type"] = "application/x-www-form-urlencoded"
@@ -98,7 +97,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
   headers["x-youtube-client-name"] = "1"
   headers["x-youtube-client-version"] = "2.20180719"
 
-  response = client.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req)
+  response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req))
   response = JSON.parse(response.body)
 
   if !response["response"]["continuationContents"]?
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index d23d5f07..c2282ec0 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -234,6 +234,7 @@ struct Config
     force_resolve:     {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
     port:              {type: Int32, default: 3000},                                                        # Port to listen for connections (overrided by command line argument)
     host_binding:      {type: String, default: "0.0.0.0"},                                                  # Host to bind (overrided by command line argument)
+    pool_size:         {type: Int32, default: 100},
   })
 end
 
diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr
index 0ba3c1fe..65880db9 100644
--- a/src/invidious/helpers/proxy.cr
+++ b/src/invidious/helpers/proxy.cr
@@ -77,6 +77,10 @@ class HTTPClient < HTTP::Client
     end
   end
 
+  def unset_proxy
+    @socket = nil
+  end
+
   def proxy_connection_options
     opts = {} of Symbol => Float64 | Nil
 
diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr
index 8b760398..1d238576 100644
--- a/src/invidious/helpers/signatures.cr
+++ b/src/invidious/helpers/signatures.cr
@@ -1,8 +1,7 @@
 def fetch_decrypt_function(id = "CvFH_6DNRCY")
-  client = make_client(YT_URL)
-  document = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
+  document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
   url = document.match(/src="(?<url>\/yts\/jsbin\/player_ias-.{9}\/en_US\/base.js)"/).not_nil!["url"]
-  player = client.get(url).body
+  player = YT_POOL.client &.get(url).body
 
   function_name = player.match(/^(?<name>[^=]+)=function\(a\){a=a\.split\(""\)/m).not_nil!["name"]
   function_body = player.match(/^#{Regex.escape(function_name)}=function\(a\){(?<body>[^}]+)}/m).not_nil!["body"]
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr
index 117a5dbe..ae12a932 100644
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@@ -1,3 +1,46 @@
+require "pool/connection"
+
+struct HTTPPool
+  property! url : URI
+  property! capacity : Int32
+  property! timeout : Float64
+  property pool : ConnectionPool(HTTPClient)
+
+  def initialize(url : URI, @capacity = 5, @timeout = 5.0)
+    @url = url
+    @pool = build_pool
+  end
+
+  def client(region = nil, &block)
+    pool.connection do |conn|
+      if region
+        PROXY_LIST[region]?.try &.sample(40).each do |proxy|
+          begin
+            proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
+            conn.set_proxy(proxy)
+            break
+          rescue ex
+          end
+        end
+      end
+
+      response = yield conn
+      conn.unset_proxy
+      response
+    end
+  end
+
+  private def build_pool
+    ConnectionPool(HTTPClient).new(capacity: capacity, timeout: timeout) do
+      client = HTTPClient.new(url)
+      client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
+      client.read_timeout = 5.seconds
+      client.connect_timeout = 5.seconds
+      client
+    end
+  end
+end
+
 # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
 def ci_lower_bound(pos, n)
   if n == 0
@@ -21,8 +64,8 @@ end
 def make_client(url : URI, region = nil)
   client = HTTPClient.new(url)
   client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
-  client.read_timeout = 15.seconds
-  client.connect_timeout = 15.seconds
+  client.read_timeout = 5.seconds
+  client.connect_timeout = 5.seconds
 
   if region
     PROXY_LIST[region]?.try &.sample(40).each do |proxy|
diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr
index a1d592f0..04a37b87 100644
--- a/src/invidious/mixes.cr
+++ b/src/invidious/mixes.cr
@@ -19,14 +19,13 @@ struct Mix
 end
 
 def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
-  client = make_client(YT_URL)
   headers = HTTP::Headers.new
   headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
 
   if cookies
     headers = cookies.add_request_headers(headers)
   end
-  response = client.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers)
+  response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers)
 
   initial_data = extract_initial_data(response.body)
 
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
index f65e434d..8d071acc 100644
--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@@ -384,13 +384,11 @@ def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
 end
 
 def fetch_playlist(plid, locale)
-  client = make_client(YT_URL)
-
   if plid.starts_with? "UC"
     plid = "UU#{plid.lchop("UC")}"
   end
 
-  response = client.get("/playlist?list=#{plid}&hl=en&disable_polymer=1")
+  response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en&disable_polymer=1")
   if response.status_code != 200
     raise translate(locale, "Not a playlist.")
   end
@@ -458,10 +456,8 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
 end
 
 def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil)
-  client = make_client(YT_URL)
-
   if continuation
-    html = client.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
+    html = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
     html = XML.parse_html(html.body)
 
     index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1
@@ -471,7 +467,7 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat
   if video_count > 100
     url = produce_playlist_url(plid, offset)
 
-    response = client.get(url)
+    response = YT_POOL.client &.get(url)
     response = JSON.parse(response.body)
     if !response["content_html"]? || response["content_html"].as_s.empty?
       raise translate(locale, "Empty playlist")
@@ -483,7 +479,7 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat
   elsif offset > 100
     return [] of PlaylistVideo
   else # Extract first page of videos
-    response = client.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1")
+    response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1")
     document = XML.parse_html(response.body)
     nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
 
diff --git a/src/invidious/search.cr b/src/invidious/search.cr
index 3a31c5e7..2acb4057 100644
--- a/src/invidious/search.cr
+++ b/src/invidious/search.cr
@@ -222,20 +222,18 @@ end
 alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist
 
 def channel_search(query, page, channel)
-  client = make_client(YT_URL)
-
-  response = client.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
+  response = YT_POOL.client &.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US")
   document = XML.parse_html(response.body)
   canonical = document.xpath_node(%q(//link[@rel="canonical"]))
 
   if !canonical
-    response = client.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US")
+    response = YT_POOL.client &.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US")
     document = XML.parse_html(response.body)
     canonical = document.xpath_node(%q(//link[@rel="canonical"]))
   end
 
   if !canonical
-    response = client.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
+    response = YT_POOL.client &.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US")
     document = XML.parse_html(response.body)
     canonical = document.xpath_node(%q(//link[@rel="canonical"]))
   end
@@ -247,7 +245,7 @@ def channel_search(query, page, channel)
   ucid = canonical["href"].split("/")[-1]
 
   url = produce_channel_search_url(ucid, query, page)
-  response = client.get(url)
+  response = YT_POOL.client &.get(url)
   json = JSON.parse(response.body)
 
   if json["content_html"]? && !json["content_html"].as_s.empty?
@@ -265,12 +263,11 @@ def channel_search(query, page, channel)
 end
 
 def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil)
-  client = make_client(YT_URL, region)
   if query.empty?
     return {0, [] of SearchItem}
   end
 
-  html = client.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body
+  html = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body)
   if html.empty?
     return {0, [] of SearchItem}
   end
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index 26db51ea..3a9c6935 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -1,5 +1,4 @@
 def fetch_trending(trending_type, region, locale)
-  client = make_client(YT_URL)
   headers = HTTP::Headers.new
   headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
 
@@ -12,7 +11,7 @@ def fetch_trending(trending_type, region, locale)
   if trending_type && trending_type != "Default"
     trending_type = trending_type.downcase.capitalize
 
-    response = client.get("/feed/trending?gl=#{region}&hl=en", headers).body
+    response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en", headers).body
 
     initial_data = extract_initial_data(response)
 
@@ -23,13 +22,13 @@ def fetch_trending(trending_type, region, locale)
       url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
       url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
       url += "&disable_polymer=1&gl=#{region}&hl=en"
-      trending = client.get(url).body
+      trending = YT_POOL.client &.get(url).body
       plid = extract_plid(url)
     else
-      trending = client.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+      trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
     end
   else
-    trending = client.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
+    trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body
   end
 
   trending = XML.parse_html(trending)
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 2d45d69d..4a3dba0a 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -143,8 +143,7 @@ def get_user(sid, headers, db, refresh = true)
 end
 
 def fetch_user(sid, headers, db)
-  client = make_client(YT_URL)
-  feed = client.get("/subscription_manager?disable_polymer=1", headers)
+  feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
   feed = XML.parse_html(feed.body)
 
   channels = [] of String
@@ -254,8 +253,7 @@ def subscribe_ajax(channel_id, action, env_headers)
   headers = HTTP::Headers.new
   headers["Cookie"] = env_headers["Cookie"]
 
-  client = make_client(YT_URL)
-  html = client.get("/subscription_manager?disable_polymer=1", headers)
+  html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
 
   cookies = HTTP::Cookies.from_headers(headers)
   html.cookies.each do |cookie|
@@ -279,7 +277,7 @@ def subscribe_ajax(channel_id, action, env_headers)
     }
     post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}"
 
-    client.post(post_url, headers, form: post_req)
+    YT_POOL.client &.post(post_url, headers, form: post_req)
   end
 end
 
@@ -288,8 +286,7 @@ end
 #   headers = HTTP::Headers.new
 #   headers["Cookie"] = env_headers["Cookie"]
 #
-#   client = make_client(YT_URL)
-#   html = client.get("/view_all_playlists?disable_polymer=1", headers)
+#   html = YT_POOL.client &.get("/view_all_playlists?disable_polymer=1", headers)
 #
 #   cookies = HTTP::Cookies.from_headers(headers)
 #   html.cookies.each do |cookie|
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 1ae31257..21233ff5 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -1146,8 +1146,7 @@ def extract_player_config(body, html)
 end
 
 def fetch_video(id, region)
-  client = make_client(YT_URL, region)
-  response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
+  response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))
 
   if md = response.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/)
     raise VideoRedirect.new(video_id: md["id"])
@@ -1167,8 +1166,7 @@ def fetch_video(id, region)
     bypass_regions = PROXY_LIST.keys & allowed_regions
     if !bypass_regions.empty?
       region = bypass_regions[rand(bypass_regions.size)]
-      client = make_client(YT_URL, region)
-      response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
+      response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999"))
 
       html = XML.parse_html(response.body)
       info = extract_player_config(response.body, html)
@@ -1180,10 +1178,10 @@ def fetch_video(id, region)
 
   # Try to pull streams from embed URL
   if info["reason"]?
-    embed_page = client.get("/embed/#{id}").body
+    embed_page = YT_POOL.client &.get("/embed/#{id}").body
     sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]?
     sts ||= ""
-    embed_info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body)
+    embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body)
 
     if !embed_info["reason"]?
       embed_info.each do |key, value|