Expose pooling configuration for YT pool

config/config.example.yml

@@ -163,6 +163,29 @@ https_only: false
 ##
 #pool_size: 100
 
+##
+## Granular pooling behavior controls for HTTP connections.
+## The following is entirely optional.
+##
+## Warning: This is NOT used to adjust the behavior of the database pool. Please look
+## Inside the DB YAML map for those controls.
+##
+
+## Initial amount of connections in the pool 
+#initial_http_pool_size: 0
+
+## Amount of connections when idle
+#max_idle_http_pool_size: 1
+
+## Amount of seconds to wait if the connection pool is full, and a connection is unavailable. 
+#http_pool_checkout_timeout: 5 
+
+## Amount of retries when a connection is either lost or can't be established.
+#http_pool_retry_attempts: 1
+
+## Amount of seconds between each retry
+#http_pool_retry_delay: 0.2
+
 ##
 ## Enable/Disable the use of QUIC (HTTP/3) when connecting
 ## to the youtube API and websites ('youtube.com', 'ytimg.com').

src/invidious.cr

@@ -67,7 +67,17 @@ SOFTWARE = {
   "branch"  => "#{CURRENT_BRANCH}",
 }
 
-YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic)
+YT_POOL = YoutubeConnectionPool.new(YT_URL,
+
+  initial_pool_size: CONFIG.initial_http_pool_size,
+  max_pool_size: CONFIG.pool_size,
+  max_idle_pool_size: CONFIG.max_idle_http_pool_size,
+  checkout_timeout: CONFIG.http_pool_checkout_timeout,
+  retry_attempts: CONFIG.http_pool_retry_attempts,
+  retry_delay: CONFIG.http_pool_retry_delay,
+
+  use_quic: CONFIG.use_quic
+)
 
 # CLI
 Kemal.config.extra_options do |parser|

src/invidious/helpers/helpers.cr

@@ -102,6 +102,13 @@ class Config
   property port : Int32 = 3000                                     # Port to listen for connections (overrided by command line argument)
   property host_binding : String = "0.0.0.0"                       # Host to bind (overrided by command line argument)
   property pool_size : Int32 = 100                                 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
+
+  property initial_http_pool_size : Int32 = 0       # Initial amount of connections in the http pool
+  property max_idle_http_pool_size : Int32 = 1      # Amount of connections when idle
+  property http_pool_checkout_timeout : Float64 = 5 # Amount of seconds to wait if the connection pool is full, and a connection is unavailable.
+  property http_pool_retry_attempts : Int32 = 1     # Amount of retries when a connection is either lost or can't be established
+  property http_pool_retry_delay : Float64 = 0.2    # Amount of seconds between each retry
+
   property use_quic : Bool = true # Use quic transport for youtube api
 
   @[YAML::Field(converter: Preferences::StringToCookies)]

src/invidious/helpers/utils.cr

@@ -1,6 +1,10 @@
 require "lsquic"
 require "db"
 
+# Alias of DB::Pool for easier debugging
+class YTPool(T) < DB::Pool(T)
+end
+
 def add_yt_headers(request)
   request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
   request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
@@ -20,9 +24,16 @@ struct YoutubeConnectionPool
   property! url : URI
   property! capacity : Int32
   property! timeout : Float64
-  property pool : DB::Pool(QUIC::Client | HTTP::Client)
+  property pool : YTPool(QUIC::Client | HTTP::Client)
 
-  def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true)
+  def initialize(url : URI,
+                 @initial_pool_size = 1,
+                 @max_pool_size = 0,
+                 @max_idle_pool_size = 1,
+                 @checkout_timeout : Float64 = 5.0,
+                 @retry_attempts = 1,
+                 @retry_delay : Float64 = 0.2,
+                 use_quic = true)
     @url = url
     @pool = build_pool(use_quic)
   end
@@ -51,12 +62,20 @@ struct YoutubeConnectionPool
   end
 
   private def build_pool(use_quic)
-    DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
+    YTPool(QUIC::Client | HTTP::Client).new(
+      initial_pool_size: @initial_pool_size,
+      max_pool_size: @max_pool_size,
+      max_idle_pool_size: @max_idle_pool_size,
+      checkout_timeout: @checkout_timeout,
+      retry_attempts: @retry_attempts,
+      retry_delay: @retry_delay
+    ) do
       if use_quic
         conn = QUIC::Client.new(url)
       else
         conn = HTTP::Client.new(url)
       end
+
       conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"