From ad043a75f1df77376852515ed1ab95cd74af85ec Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 28 Sep 2021 12:04:00 -0700 Subject: [PATCH] Expose pooling configuration for YT pool --- config/config.example.yml | 23 +++++++++++++++++++++++ src/invidious.cr | 12 +++++++++++- src/invidious/helpers/helpers.cr | 9 ++++++++- src/invidious/helpers/utils.cr | 25 ++++++++++++++++++++++--- 4 files changed, 64 insertions(+), 5 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 4595af4b..b2541fec 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -163,6 +163,29 @@ https_only: false ## #pool_size: 100 +## +## Granular pooling behavior controls for HTTP connections. +## The following is entirely optional. +## +## Warning: This is NOT used to adjust the behavior of the database pool. Please look +## Inside the DB YAML map for those controls. +## + +## Initial amount of connections in the pool +#initial_http_pool_size: 0 + +## Amount of connections when idle +#max_idle_http_pool_size: 1 + +## Amount of seconds to wait if the connection pool is full, and a connection is unavailable. +#http_pool_checkout_timeout: 5 + +## Amount of retries when a connection is either lost or can't be established. +#http_pool_retry_attempts: 1 + +## Amount of seconds between each retry +#http_pool_retry_delay: 0.2 + ## ## Enable/Disable the use of QUIC (HTTP/3) when connecting ## to the youtube API and websites ('youtube.com', 'ytimg.com'). diff --git a/src/invidious.cr b/src/invidious.cr index 9229c9d1..9064d11b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -67,7 +67,17 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic) +YT_POOL = YoutubeConnectionPool.new(YT_URL, + + initial_pool_size: CONFIG.initial_http_pool_size, + max_pool_size: CONFIG.pool_size, + max_idle_pool_size: CONFIG.max_idle_http_pool_size, + checkout_timeout: CONFIG.http_pool_checkout_timeout, + retry_attempts: CONFIG.http_pool_retry_attempts, + retry_delay: CONFIG.http_pool_retry_delay, + + use_quic: CONFIG.use_quic +) # CLI Kemal.config.extra_options do |parser| diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 589d230e..5bfa22a0 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -102,7 +102,14 @@ class Config property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - property use_quic : Bool = true # Use quic transport for youtube api + + property initial_http_pool_size : Int32 = 0 # Initial amount of connections in the http pool + property max_idle_http_pool_size : Int32 = 1 # Amount of connections when idle + property http_pool_checkout_timeout : Float64 = 5 # Amount of seconds to wait if the connection pool is full, and a connection is unavailable. + property http_pool_retry_attempts : Int32 = 1 # Amount of retries when a connection is either lost or can't be established + property http_pool_retry_delay : Float64 = 0.2 # Amount of seconds between each retry + + property use_quic : Bool = true # Use quic transport for youtube api @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 6ee07d7a..d7710c0c 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,6 +1,10 @@ require "lsquic" require "db" +# Alias of DB::Pool for easier debugging +class YTPool(T) < DB::Pool(T) +end + def add_yt_headers(request) request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" @@ -20,9 +24,16 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : DB::Pool(QUIC::Client | HTTP::Client) + property pool : YTPool(QUIC::Client | HTTP::Client) - def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) + def initialize(url : URI, + @initial_pool_size = 1, + @max_pool_size = 0, + @max_idle_pool_size = 1, + @checkout_timeout : Float64 = 5.0, + @retry_attempts = 1, + @retry_delay : Float64 = 0.2, + use_quic = true) @url = url @pool = build_pool(use_quic) end @@ -51,12 +62,20 @@ struct YoutubeConnectionPool end private def build_pool(use_quic) - DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + YTPool(QUIC::Client | HTTP::Client).new( + initial_pool_size: @initial_pool_size, + max_pool_size: @max_pool_size, + max_idle_pool_size: @max_idle_pool_size, + checkout_timeout: @checkout_timeout, + retry_attempts: @retry_attempts, + retry_delay: @retry_delay + ) do if use_quic conn = QUIC::Client.new(url) else conn = HTTP::Client.new(url) end + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"