Use QUIC for connections to YouTube

docker/Dockerfile

@@ -1,15 +1,24 @@
 FROM alpine:edge
 RUN apk add --no-cache crystal shards libc-dev \
-    yaml-dev libxml2-dev sqlite-dev zlib-dev openssl-dev
+    yaml-dev libxml2-dev sqlite-dev zlib-dev curl && \
+    curl -Lo /etc/apk/keys/omarroth.rsa.pub https://github.com/omarroth/boringssl-alpine/releases/download/1.1.0-r0/omarroth.rsa.pub && \
+    curl -Lo boringssl-dev.apk https://github.com/omarroth/boringssl-alpine/releases/download/1.1.0-r0/boringssl-dev-1.1.0-r0.apk && \
+    curl -Lo lsquic.apk https://github.com/omarroth/lsquic-alpine/releases/download/2.6.3-r0/lsquic-2.6.3-r0.apk && \
+    apk update && \
+    apk add boringssl-dev.apk lsquic.apk && \
+    rm -rf /var/cache/apk/* boringssl-dev.apk lsquic.apk
 WORKDIR /invidious
 COPY ./shard.yml ./shard.yml
 RUN shards update && shards install
+RUN cp /usr/lib/libcrypto.a ./lib/lsquic/src/lsquic/ext/libcrypto.a && \
+    cp /usr/lib/libssl.a ./lib/lsquic/src/lsquic/ext/libssl.a && \
+    cp /usr/lib/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a
 COPY ./src/ ./src/
 # TODO: .git folder is required for building – this is destructive.
 # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION.
 COPY ./.git/ ./.git/
 RUN crystal build --release --warnings all --error-on-warnings \
-# TODO: Remove next line, see https://github.com/crystal-lang/crystal/issues/7946
+    # TODO: Remove next line, see https://github.com/crystal-lang/crystal/issues/7946
     -Dmusl \
     ./src/invidious.cr
 

shard.yml

@@ -24,6 +24,9 @@ dependencies:
   protodec:
     github: omarroth/protodec
     version: ~> 0.1.2
+  lsquic:
+    github: omarroth/lsquic.cr
+    version: ~> 0.1.3
 
 crystal: 0.31.1
 

src/invidious.cr

@@ -94,7 +94,7 @@ LOCALES = {
   "zh-TW" => load_locale("zh-TW"),
 }
 
-YT_POOL     = HTTPPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
+YT_POOL     = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
 YT_IMG_POOL = HTTPPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05)
 
 config = CONFIG
@@ -1435,6 +1435,7 @@ post "/login" do |env|
     traceback = IO::Memory.new
 
     # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
+    # TODO: Convert to QUIC
     begin
       client = make_client(LOGIN_URL)
       headers = HTTP::Headers.new
@@ -1459,7 +1460,7 @@ post "/login" do |env|
 
       headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
       headers["Google-Accounts-XSRF"] = "1"
-      headers["User-Agent"] = random_user_agent
+      headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
 
       response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
       lookup_results = JSON.parse(response.body[5..-1])
@@ -4509,8 +4510,9 @@ get "/api/v1/search/suggestions" do |env|
   query ||= ""
 
   begin
-    client = make_client(URI.parse("https://suggestqueries.google.com"))
+    response = QUIC::Client.get(
-    response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body
+      "https://suggestqueries.google.com/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback"
+    ).body
 
     body = response[35..-2]
     body = JSON.parse(body).as_a

src/invidious/helpers/jobs.cr

@@ -263,7 +263,7 @@ def bypass_captcha(captcha_key, logger)
             # "proxyPort"     => CONFIG.proxy_port,
             # "proxyLogin"    => CONFIG.proxy_user,
             # "proxyPassword" => CONFIG.proxy_pass,
-            # "userAgent"     => random_user_agent,
+            # "userAgent"     => "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
           },
         }.to_json).body)
 

src/invidious/helpers/proxy.cr

@@ -90,20 +90,6 @@ class HTTPClient < HTTP::Client
 
     return opts
   end
-
-  def exec(request)
-    if self.host == "www.youtube.com"
-      request.headers["x-youtube-client-name"] ||= "1"
-      request.headers["x-youtube-client-version"] ||= "1.20180719"
-      request.headers["user-agent"] ||= random_user_agent
-      request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
-      request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-      request.headers["accept-language"] ||= "en-us,en;q=0.5"
-      request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
-    end
-
-    super
-  end
 end
 
 def get_proxies(country_code = "US")
@@ -115,6 +101,7 @@ def filter_proxies(proxies)
   proxies.select! do |proxy|
     begin
       client = HTTPClient.new(YT_URL)
+      client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
       client.read_timeout = 10.seconds
       client.connect_timeout = 10.seconds