Merge 865b7e584d into 545a5937d8

assets/js/player.js

@@ -43,6 +43,14 @@ embed_url = location.origin + '/embed/' + video_data.id + embed_url.search;
 var save_player_pos_key = 'save_player_pos';
 
 videojs.Vhs.xhr.beforeRequest = function(options) {
+    // Pass range as an URL parameter, not as a header
+    if (options.headers){
+        if (options.headers.Range) {
+            options.uri = options.uri + '&range=' + options.headers.Range.split("=")[1];
+            delete options.headers.Range;
+        }
+    }
+
     // set local if requested not videoplayback
     if (!options.uri.includes('videoplayback')) {
         if (!options.uri.includes('local=true'))

src/invidious.cr

@@ -73,9 +73,7 @@ CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz012345
 TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
 MAX_ITEMS_PER_PAGE = 1500
 
-REQUEST_HEADERS_WHITELIST  = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
+HTTP_CHUNK_SIZE = 10485760 # ~10MB
-RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
-HTTP_CHUNK_SIZE            = 10485760 # ~10MB
 
 CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
 CURRENT_COMMIT  = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}

src/invidious/config.cr

@@ -126,8 +126,10 @@ class Config
   property host_binding : String = "0.0.0.0"
   # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
   property pool_size : Int32 = 100
+
   # Use quic transport for youtube api
-  property use_quic : Bool = false
+  # Note: The getter function is defined below to take into account the compile-time flag
+  setter use_quic : Bool = false
 
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
@@ -155,6 +157,16 @@ class Config
     end
   end
 
+  # Return whether QUIC is enabled or not.
+  # Takes into account compile-time flag AND runtime config.
+  def use_quic : Bool
+    {% if flag?(:disable_quic) %}
+      return false
+    {% else %}
+      return @use_quic
+    {% end %}
+  end
+
   def self.load
     # Load config from file or YAML string env var
     env_config_file = "INVIDIOUS_CONFIG_FILE"

src/invidious/routes/images.cr

@@ -3,34 +3,17 @@ module Invidious::Routes::Images
   def self.ggpht(env)
     url = env.request.path.lchop("/ggpht")
 
-    headers = (
+    headers = HTTP::Headers.new
-      {% unless flag?(:disable_quic) %}
+    headers[":authority"] = "yt3.ggpht.com" if CONFIG.use_quic
-        if CONFIG.use_quic
-          HTTP::Headers{":authority" => "yt3.ggpht.com"}
-        else
-          HTTP::Headers.new
-        end
-      {% else %}
-        HTTP::Headers.new
-      {% end %}
-    )
 
-    REQUEST_HEADERS_WHITELIST.each do |header|
+    MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
-      if env.request.headers[header]?
-        headers[header] = env.request.headers[header]
-      end
-    end
 
     # We're encapsulating this into a proc in order to easily reuse this
     # portion of the code for each request block below.
     request_proc = ->(response : HTTP::Client::Response) {
       env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
 
+      MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
       env.response.headers["Access-Control-Allow-Origin"] = "*"
 
       if response.status_code >= 300
@@ -42,22 +25,16 @@ module Invidious::Routes::Images
     }
 
     begin
-      {% unless flag?(:disable_quic) %}
+      if CONFIG.use_quic
-        if CONFIG.use_quic
+        YT_POOL.client &.get(url, headers) do |resp|
-          YT_POOL.client &.get(url, headers) do |resp|
+          return request_proc.call(resp)
-            return request_proc.call(resp)
-          end
-        else
-          HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
-            return request_proc.call(resp)
-          end
         end
-      {% else %}
+      else
         # This can likely be optimized into a (small) pool sometime in the future.
         HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
           return request_proc.call(resp)
         end
-      {% end %}
+      end
     rescue ex
     end
   end
@@ -77,25 +54,14 @@ module Invidious::Routes::Images
     url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}"
 
     headers = HTTP::Headers.new
+    headers[":authority"] = "#{authority}.ytimg.com" if CONFIG.use_quic
 
-    {% unless flag?(:disable_quic) %}
+    MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
-      headers[":authority"] = "#{authority}.ytimg.com"
-    {% end %}
-
-    REQUEST_HEADERS_WHITELIST.each do |header|
-      if env.request.headers[header]?
-        headers[header] = env.request.headers[header]
-      end
-    end
 
     request_proc = ->(response : HTTP::Client::Response) {
       env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
 
+      MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
       env.response.headers["Connection"] = "close"
       env.response.headers["Access-Control-Allow-Origin"] = "*"
 
@@ -107,22 +73,16 @@ module Invidious::Routes::Images
     }
 
     begin
-      {% unless flag?(:disable_quic) %}
+      if CONFIG.use_quic
-        if CONFIG.use_quic
+        YT_POOL.client &.get(url, headers) do |resp|
-          YT_POOL.client &.get(url, headers) do |resp|
+          return request_proc.call(resp)
-            return request_proc.call(resp)
-          end
-        else
-          HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
-            return request_proc.call(resp)
-          end
         end
-      {% else %}
+      else
         # This can likely be optimized into a (small) pool sometime in the future.
         HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
           return request_proc.call(resp)
         end
-      {% end %}
+      end
     rescue ex
     end
   end
@@ -133,32 +93,15 @@ module Invidious::Routes::Images
     name = env.params.url["name"]
     url = env.request.resource
 
-    headers = (
+    headers = HTTP::Headers.new
-      {% unless flag?(:disable_quic) %}
+    headers[":authority"] = "i9.ytimg.com" if CONFIG.use_quic
-        if CONFIG.use_quic
-          HTTP::Headers{":authority" => "i9.ytimg.com"}
-        else
-          HTTP::Headers.new
-        end
-      {% else %}
-        HTTP::Headers.new
-      {% end %}
-    )
 
-    REQUEST_HEADERS_WHITELIST.each do |header|
+    MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
-      if env.request.headers[header]?
-        headers[header] = env.request.headers[header]
-      end
-    end
 
     request_proc = ->(response : HTTP::Client::Response) {
       env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
 
+      MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
       env.response.headers["Access-Control-Allow-Origin"] = "*"
 
       if response.status_code >= 300 && response.status_code != 404
@@ -169,43 +112,29 @@ module Invidious::Routes::Images
     }
 
     begin
-      {% unless flag?(:disable_quic) %}
+      if CONFIG.use_quic
-        if CONFIG.use_quic
+        YT_POOL.client &.get(url, headers) do |resp|
-          YT_POOL.client &.get(url, headers) do |resp|
+          return request_proc.call(resp)
-            return request_proc.call(resp)
-          end
-        else
-          HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
-            return request_proc.call(resp)
-          end
         end
-      {% else %}
+      else
         # This can likely be optimized into a (small) pool sometime in the future.
         HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
           return request_proc.call(resp)
         end
-      {% end %}
+      end
     rescue ex
     end
   end
 
   def self.yts_image(env)
     headers = HTTP::Headers.new
-    REQUEST_HEADERS_WHITELIST.each do |header|
+    MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
-      if env.request.headers[header]?
-        headers[header] = env.request.headers[header]
-      end
-    end
 
     begin
       YT_POOL.client &.get(env.request.resource, headers) do |response|
         env.response.status_code = response.status_code
-        response.headers.each do |key, value|
-          if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-            env.response.headers[key] = value
-          end
-        end
 
+        MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
         env.response.headers["Access-Control-Allow-Origin"] = "*"
 
         if response.status_code >= 300 && response.status_code != 404
@@ -223,60 +152,37 @@ module Invidious::Routes::Images
     id = env.params.url["id"]
     name = env.params.url["name"]
 
-    headers = (
+    headers = HTTP::Headers.new
-      {% unless flag?(:disable_quic) %}
+    headers[":authority"] = "i.ytimg.com" if CONFIG.use_quic
-        if CONFIG.use_quic
-          HTTP::Headers{":authority" => "i.ytimg.com"}
-        else
-          HTTP::Headers.new
-        end
-      {% else %}
-        HTTP::Headers.new
-      {% end %}
-    )
 
     if name == "maxres.jpg"
       build_thumbnails(id).each do |thumb|
         thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
+
         # Logic here is short enough that manually typing them out should be fine.
-        {% unless flag?(:disable_quic) %}
+        if CONFIG.use_quic
-          if CONFIG.use_quic
+          if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
-            if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
+            name = thumb[:url] + ".jpg"
-              name = thumb[:url] + ".jpg"
+            break
-              break
-            end
-          else
-            if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
-              name = thumb[:url] + ".jpg"
-              break
-            end
           end
-        {% else %}
+        else
           # This can likely be optimized into a (small) pool sometime in the future.
           if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
             name = thumb[:url] + ".jpg"
             break
           end
-        {% end %}
+        end
       end
     end
 
     url = "/vi/#{id}/#{name}"
 
-    REQUEST_HEADERS_WHITELIST.each do |header|
+    MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
-      if env.request.headers[header]?
-        headers[header] = env.request.headers[header]
-      end
-    end
 
     request_proc = ->(response : HTTP::Client::Response) {
       env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
 
+      MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
       env.response.headers["Access-Control-Allow-Origin"] = "*"
 
       if response.status_code >= 300 && response.status_code != 404
@@ -287,22 +193,16 @@ module Invidious::Routes::Images
     }
 
     begin
-      {% unless flag?(:disable_quic) %}
+      if CONFIG.use_quic
-        if CONFIG.use_quic
+        YT_POOL.client &.get(url, headers) do |resp|
-          YT_POOL.client &.get(url, headers) do |resp|
+          return request_proc.call(resp)
-            return request_proc.call(resp)
-          end
-        else
-          HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
-            return request_proc.call(resp)
-          end
         end
-      {% else %}
+      else
         # This can likely be optimized into a (small) pool sometime in the future.
         HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
           return request_proc.call(resp)
         end
-      {% end %}
+      end
     rescue ex
     end
   end

src/invidious/routes/video_playback.cr

@@ -8,13 +8,17 @@ module Invidious::Routes::VideoPlayback
     mns = query_params["mn"]?.try &.split(",")
     mns ||= [] of String
 
-    if query_params["region"]?
+    # Extract some invidious-specific parameters
-      region = query_params["region"]
+
+    if region = query_params["region"]?
       query_params.delete("region")
     end
 
-    if query_params["host"]? && !query_params["host"].empty?
+    if title = query_params["title"]?
-      host = query_params["host"]
+      query_params.delete("title")
+    end
+
+    if host = query_params["host"]?
       query_params.delete("host")
     else
       host = "r#{fvip}---#{mns.pop}.googlevideo.com"
@@ -25,22 +29,31 @@ module Invidious::Routes::VideoPlayback
       return error_template(400, "Invalid \"host\" parameter.")
     end
 
+    # Range manipulation
+
+    has_range_param = false
+    has_range_header = false
+
+    if range = query_params["range"]?
+      query_params.delete("range")
+      has_range_param = true
+    end
+
+    if range_header = env.request.headers["Range"]?
+      env.request.headers.delete("Range")
+      range ||= range_header.split('=')[1]
+      has_range_header = true if !has_range_param
+    end
+
+    # Skip redirections
+
     host = "https://#{host}"
     url = "/videoplayback?#{query_params}"
 
     headers = HTTP::Headers.new
-    REQUEST_HEADERS_WHITELIST.each do |header|
+    MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
-      if env.request.headers[header]?
-        headers[header] = env.request.headers[header]
-      end
-    end
 
-    # See: https://github.com/iv-org/invidious/issues/3302
+    headers["Range"] = "bytes=" + (range || "0-")
-    range_header = env.request.headers["Range"]?
-    if range_header.nil?
-      range_for_head = query_params["range"]? || "0-640"
-      headers["Range"] = "bytes=#{range_for_head}"
-    end
 
     client = make_client(URI.parse(host), region)
     response = HTTP::Client::Response.new(500)
@@ -78,7 +91,7 @@ module Invidious::Routes::VideoPlayback
     end
 
     # Remove the Range header added previously.
-    headers.delete("Range") if range_header.nil?
+    headers.delete("Range")
 
     if response.status_code >= 400
       env.response.content_type = "text/plain"
@@ -90,34 +103,24 @@ module Invidious::Routes::VideoPlayback
         return error_template(403, "Administrator has disabled this endpoint.")
       end
 
-      begin
+      # Add back the range header. Important!
-        client.get(url, headers) do |resp|
+      headers["Range"] = "bytes=" + (range || "0-")
-          resp.headers.each do |key, value|
-            if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-              env.response.headers[key] = value
-            end
-          end
 
-          env.response.headers["Access-Control-Allow-Origin"] = "*"
+      MediaProxy.proxy_dash_chunk(env, client, url, region)
-
+    elsif has_range_param
-          if location = resp.headers["Location"]?
+      if CONFIG.disabled?("dash")
-            url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
+        return error_template(403, "Administrator has disabled this endpoint.")
-            return env.redirect url
-          end
-
-          IO.copy(resp.body_io, env.response)
-        end
-      rescue ex
       end
+
+      MediaProxy.proxy_dash_chunk(env, client, url + "&range=#{range}", region)
     else
-      if query_params["title"]? && CONFIG.disabled?("downloads") ||
+      if (title && CONFIG.disabled?("downloads")) || (title.nil? && CONFIG.disabled?("local"))
-         CONFIG.disabled?("dash")
         return error_template(403, "Administrator has disabled this endpoint.")
       end
 
       content_length = nil
       first_chunk = true
-      range_start, range_end = parse_range(env.request.headers["Range"]?)
+      range_start, range_end = parse_range(range)
       chunk_start = range_start
       chunk_end = range_end
 
@@ -144,20 +147,12 @@ module Invidious::Routes::VideoPlayback
         begin
           client.get(url, headers) do |resp|
             if first_chunk
-              if !env.request.headers["Range"]? && resp.status_code == 206
+              if !has_range_header && resp.status_code == 206
                 env.response.status_code = 200
               else
                 env.response.status_code = resp.status_code
               end
 
-              resp.headers.each do |key, value|
-                if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range"
-                  env.response.headers[key] = value
-                end
-              end
-
-              env.response.headers["Access-Control-Allow-Origin"] = "*"
-
               if location = resp.headers["Location"]?
                 location = URI.parse(location)
                 location = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
@@ -166,8 +161,12 @@ module Invidious::Routes::VideoPlayback
                 break
               end
 
-              if title = query_params["title"]?
+              MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers)
-                # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
+              env.response.headers.delete("Content-Range") # Important!
+              env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+              if title
+                # http://web.archive.org/web/20150701003254/https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
                 filename = URI.encode_www_form(title, space_to_plus: false)
                 header = "attachment; filename=\"#{filename}\"; filename*=UTF-8''#{filename}"
                 env.response.headers["Content-Disposition"] = header
@@ -175,7 +174,7 @@ module Invidious::Routes::VideoPlayback
 
               if !resp.headers.includes_word?("Transfer-Encoding", "chunked")
                 content_length = resp.headers["Content-Range"].split("/")[-1].to_i64
-                if env.request.headers["Range"]?
+                if has_range_header
                   env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}"
                   env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start
                 else

src/invidious/yt_backend/connection_pool.cr

@@ -63,11 +63,7 @@ struct YoutubeConnectionPool
     DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
       conn = nil # Declare
       {% unless flag?(:disable_quic) %}
-        if use_quic
+        conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url)
-          conn = QUIC::Client.new(url)
-        else
-          conn = HTTP::Client.new(url)
-        end
       {% else %}
         conn = HTTP::Client.new(url)
       {% end %}

src/invidious/yt_backend/media_proxy.cr

@@ -0,0 +1,70 @@
+module Invidious::MediaProxy
+  extend self
+
+  # -------------------
+  #  Constants
+  # -------------------
+
+  private REQUEST_HEADERS_WHITELIST = {
+    "accept", "accept-encoding", "cache-control",
+    "content-length", "if-none-match", "range",
+  }
+
+  private RESPONSE_HEADERS_BLACKLIST = {
+    "access-control-allow-origin", "alt-svc", "server",
+  }
+
+  # -------------------
+  #  Headers functions
+  # -------------------
+
+  # Copy only the selected headers from the client to youtube servers
+  # (in general, from `env.request` to a temporary `HTTP::Headers` object).
+  def copy_request_headers(*, from : HTTP::Headers, to : HTTP::Headers)
+    REQUEST_HEADERS_WHITELIST.each do |header|
+      to[header] = from[header] if from[header]?
+    end
+  end
+
+  # Copy only the selected headers from youtube servers to the client
+  # (generally, from a response block to `env.response`).
+  def copy_response_headers(*, from : HTTP::Headers, to : HTTP::Headers)
+    from.each do |key, value|
+      if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+        to[key] = value
+      end
+    end
+  end
+
+  # -------------------
+  #  Proxy functions
+  # -------------------
+
+  def proxy_dash_chunk(
+    env : HTTP::Server::Context,
+    client : HTTP::Client,
+    url : URI | String,
+    region : String?
+  )
+    headers = HTTP::Headers.new
+    self.copy_request_headers(from: env.request.headers, to: headers)
+
+    # Make sure to remove a potential range header, to avoid throttling
+    headers.delete("Range")
+
+    client.get(url, headers) do |resp|
+      env.response.status_code = resp.status_code
+
+      self.copy_response_headers(from: resp.headers, to: env.response.headers)
+      env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+      if location = resp.headers["Location"]?
+        url = HttpServer::Utils.proxy_video_url(location, region: region)
+        env.redirect url
+      else
+        IO.copy(resp.body_io, env.response)
+      end
+    end
+  rescue ex
+  end
+end

src/invidious/yt_backend/youtube_api.cr

@@ -593,7 +593,7 @@ module YoutubeAPI
     LOGGER.trace("YoutubeAPI: POST data: #{data}")
 
     # Send the POST request
-    if {{ !flag?(:disable_quic) }} && CONFIG.use_quic
+    if CONFIG.use_quic
       # Using QUIC client
       body = YT_POOL.client(client_config.proxy_region,
         &.post(url, headers: headers, body: data.to_json)