From 7e9c74423e4f3721f4884fe3e0ff5b93b093d71c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 5 Mar 2023 00:12:24 +0100 Subject: [PATCH 1/5] Config: add 'use_quic' getter that respects compile-time flag --- src/invidious/config.cr | 14 ++- src/invidious/routes/images.cr | 128 ++++++-------------- src/invidious/yt_backend/connection_pool.cr | 6 +- src/invidious/yt_backend/youtube_api.cr | 2 +- 4 files changed, 49 insertions(+), 101 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 9fc58409..3856a454 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -126,8 +126,10 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # Use quic transport for youtube api - property use_quic : Bool = false + # Note: The getter function is defined below to take into account the compile-time flag + setter use_quic : Bool = false # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] @@ -155,6 +157,16 @@ class Config end end + # Return whether QUIC is enabled or not. + # Takes into account compile-time flag AND runtime config. + def use_quic : Bool + {% if flag?(:disable_quic) %} + return false + {% else %} + return @use_quic + {% end %} + end + def self.load # Load config from file or YAML string env var env_config_file = "INVIDIOUS_CONFIG_FILE" diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 594a7869..d6f6d3db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -3,17 +3,8 @@ module Invidious::Routes::Images def self.ggpht(env) url = env.request.path.lchop("/ggpht") - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "yt3.ggpht.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new + headers[":authority"] = "yt3.ggpht.com" if CONFIG.use_quic REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -42,22 +33,16 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end @@ -77,10 +62,7 @@ module Invidious::Routes::Images url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" headers = HTTP::Headers.new - - {% unless flag?(:disable_quic) %} - headers[":authority"] = "#{authority}.ytimg.com" - {% end %} + headers[":authority"] = "#{authority}.ytimg.com" if CONFIG.use_quic REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -107,22 +89,16 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end @@ -133,17 +109,8 @@ module Invidious::Routes::Images name = env.params.url["name"] url = env.request.resource - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i9.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new + headers[":authority"] = "i9.ytimg.com" if CONFIG.use_quic REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -169,22 +136,16 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end @@ -223,41 +184,26 @@ module Invidious::Routes::Images id = env.params.url["id"] name = env.params.url["name"] - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new + headers[":authority"] = "i.ytimg.com" if CONFIG.use_quic if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" + # Logic here is short enough that manually typing them out should be fine. - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end - else - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 - name = thumb[:url] + ".jpg" - break - end + if CONFIG.use_quic + if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 + name = thumb[:url] + ".jpg" + break end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 name = thumb[:url] + ".jpg" break end - {% end %} + end end end @@ -287,22 +233,16 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 46e5bf85..8d727129 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -63,11 +63,7 @@ struct YoutubeConnectionPool DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = nil # Declare {% unless flag?(:disable_quic) %} - if use_quic - conn = QUIC::Client.new(url) - else - conn = HTTP::Client.new(url) - end + conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url) {% else %} conn = HTTP::Client.new(url) {% end %} diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 91a9332c..7a2907be 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -593,7 +593,7 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - if {{ !flag?(:disable_quic) }} && CONFIG.use_quic + if CONFIG.use_quic # Using QUIC client body = YT_POOL.client(client_config.proxy_region, &.post(url, headers: headers, body: data.to_json) From e62d61abc1297620501103d5c7aea9df3a2e95bc Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 5 Mar 2023 00:55:27 +0100 Subject: [PATCH 2/5] Proxy: de-duplicate the code that copies headers --- src/invidious.cr | 4 +- src/invidious/routes/images.cr | 60 +++++-------------------- src/invidious/routes/video_playback.cr | 21 ++------- src/invidious/yt_backend/media_proxy.cr | 38 ++++++++++++++++ 4 files changed, 53 insertions(+), 70 deletions(-) create mode 100644 src/invidious/yt_backend/media_proxy.cr diff --git a/src/invidious.cr b/src/invidious.cr index d4f8e0fb..a6f12ba6 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -71,9 +71,7 @@ CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz012345 TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} MAX_ITEMS_PER_PAGE = 1500 -REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"} -RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"} -HTTP_CHUNK_SIZE = 10485760 # ~10MB +HTTP_CHUNK_SIZE = 10485760 # ~10MB CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index d6f6d3db..db7359b1 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -6,22 +6,14 @@ module Invidious::Routes::Images headers = HTTP::Headers.new headers[":authority"] = "yt3.ggpht.com" if CONFIG.use_quic - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) # We're encapsulating this into a proc in order to easily reuse this # portion of the code for each request block below. request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 @@ -64,20 +56,12 @@ module Invidious::Routes::Images headers = HTTP::Headers.new headers[":authority"] = "#{authority}.ytimg.com" if CONFIG.use_quic - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Connection"] = "close" env.response.headers["Access-Control-Allow-Origin"] = "*" @@ -112,20 +96,12 @@ module Invidious::Routes::Images headers = HTTP::Headers.new headers[":authority"] = "i9.ytimg.com" if CONFIG.use_quic - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 && response.status_code != 404 @@ -152,21 +128,13 @@ module Invidious::Routes::Images def self.yts_image(env) headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) begin YT_POOL.client &.get(env.request.resource, headers) do |response| env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 && response.status_code != 404 @@ -209,20 +177,12 @@ module Invidious::Routes::Images url = "/vi/#{id}/#{name}" - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 && response.status_code != 404 diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 1e932d11..63559ab5 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -29,11 +29,7 @@ module Invidious::Routes::VideoPlayback url = "/videoplayback?#{query_params}" headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) # See: https://github.com/iv-org/invidious/issues/3302 range_header = env.request.headers["Range"]? @@ -92,12 +88,7 @@ module Invidious::Routes::VideoPlayback begin client.get(url, headers) do |resp| - resp.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - + MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? @@ -150,12 +141,8 @@ module Invidious::Routes::VideoPlayback env.response.status_code = resp.status_code end - resp.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range" - env.response.headers[key] = value - end - end - + MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers) + env.response.headers.delete("Content-Range") # Important! env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? diff --git a/src/invidious/yt_backend/media_proxy.cr b/src/invidious/yt_backend/media_proxy.cr new file mode 100644 index 00000000..3eebb3ad --- /dev/null +++ b/src/invidious/yt_backend/media_proxy.cr @@ -0,0 +1,38 @@ +module Invidious::MediaProxy + extend self + + # ------------------- + # Constants + # ------------------- + + private REQUEST_HEADERS_WHITELIST = { + "accept", "accept-encoding", "cache-control", + "content-length", "if-none-match", "range", + } + + private RESPONSE_HEADERS_BLACKLIST = { + "access-control-allow-origin", "alt-svc", "server", + } + + # ------------------- + # Headers functions + # ------------------- + + # Copy only the selected headers from the client to youtube servers + # (in general, from `env.request` to a temporary `HTTP::Headers` object). + def copy_request_headers(*, from : HTTP::Headers, to : HTTP::Headers) + REQUEST_HEADERS_WHITELIST.each do |header| + to[header] = from[header] if from[header]? + end + end + + # Copy only the selected headers from youtube servers to the client + # (generally, from a response block to `env.response`). + def copy_response_headers(*, from : HTTP::Headers, to : HTTP::Headers) + from.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + to[key] = value + end + end + end +end From 3474744f577a771718d0382ac634aa39424bf811 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 5 Mar 2023 13:43:13 +0100 Subject: [PATCH 3/5] Proxy: pass range as URL parameter to avoid throttling --- assets/js/player.js | 8 +++ src/invidious/routes/video_playback.cr | 79 ++++++++++++++----------- src/invidious/yt_backend/media_proxy.cr | 32 ++++++++++ 3 files changed, 86 insertions(+), 33 deletions(-) diff --git a/assets/js/player.js b/assets/js/player.js index ee678663..5bc371dc 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -43,6 +43,14 @@ embed_url = location.origin + '/embed/' + video_data.id + embed_url.search; var save_player_pos_key = 'save_player_pos'; videojs.Vhs.xhr.beforeRequest = function(options) { + // Pass range as an URL parameter, not as a header + if (options.headers){ + if (options.headers.Range) { + options.uri = options.uri + '&range=' + options.headers.Range.split("=")[1]; + delete options.headers.Range; + } + } + // set local if requested not videoplayback if (!options.uri.includes('videoplayback')) { if (!options.uri.includes('local=true')) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 63559ab5..d27b214d 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -8,13 +8,17 @@ module Invidious::Routes::VideoPlayback mns = query_params["mn"]?.try &.split(",") mns ||= [] of String - if query_params["region"]? - region = query_params["region"] + # Extract some invidious-specific parameters + + if region = query_params["region"]? query_params.delete("region") end - if query_params["host"]? && !query_params["host"].empty? - host = query_params["host"] + if title = query_params["title"]? + query_params.delete("title") + end + + if host = query_params["host"]? query_params.delete("host") else host = "r#{fvip}---#{mns.pop}.googlevideo.com" @@ -25,17 +29,34 @@ module Invidious::Routes::VideoPlayback return error_template(400, "Invalid \"host\" parameter.") end + # Range manipulation + + has_range_param = false + has_range_header = false + + if range = query_params["range"]? + query_params.delete("range") + has_range_param = true + end + + if range_header = env.request.headers["Range"]? + env.request.headers.delete("Range") + range ||= range_header.split('=')[1] + has_range_header = true if !has_range_param + end + + # Skip redirections + host = "https://#{host}" url = "/videoplayback?#{query_params}" headers = HTTP::Headers.new MediaProxy.copy_request_headers(from: env.request.headers, to: headers) - # See: https://github.com/iv-org/invidious/issues/3302 - range_header = env.request.headers["Range"]? - if range_header.nil? - range_for_head = query_params["range"]? || "0-640" - headers["Range"] = "bytes=#{range_for_head}" + if has_range_param + url += "&range=#{range}" + else + headers["Range"] = "bytes=#{range || "0-"}" end client = make_client(URI.parse(host), region) @@ -74,7 +95,7 @@ module Invidious::Routes::VideoPlayback end # Remove the Range header added previously. - headers.delete("Range") if range_header.nil? + headers.delete("Range") if response.status_code >= 400 env.response.content_type = "text/plain" @@ -86,29 +107,21 @@ module Invidious::Routes::VideoPlayback return error_template(403, "Administrator has disabled this endpoint.") end - begin - client.get(url, headers) do |resp| - MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers) - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if location = resp.headers["Location"]? - url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) - return env.redirect url - end - - IO.copy(resp.body_io, env.response) - end - rescue ex + MediaProxy.proxy_dash_chunk(env, client, url, region) + elsif has_range_param + if CONFIG.disabled?("dash") + return error_template(403, "Administrator has disabled this endpoint.") end + + MediaProxy.proxy_dash_chunk(env, client, url, region) else - if query_params["title"]? && CONFIG.disabled?("downloads") || - CONFIG.disabled?("dash") + if (title && CONFIG.disabled?("downloads")) || (title.nil? && CONFIG.disabled?("local")) return error_template(403, "Administrator has disabled this endpoint.") end content_length = nil first_chunk = true - range_start, range_end = parse_range(env.request.headers["Range"]?) + range_start, range_end = parse_range(range) chunk_start = range_start chunk_end = range_end @@ -135,16 +148,12 @@ module Invidious::Routes::VideoPlayback begin client.get(url, headers) do |resp| if first_chunk - if !env.request.headers["Range"]? && resp.status_code == 206 + if !has_range_header && resp.status_code == 206 env.response.status_code = 200 else env.response.status_code = resp.status_code end - MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers) - env.response.headers.delete("Content-Range") # Important! - env.response.headers["Access-Control-Allow-Origin"] = "*" - if location = resp.headers["Location"]? location = URI.parse(location) location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -153,7 +162,11 @@ module Invidious::Routes::VideoPlayback break end - if title = query_params["title"]? + MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers) + env.response.headers.delete("Content-Range") # Important! + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if title # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ filename = URI.encode_www_form(title, space_to_plus: false) header = "attachment; filename=\"#{filename}\"; filename*=UTF-8''#{filename}" @@ -162,7 +175,7 @@ module Invidious::Routes::VideoPlayback if !resp.headers.includes_word?("Transfer-Encoding", "chunked") content_length = resp.headers["Content-Range"].split("/")[-1].to_i64 - if env.request.headers["Range"]? + if has_range_header env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}" env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start else diff --git a/src/invidious/yt_backend/media_proxy.cr b/src/invidious/yt_backend/media_proxy.cr index 3eebb3ad..1b246a7e 100644 --- a/src/invidious/yt_backend/media_proxy.cr +++ b/src/invidious/yt_backend/media_proxy.cr @@ -35,4 +35,36 @@ module Invidious::MediaProxy end end end + + # ------------------- + # Proxy functions + # ------------------- + + def proxy_dash_chunk( + env : HTTP::Server::Context, + client : HTTP::Client, + url : URI | String, + region : String? + ) + headers = HTTP::Headers.new + self.copy_request_headers(from: env.request.headers, to: headers) + + # Make sure to remove a potential range header, to avoid throttling + headers.delete("Range") + + client.get(url, headers) do |resp| + env.response.status_code = resp.status_code + + self.copy_response_headers(from: resp.headers, to: env.response.headers) + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if location = resp.headers["Location"]? + url = HttpServer::Utils.proxy_video_url(location, region: region) + env.redirect url + else + IO.copy(resp.body_io, env.response) + end + end + rescue ex + end end From a743d01a10a700fbcaed65bced00217f49282047 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 5 Mar 2023 13:44:23 +0100 Subject: [PATCH 4/5] Download: update link in video_playback comment --- src/invidious/routes/video_playback.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index d27b214d..1d1c1648 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -167,7 +167,7 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if title - # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ + # http://web.archive.org/web/20150701003254/https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ filename = URI.encode_www_form(title, space_to_plus: false) header = "attachment; filename=\"#{filename}\"; filename*=UTF-8''#{filename}" env.response.headers["Content-Disposition"] = header From 865b7e584d16aa8947b3f0449bf4c048ca01a7f4 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 8 Mar 2023 19:38:15 +0100 Subject: [PATCH 5/5] Proxy: fix the throttling issue once for all --- src/invidious/routes/video_playback.cr | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 1d1c1648..562a1ed3 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -53,11 +53,7 @@ module Invidious::Routes::VideoPlayback headers = HTTP::Headers.new MediaProxy.copy_request_headers(from: env.request.headers, to: headers) - if has_range_param - url += "&range=#{range}" - else - headers["Range"] = "bytes=#{range || "0-"}" - end + headers["Range"] = "bytes=" + (range || "0-") client = make_client(URI.parse(host), region) response = HTTP::Client::Response.new(500) @@ -107,13 +103,16 @@ module Invidious::Routes::VideoPlayback return error_template(403, "Administrator has disabled this endpoint.") end + # Add back the range header. Important! + headers["Range"] = "bytes=" + (range || "0-") + MediaProxy.proxy_dash_chunk(env, client, url, region) elsif has_range_param if CONFIG.disabled?("dash") return error_template(403, "Administrator has disabled this endpoint.") end - MediaProxy.proxy_dash_chunk(env, client, url, region) + MediaProxy.proxy_dash_chunk(env, client, url + "&range=#{range}", region) else if (title && CONFIG.disabled?("downloads")) || (title.nil? && CONFIG.disabled?("local")) return error_template(403, "Administrator has disabled this endpoint.")