diff --git a/assets/js/player.js b/assets/js/player.js index bb53ac24..5b6facf9 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -43,6 +43,14 @@ embed_url = location.origin + '/embed/' + video_data.id + embed_url.search; var save_player_pos_key = 'save_player_pos'; videojs.Vhs.xhr.beforeRequest = function(options) { + // Pass range as an URL parameter, not as a header + if (options.headers){ + if (options.headers.Range) { + options.uri = options.uri + '&range=' + options.headers.Range.split("=")[1]; + delete options.headers.Range; + } + } + // set local if requested not videoplayback if (!options.uri.includes('videoplayback')) { if (!options.uri.includes('local=true')) diff --git a/src/invidious.cr b/src/invidious.cr index 27c4775e..fa89deb5 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -73,9 +73,7 @@ CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz012345 TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} MAX_ITEMS_PER_PAGE = 1500 -REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"} -RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"} -HTTP_CHUNK_SIZE = 10485760 # ~10MB +HTTP_CHUNK_SIZE = 10485760 # ~10MB CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 9fc58409..3856a454 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -126,8 +126,10 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # Use quic transport for youtube api - property use_quic : Bool = false + # Note: The getter function is defined below to take into account the compile-time flag + setter use_quic : Bool = false # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] @@ -155,6 +157,16 @@ class Config end end + # Return whether QUIC is enabled or not. + # Takes into account compile-time flag AND runtime config. + def use_quic : Bool + {% if flag?(:disable_quic) %} + return false + {% else %} + return @use_quic + {% end %} + end + def self.load # Load config from file or YAML string env var env_config_file = "INVIDIOUS_CONFIG_FILE" diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 594a7869..db7359b1 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -3,34 +3,17 @@ module Invidious::Routes::Images def self.ggpht(env) url = env.request.path.lchop("/ggpht") - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "yt3.ggpht.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new + headers[":authority"] = "yt3.ggpht.com" if CONFIG.use_quic - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) # We're encapsulating this into a proc in order to easily reuse this # portion of the code for each request block below. request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 @@ -42,22 +25,16 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end @@ -77,25 +54,14 @@ module Invidious::Routes::Images url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" headers = HTTP::Headers.new + headers[":authority"] = "#{authority}.ytimg.com" if CONFIG.use_quic - {% unless flag?(:disable_quic) %} - headers[":authority"] = "#{authority}.ytimg.com" - {% end %} - - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Connection"] = "close" env.response.headers["Access-Control-Allow-Origin"] = "*" @@ -107,22 +73,16 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end @@ -133,32 +93,15 @@ module Invidious::Routes::Images name = env.params.url["name"] url = env.request.resource - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i9.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new + headers[":authority"] = "i9.ytimg.com" if CONFIG.use_quic - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 && response.status_code != 404 @@ -169,43 +112,29 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end def self.yts_image(env) headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) begin YT_POOL.client &.get(env.request.resource, headers) do |response| env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 && response.status_code != 404 @@ -223,60 +152,37 @@ module Invidious::Routes::Images id = env.params.url["id"] name = env.params.url["name"] - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new + headers[":authority"] = "i.ytimg.com" if CONFIG.use_quic if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" + # Logic here is short enough that manually typing them out should be fine. - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end - else - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 - name = thumb[:url] + ".jpg" - break - end + if CONFIG.use_quic + if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 + name = thumb[:url] + ".jpg" + break end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 name = thumb[:url] + ".jpg" break end - {% end %} + end end end url = "/vi/#{id}/#{name}" - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) request_proc = ->(response : HTTP::Client::Response) { env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers) env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 && response.status_code != 404 @@ -287,22 +193,16 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) end - {% else %} + else # This can likely be optimized into a (small) pool sometime in the future. HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| return request_proc.call(resp) end - {% end %} + end rescue ex end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 9641e01a..d30b6cd8 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -8,13 +8,17 @@ module Invidious::Routes::VideoPlayback mns = query_params["mn"]?.try &.split(",") mns ||= [] of String - if query_params["region"]? - region = query_params["region"] + # Extract some invidious-specific parameters + + if region = query_params["region"]? query_params.delete("region") end - if query_params["host"]? && !query_params["host"].empty? - host = query_params["host"] + if title = query_params["title"]? + query_params.delete("title") + end + + if host = query_params["host"]? query_params.delete("host") else host = "r#{fvip}---#{mns.pop}.googlevideo.com" @@ -25,22 +29,31 @@ module Invidious::Routes::VideoPlayback return error_template(400, "Invalid \"host\" parameter.") end + # Range manipulation + + has_range_param = false + has_range_header = false + + if range = query_params["range"]? + query_params.delete("range") + has_range_param = true + end + + if range_header = env.request.headers["Range"]? + env.request.headers.delete("Range") + range ||= range_header.split('=')[1] + has_range_header = true if !has_range_param + end + + # Skip redirections + host = "https://#{host}" url = "/videoplayback?#{query_params}" headers = HTTP::Headers.new - REQUEST_HEADERS_WHITELIST.each do |header| - if env.request.headers[header]? - headers[header] = env.request.headers[header] - end - end + MediaProxy.copy_request_headers(from: env.request.headers, to: headers) - # See: https://github.com/iv-org/invidious/issues/3302 - range_header = env.request.headers["Range"]? - if range_header.nil? - range_for_head = query_params["range"]? || "0-640" - headers["Range"] = "bytes=#{range_for_head}" - end + headers["Range"] = "bytes=" + (range || "0-") client = make_client(URI.parse(host), region) response = HTTP::Client::Response.new(500) @@ -78,7 +91,7 @@ module Invidious::Routes::VideoPlayback end # Remove the Range header added previously. - headers.delete("Range") if range_header.nil? + headers.delete("Range") if response.status_code >= 400 env.response.content_type = "text/plain" @@ -90,34 +103,24 @@ module Invidious::Routes::VideoPlayback return error_template(403, "Administrator has disabled this endpoint.") end - begin - client.get(url, headers) do |resp| - resp.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end + # Add back the range header. Important! + headers["Range"] = "bytes=" + (range || "0-") - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if location = resp.headers["Location"]? - url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) - return env.redirect url - end - - IO.copy(resp.body_io, env.response) - end - rescue ex + MediaProxy.proxy_dash_chunk(env, client, url, region) + elsif has_range_param + if CONFIG.disabled?("dash") + return error_template(403, "Administrator has disabled this endpoint.") end + + MediaProxy.proxy_dash_chunk(env, client, url + "&range=#{range}", region) else - if query_params["title"]? && CONFIG.disabled?("downloads") || - CONFIG.disabled?("dash") + if (title && CONFIG.disabled?("downloads")) || (title.nil? && CONFIG.disabled?("local")) return error_template(403, "Administrator has disabled this endpoint.") end content_length = nil first_chunk = true - range_start, range_end = parse_range(env.request.headers["Range"]?) + range_start, range_end = parse_range(range) chunk_start = range_start chunk_end = range_end @@ -144,20 +147,12 @@ module Invidious::Routes::VideoPlayback begin client.get(url, headers) do |resp| if first_chunk - if !env.request.headers["Range"]? && resp.status_code == 206 + if !has_range_header && resp.status_code == 206 env.response.status_code = 200 else env.response.status_code = resp.status_code end - resp.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range" - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - if location = resp.headers["Location"]? location = URI.parse(location) location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -166,8 +161,12 @@ module Invidious::Routes::VideoPlayback break end - if title = query_params["title"]? - # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ + MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers) + env.response.headers.delete("Content-Range") # Important! + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if title + # http://web.archive.org/web/20150701003254/https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ filename = URI.encode_www_form(title, space_to_plus: false) header = "attachment; filename=\"#{filename}\"; filename*=UTF-8''#{filename}" env.response.headers["Content-Disposition"] = header @@ -175,7 +174,7 @@ module Invidious::Routes::VideoPlayback if !resp.headers.includes_word?("Transfer-Encoding", "chunked") content_length = resp.headers["Content-Range"].split("/")[-1].to_i64 - if env.request.headers["Range"]? + if has_range_header env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}" env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start else diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 46e5bf85..8d727129 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -63,11 +63,7 @@ struct YoutubeConnectionPool DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = nil # Declare {% unless flag?(:disable_quic) %} - if use_quic - conn = QUIC::Client.new(url) - else - conn = HTTP::Client.new(url) - end + conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url) {% else %} conn = HTTP::Client.new(url) {% end %} diff --git a/src/invidious/yt_backend/media_proxy.cr b/src/invidious/yt_backend/media_proxy.cr new file mode 100644 index 00000000..1b246a7e --- /dev/null +++ b/src/invidious/yt_backend/media_proxy.cr @@ -0,0 +1,70 @@ +module Invidious::MediaProxy + extend self + + # ------------------- + # Constants + # ------------------- + + private REQUEST_HEADERS_WHITELIST = { + "accept", "accept-encoding", "cache-control", + "content-length", "if-none-match", "range", + } + + private RESPONSE_HEADERS_BLACKLIST = { + "access-control-allow-origin", "alt-svc", "server", + } + + # ------------------- + # Headers functions + # ------------------- + + # Copy only the selected headers from the client to youtube servers + # (in general, from `env.request` to a temporary `HTTP::Headers` object). + def copy_request_headers(*, from : HTTP::Headers, to : HTTP::Headers) + REQUEST_HEADERS_WHITELIST.each do |header| + to[header] = from[header] if from[header]? + end + end + + # Copy only the selected headers from youtube servers to the client + # (generally, from a response block to `env.response`). + def copy_response_headers(*, from : HTTP::Headers, to : HTTP::Headers) + from.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + to[key] = value + end + end + end + + # ------------------- + # Proxy functions + # ------------------- + + def proxy_dash_chunk( + env : HTTP::Server::Context, + client : HTTP::Client, + url : URI | String, + region : String? + ) + headers = HTTP::Headers.new + self.copy_request_headers(from: env.request.headers, to: headers) + + # Make sure to remove a potential range header, to avoid throttling + headers.delete("Range") + + client.get(url, headers) do |resp| + env.response.status_code = resp.status_code + + self.copy_response_headers(from: resp.headers, to: env.response.headers) + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if location = resp.headers["Location"]? + url = HttpServer::Utils.proxy_video_url(location, region: region) + env.redirect url + else + IO.copy(resp.body_io, env.response) + end + end + rescue ex + end +end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 91a9332c..7a2907be 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -593,7 +593,7 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - if {{ !flag?(:disable_quic) }} && CONFIG.use_quic + if CONFIG.use_quic # Using QUIC client body = YT_POOL.client(client_config.proxy_region, &.post(url, headers: headers, body: data.to_json)