iv-org-mirror/invidious-copy-2023-06-08
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious-copy-2023-06-08.git synced 2024-08-15 00:53:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge 865b7e584d into 545a5937d8

Browse source
This commit is contained in:
Samantaz Fox 2023-06-07 14:41:58 +02:00 • committed by GitHub
commit 83942bb8c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 185 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

8
assets/js/player.js
View file

@ -43,6 +43,14 @@ embed_url = location.origin + '/embed/' + video_data.id + embed_url.search;
var save_player_pos_key = 'save_player_pos';
videojs.Vhs.xhr.beforeRequest = function(options) {
// Pass range as an URL parameter, not as a header
if (options.headers){
if (options.headers.Range) {
options.uri = options.uri + '&range=' + options.headers.Range.split("=")[1];
delete options.headers.Range;
}
}
// set local if requested not videoplayback
if (!options.uri.includes('videoplayback')) {
if (!options.uri.includes('local=true'))

4
src/invidious.cr
View file

@ -73,9 +73,7 @@ CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz012345
TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
MAX_ITEMS_PER_PAGE = 1500
REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
HTTP_CHUNK_SIZE = 10485760 # ~10MB
HTTP_CHUNK_SIZE = 10485760 # ~10MB
CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}

14
src/invidious/config.cr
View file

@ -126,8 +126,10 @@ class Config
property host_binding : String = "0.0.0.0"
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100
# Use quic transport for youtube api
property use_quic : Bool = false
# Note: The getter function is defined below to take into account the compile-time flag
setter use_quic : Bool = false
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
@ -155,6 +157,16 @@ class Config
end
end
# Return whether QUIC is enabled or not.
# Takes into account compile-time flag AND runtime config.
def use_quic : Bool
{% if flag?(:disable_quic) %}
return false
{% else %}
return @use_quic
{% end %}
end
def self.load
# Load config from file or YAML string env var
env_config_file = "INVIDIOUS_CONFIG_FILE"

188
src/invidious/routes/images.cr
View file

@ -3,34 +3,17 @@ module Invidious::Routes::Images
def self.ggpht(env)
url = env.request.path.lchop("/ggpht")
headers = (
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
HTTP::Headers{":authority" => "yt3.ggpht.com"}
else
HTTP::Headers.new
end
{% else %}
HTTP::Headers.new
{% end %}
)
headers = HTTP::Headers.new
headers[":authority"] = "yt3.ggpht.com" if CONFIG.use_quic
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
# We're encapsulating this into a proc in order to easily reuse this
# portion of the code for each request block below.
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
@ -42,22 +25,16 @@ module Invidious::Routes::Images
}
begin
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
else
HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
return request_proc.call(resp)
end
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
{% else %}
else
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
end
rescue ex
end
end
@ -77,25 +54,14 @@ module Invidious::Routes::Images
url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}"
headers = HTTP::Headers.new
headers[":authority"] = "#{authority}.ytimg.com" if CONFIG.use_quic
{% unless flag?(:disable_quic) %}
headers[":authority"] = "#{authority}.ytimg.com"
{% end %}
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
env.response.headers["Connection"] = "close"
env.response.headers["Access-Control-Allow-Origin"] = "*"
@ -107,22 +73,16 @@ module Invidious::Routes::Images
}
begin
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
else
HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
{% else %}
else
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
end
rescue ex
end
end
@ -133,32 +93,15 @@ module Invidious::Routes::Images
name = env.params.url["name"]
url = env.request.resource
headers = (
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
HTTP::Headers{":authority" => "i9.ytimg.com"}
else
HTTP::Headers.new
end
{% else %}
HTTP::Headers.new
{% end %}
)
headers = HTTP::Headers.new
headers[":authority"] = "i9.ytimg.com" if CONFIG.use_quic
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
@ -169,43 +112,29 @@ module Invidious::Routes::Images
}
begin
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
else
HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
{% else %}
else
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
end
rescue ex
end
end
def self.yts_image(env)
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
begin
YT_POOL.client &.get(env.request.resource, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
@ -223,60 +152,37 @@ module Invidious::Routes::Images
id = env.params.url["id"]
name = env.params.url["name"]
headers = (
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
HTTP::Headers{":authority" => "i.ytimg.com"}
else
HTTP::Headers.new
end
{% else %}
HTTP::Headers.new
{% end %}
)
headers = HTTP::Headers.new
headers[":authority"] = "i.ytimg.com" if CONFIG.use_quic
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
# Logic here is short enough that manually typing them out should be fine.
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
name = thumb[:url] + ".jpg"
break
end
else
if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
name = thumb[:url] + ".jpg"
break
end
if CONFIG.use_quic
if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
name = thumb[:url] + ".jpg"
break
end
{% else %}
else
# This can likely be optimized into a (small) pool sometime in the future.
if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
name = thumb[:url] + ".jpg"
break
end
{% end %}
end
end
end
url = "/vi/#{id}/#{name}"
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
MediaProxy.copy_response_headers(from: response.headers, to: env.response.headers)
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
@ -287,22 +193,16 @@ module Invidious::Routes::Images
}
begin
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
else
HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
if CONFIG.use_quic
YT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp)
end
{% else %}
else
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
end
rescue ex
end
end

95
src/invidious/routes/video_playback.cr
View file

@ -8,13 +8,17 @@ module Invidious::Routes::VideoPlayback
mns = query_params["mn"]?.try &.split(",")
mns ||= [] of String
if query_params["region"]?
region = query_params["region"]
# Extract some invidious-specific parameters
if region = query_params["region"]?
query_params.delete("region")
end
if query_params["host"]? && !query_params["host"].empty?
host = query_params["host"]
if title = query_params["title"]?
query_params.delete("title")
end
if host = query_params["host"]?
query_params.delete("host")
else
host = "r#{fvip}---#{mns.pop}.googlevideo.com"
@ -25,22 +29,31 @@ module Invidious::Routes::VideoPlayback
return error_template(400, "Invalid \"host\" parameter.")
end
# Range manipulation
has_range_param = false
has_range_header = false
if range = query_params["range"]?
query_params.delete("range")
has_range_param = true
end
if range_header = env.request.headers["Range"]?
env.request.headers.delete("Range")
range ||= range_header.split('=')[1]
has_range_header = true if !has_range_param
end
# Skip redirections
host = "https://#{host}"
url = "/videoplayback?#{query_params}"
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
MediaProxy.copy_request_headers(from: env.request.headers, to: headers)
# See: https://github.com/iv-org/invidious/issues/3302
range_header = env.request.headers["Range"]?
if range_header.nil?
range_for_head = query_params["range"]? || "0-640"
headers["Range"] = "bytes=#{range_for_head}"
end
headers["Range"] = "bytes=" + (range || "0-")
client = make_client(URI.parse(host), region)
response = HTTP::Client::Response.new(500)
@ -78,7 +91,7 @@ module Invidious::Routes::VideoPlayback
end
# Remove the Range header added previously.
headers.delete("Range") if range_header.nil?
headers.delete("Range")
if response.status_code >= 400
env.response.content_type = "text/plain"
@ -90,34 +103,24 @@ module Invidious::Routes::VideoPlayback
return error_template(403, "Administrator has disabled this endpoint.")
end
begin
client.get(url, headers) do |resp|
resp.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
# Add back the range header. Important!
headers["Range"] = "bytes=" + (range || "0-")
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
return env.redirect url
end
IO.copy(resp.body_io, env.response)
end
rescue ex
MediaProxy.proxy_dash_chunk(env, client, url, region)
elsif has_range_param
if CONFIG.disabled?("dash")
return error_template(403, "Administrator has disabled this endpoint.")
end
MediaProxy.proxy_dash_chunk(env, client, url + "&range=#{range}", region)
else
if query_params["title"]? && CONFIG.disabled?("downloads") ||
CONFIG.disabled?("dash")
if (title && CONFIG.disabled?("downloads")) || (title.nil? && CONFIG.disabled?("local"))
return error_template(403, "Administrator has disabled this endpoint.")
end
content_length = nil
first_chunk = true
range_start, range_end = parse_range(env.request.headers["Range"]?)
range_start, range_end = parse_range(range)
chunk_start = range_start
chunk_end = range_end
@ -144,20 +147,12 @@ module Invidious::Routes::VideoPlayback
begin
client.get(url, headers) do |resp|
if first_chunk
if !env.request.headers["Range"]? && resp.status_code == 206
if !has_range_header && resp.status_code == 206
env.response.status_code = 200
else
env.response.status_code = resp.status_code
end
resp.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range"
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
@ -166,8 +161,12 @@ module Invidious::Routes::VideoPlayback
break
end
if title = query_params["title"]?
# https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
MediaProxy.copy_response_headers(from: resp.headers, to: env.response.headers)
env.response.headers.delete("Content-Range") # Important!
env.response.headers["Access-Control-Allow-Origin"] = "*"
if title
# http://web.archive.org/web/20150701003254/https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
filename = URI.encode_www_form(title, space_to_plus: false)
header = "attachment; filename=\"#{filename}\"; filename*=UTF-8''#{filename}"
env.response.headers["Content-Disposition"] = header
@ -175,7 +174,7 @@ module Invidious::Routes::VideoPlayback
if !resp.headers.includes_word?("Transfer-Encoding", "chunked")
content_length = resp.headers["Content-Range"].split("/")[-1].to_i64
if env.request.headers["Range"]?
if has_range_header
env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}"
env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start
else

6
src/invidious/yt_backend/connection_pool.cr
View file

@ -63,11 +63,7 @@ struct YoutubeConnectionPool
DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
conn = nil # Declare
{% unless flag?(:disable_quic) %}
if use_quic
conn = QUIC::Client.new(url)
else
conn = HTTP::Client.new(url)
end
conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url)
{% else %}
conn = HTTP::Client.new(url)
{% end %}

70
src/invidious/yt_backend/media_proxy.cr Normal file
View file

@ -0,0 +1,70 @@
module Invidious::MediaProxy
extend self
# -------------------
# Constants
# -------------------
private REQUEST_HEADERS_WHITELIST = {
"accept", "accept-encoding", "cache-control",
"content-length", "if-none-match", "range",
}
private RESPONSE_HEADERS_BLACKLIST = {
"access-control-allow-origin", "alt-svc", "server",
}
# -------------------
# Headers functions
# -------------------
# Copy only the selected headers from the client to youtube servers
# (in general, from `env.request` to a temporary `HTTP::Headers` object).
def copy_request_headers(*, from : HTTP::Headers, to : HTTP::Headers)
REQUEST_HEADERS_WHITELIST.each do |header|
to[header] = from[header] if from[header]?
end
end
# Copy only the selected headers from youtube servers to the client
# (generally, from a response block to `env.response`).
def copy_response_headers(*, from : HTTP::Headers, to : HTTP::Headers)
from.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
to[key] = value
end
end
end
# -------------------
# Proxy functions
# -------------------
def proxy_dash_chunk(
env : HTTP::Server::Context,
client : HTTP::Client,
url : URI | String,
region : String?
)
headers = HTTP::Headers.new
self.copy_request_headers(from: env.request.headers, to: headers)
# Make sure to remove a potential range header, to avoid throttling
headers.delete("Range")
client.get(url, headers) do |resp|
env.response.status_code = resp.status_code
self.copy_response_headers(from: resp.headers, to: env.response.headers)
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
url = HttpServer::Utils.proxy_video_url(location, region: region)
env.redirect url
else
IO.copy(resp.body_io, env.response)
end
end
rescue ex
end
end

2
src/invidious/yt_backend/youtube_api.cr
View file

@ -593,7 +593,7 @@ module YoutubeAPI
LOGGER.trace("YoutubeAPI: POST data: #{data}")
# Send the POST request
if {{ !flag?(:disable_quic) }} && CONFIG.use_quic
if CONFIG.use_quic
# Using QUIC client
body = YT_POOL.client(client_config.proxy_region,
&.post(url, headers: headers, body: data.to_json)