iv-org-mirror/invidious-copy-2022-08-14
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious-copy-2022-08-14.git synced 2024-08-15 00:53:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add HTTPClient pool

Browse source
This commit is contained in:
Omar Roth 2019-10-25 12:58:16 -04:00
parent aba2c5b938
commit 6930570fa2
No known key found for this signature in database
GPG key ID: B8254FB7EC3D37F2
14 changed files with 115 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

69
src/invidious.cr
View file

@ -46,6 +46,7 @@ PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
REDDIT_URL = URI.parse("https://www.reddit.com")
TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com")
YT_URL = URI.parse("https://www.youtube.com")
YT_IMG_URL = URI.parse("https://i.ytimg.com")
CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@ -91,6 +92,9 @@ LOCALES = {
"zh-TW" => load_locale("zh-TW"),
}
YT_POOL = HTTPPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
YT_IMG_POOL = HTTPPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05)
config = CONFIG
logger = Invidious::LogHandler.new
@ -658,8 +662,7 @@ get "/embed/:id" do |env|
next env.redirect url
when "live_stream"
client = make_client(YT_URL)
response = client.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
env.params.query.delete_all("channel")
@ -2250,8 +2253,7 @@ get "/modify_notifications" do |env|
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]
client = make_client(YT_URL)
html = client.get("/subscription_manager?disable_polymer=1", headers)
html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
cookies = HTTP::Cookies.from_headers(headers)
html.cookies.each do |cookie|
@ -2280,7 +2282,7 @@ get "/modify_notifications" do |env|
channel_id = channel.content.lstrip("/channel/").not_nil!
channel_req["channel_id"] = channel_id
client.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
end
end
@ -2558,8 +2560,7 @@ post "/data_control" do |env|
if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
next match["channel"]
elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
client = make_client(YT_URL)
response = client.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
document = XML.parse_html(response.body)
canonical = document.xpath_node(%q(//link[@rel="canonical"]))
@ -3084,8 +3085,7 @@ get "/feed/channel/:ucid" do |env|
next error_message
end
client = make_client(YT_URL)
rss = client.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body
rss = XML.parse_html(rss)
videos = [] of SearchVideo
@ -3229,8 +3229,7 @@ get "/feed/playlist/:plid" do |env|
end
end
client = make_client(YT_URL)
response = client.get("/feeds/videos.xml?playlist_id=#{plid}")
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
document = XML.parse(response.body)
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
@ -3395,14 +3394,13 @@ end
{"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route|
get route do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
client = make_client(YT_URL)
# Appears to be a bug in routing, having several routes configured
# as `/a/:a`, `/b/:a`, `/c/:a` results in 404
value = env.request.resource.split("/")[2]
body = ""
{"channel", "user", "c"}.each do |type|
response = client.get("/#{type}/#{value}/live?disable_polymer=1")
response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
if response.status_code == 200
body = response.body
end
@ -3433,10 +3431,9 @@ end
get "/c/:user" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
client = make_client(YT_URL)
user = env.params.url["user"]
response = client.get("/c/#{user}")
response = YT_POOL.client &.get("/c/#{user}")
document = XML.parse_html(response.body)
anchor = document.xpath_node(%q(//a[contains(@class,"branded-page-header-title-link")]))
@ -3676,7 +3673,6 @@ get "/api/v1/storyboards/:id" do |env|
id = env.params.url["id"]
region = env.params.query["region"]?
client = make_client(YT_URL)
begin
video = get_video(id, PG_DB, region: region)
rescue ex : VideoRedirect
@ -3764,7 +3760,6 @@ get "/api/v1/captions/:id" do |env|
# In future this should be investigated as an alternative, since it does not require
# getting video info.
client = make_client(YT_URL)
begin
video = get_video(id, PG_DB, region: region)
rescue ex : VideoRedirect
@ -3823,7 +3818,7 @@ get "/api/v1/captions/:id" do |env|
# Auto-generated captions often have cues that aren't aligned properly with the video,
# as well as some other markup that makes it cumbersome, so we try to fix that here
if caption.name.simpleText.includes? "auto-generated"
caption_xml = client.get(url).body
caption_xml = YT_POOL.client &.get(url).body
caption_xml = XML.parse(caption_xml)
webvtt = String.build do |str|
@ -3866,7 +3861,7 @@ get "/api/v1/captions/:id" do |env|
end
end
else
webvtt = client.get("#{url}&format=vtt").body
webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
end
if title = env.params.query["title"]?
@ -4013,9 +4008,7 @@ get "/api/v1/annotations/:id" do |env|
cache_annotation(PG_DB, id, annotations)
end
when "youtube"
client = make_client(YT_URL)
response = client.get("/annotations_invideo?video_id=#{id}")
response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
if response.status_code != 200
env.response.status_code = response.status_code
@ -5115,7 +5108,6 @@ get "/api/manifest/dash/id/:id" do |env|
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]? && (env.params.query["unique_res"] == "true" || env.params.query["unique_res"] == "1")
client = make_client(YT_URL)
begin
video = get_video(id, PG_DB, region: region)
rescue ex : VideoRedirect
@ -5126,7 +5118,7 @@ get "/api/manifest/dash/id/:id" do |env|
end
if dashmpd = video.player_response["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
manifest = client.get(dashmpd).body
manifest = YT_POOL.client &.get(dashmpd).body
manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
url = baseurl.lchop("<BaseURL>")
@ -5226,8 +5218,7 @@ get "/api/manifest/dash/id/:id" do |env|
end
get "/api/manifest/hls_variant/*" do |env|
client = make_client(YT_URL)
manifest = client.get(env.request.path)
manifest = YT_POOL.client &.get(env.request.path)
if manifest.status_code != 200
env.response.status_code = manifest.status_code
@ -5252,8 +5243,7 @@ get "/api/manifest/hls_variant/*" do |env|
end
get "/api/manifest/hls_playlist/*" do |env|
client = make_client(YT_URL)
manifest = client.get(env.request.path)
manifest = YT_POOL.client &.get(env.request.path)
if manifest.status_code != 200
env.response.status_code = manifest.status_code
@ -5616,10 +5606,6 @@ get "/videoplayback" do |env|
end
end
# We need this so the below route works as expected
get "/ggpht*" do |env|
end
get "/ggpht/*" do |env|
host = "https://yt3.ggpht.com"
client = make_client(URI.parse(host))
@ -5745,12 +5731,9 @@ get "/vi/:id/:name" do |env|
id = env.params.url["id"]
name = env.params.url["name"]
host = "https://i.ytimg.com"
client = make_client(URI.parse(host))
if name == "maxres.jpg"
build_thumbnails(id, config, Kemal.config).each do |thumb|
if client.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200
if YT_IMG_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200
name = thumb[:url] + ".jpg"
break
end
@ -5766,7 +5749,7 @@ get "/vi/:id/:name" do |env|
end
begin
client.get(url, headers) do |response|
YT_IMG_POOL.client &.get(url, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes? key
@ -5789,9 +5772,7 @@ end
# Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
get "/watch_videos" do |env|
client = make_client(YT_URL)
response = client.get("#{env.request.path}?#{env.request.query}")
response = YT_POOL.client &.get(env.request.resource)
if url = response.headers["Location"]?
url = URI.parse(url).full_path
next env.redirect url
@ -5805,11 +5786,10 @@ error 404 do |env|
item = md["id"]
# Check if item is branding URL e.g. https://youtube.com/gaming
client = make_client(YT_URL)
response = client.get("/#{item}")
response = YT_POOL.client &.get("/#{item}")
if response.status_code == 301
response = client.get(response.headers["Location"])
response = YT_POOL.client &.get(response.headers["Location"])
end
if response.body.empty?
@ -5837,8 +5817,7 @@ error 404 do |env|
end
# Check if item is video ID
client = make_client(YT_URL)
if item.match(/^[a-zA-Z0-9_-]{11}$/) && client.head("/watch?v=#{item}").status_code != 404
if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
env.response.headers["Location"] = url
halt env, status_code: 302
end