From aa55e67389ac8c3786c5e0ac4dca3d50f2193e16 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Jun 2021 07:51:51 -0700 Subject: [PATCH 1/6] Fix extraction of age restricted videos --- src/invidious/videos.cr | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 116aafc7..5b2158ec 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -989,9 +989,15 @@ def fetch_video(id, region) # Try to pull streams from embed URL if info["reason"]? - embed_page = YT_POOL.client &.get("/embed/#{id}").body - sts = embed_page.match(/"sts"\s*:\s*(?\d+)/).try &.["sts"]? || "" - embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?html5=1&video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&sts=#{sts}").body) + required_parameters = URI::Params.new({ + "video_id" => [id], + "eurl" => ["https://youtube.googleapis.com/v/#{id}"], + "html5" => ["1"], + "c" => ["TVHTML5"], + "cver" => ["6.20180913"], + }) + + embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{required_parameters}", headers: HTTP::Headers{"x-youtube-client-version" => "6.20180913"}).body) if embed_info["player_response"]? player_response = JSON.parse(embed_info["player_response"]) From 7da0b2fd7fb977eb9e5f790e63e93b9bde0c9768 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Jun 2021 12:14:21 -0700 Subject: [PATCH 2/6] Switch from URI::Params.new to URI::Params.encode --- src/invidious/videos.cr | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 5b2158ec..264a70e8 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -989,15 +989,22 @@ def fetch_video(id, region) # Try to pull streams from embed URL if info["reason"]? - required_parameters = URI::Params.new({ - "video_id" => [id], - "eurl" => ["https://youtube.googleapis.com/v/#{id}"], - "html5" => ["1"], - "c" => ["TVHTML5"], - "cver" => ["6.20180913"], + # The html5, c and cver parameters are required in order to extract age-restricted videos + # See https://github.com/yt-dlp/yt-dlp/commit/4e6767b5f2e2523ebd3dd1240584ead53e8c8905 + required_parameters = URI::Params.encode({ + "video_id" => id, + "eurl" => "https://youtube.googleapis.com/v/#{id}", + "html5" => "1", + "c" => "TVHTML5", + "cver" => "6.20180913", }) - embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{required_parameters}", headers: HTTP::Headers{"x-youtube-client-version" => "6.20180913"}).body) + # In order to actually extract video info without error, the `x-youtube-client-version` has to be set to the same version as `cver` above. + embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{required_parameters}", + headers: HTTP::Headers{ + "x-youtube-client-version" => "6.20180913", + }).body + ) if embed_info["player_response"]? player_response = JSON.parse(embed_info["player_response"]) From ca4df2967049ca8557506706e384d7ceab3f67a8 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Jun 2021 14:14:41 -0700 Subject: [PATCH 3/6] Wrap comment --- src/invidious/videos.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 264a70e8..3e64537e 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -999,7 +999,8 @@ def fetch_video(id, region) "cver" => "6.20180913", }) - # In order to actually extract video info without error, the `x-youtube-client-version` has to be set to the same version as `cver` above. + # In order to actually extract video info without error, the `x-youtube-client-version` + # has to be set to the same version as `cver` above. embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{required_parameters}", headers: HTTP::Headers{ "x-youtube-client-version" => "6.20180913", From 54b19a04bb11292634d5275ee25622f048212330 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 27 Jun 2021 07:18:16 -0700 Subject: [PATCH 4/6] Fix caption parsing on age restricted videos --- src/invidious.cr | 8 +++--- src/invidious/routes/embed.cr | 4 +-- src/invidious/routes/watch.cr | 4 +-- src/invidious/videos.cr | 31 +++++++++++++---------- src/invidious/views/components/player.ecr | 8 +++--- src/invidious/views/watch.ecr | 4 +-- 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index f7c8980a..57809c0b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1961,9 +1961,9 @@ get "/api/v1/captions/:id" do |env| json.array do captions.each do |caption| json.object do - json.field "label", caption.name.simpleText + json.field "label", caption.name json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" end end end @@ -1979,7 +1979,7 @@ get "/api/v1/captions/:id" do |env| if lang caption = captions.select { |caption| caption.languageCode == lang } else - caption = captions.select { |caption| caption.name.simpleText == label } + caption = captions.select { |caption| caption.name == label } end if caption.empty? @@ -1993,7 +1993,7 @@ get "/api/v1/captions/:id" do |env| # Auto-generated captions often have cues that aren't aligned properly with the video, # as well as some other markup that makes it cumbersome, so we try to fix that here - if caption.name.simpleText.includes? "auto-generated" + if caption.name.includes? "auto-generated" caption_xml = YT_POOL.client &.get(url).body caption_xml = XML.parse(caption_xml) diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 5db32788..5e1e9431 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -165,11 +165,11 @@ class Invidious::Routes::Embed < Invidious::Routes::BaseRoute captions = video.captions preferred_captions = captions.select { |caption| - params.preferred_captions.includes?(caption.name.simpleText) || + params.preferred_captions.includes?(caption.name) || params.preferred_captions.includes?(caption.languageCode.split("-")[0]) } preferred_captions.sort_by! { |caption| - (params.preferred_captions.index(caption.name.simpleText) || + (params.preferred_captions.index(caption.name) || params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil! } captions = captions - preferred_captions diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index d0338882..c6c7c154 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -150,11 +150,11 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute captions = video.captions preferred_captions = captions.select { |caption| - params.preferred_captions.includes?(caption.name.simpleText) || + params.preferred_captions.includes?(caption.name) || params.preferred_captions.includes?(caption.languageCode.split("-")[0]) } preferred_captions.sort_by! { |caption| - (params.preferred_captions.index(caption.name.simpleText) || + (params.preferred_captions.index(caption.name) || params.preferred_captions.index(caption.languageCode.split("-")[0])).not_nil! } captions = captions - preferred_captions diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 3e64537e..1c9f5d03 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -425,9 +425,9 @@ struct Video json.array do self.captions.each do |caption| json.object do - json.field "label", caption.name.simpleText + json.field "label", caption.name json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" end end end @@ -706,8 +706,12 @@ struct Video def captions : Array(Caption) return @captions.as(Array(Caption)) if @captions captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| - caption = Caption.from_json(caption.to_json) - caption.name.simpleText = caption.name.simpleText.split(" - ")[0] + name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] + languageCode = caption["languageCode"].to_s + baseUrl = caption["baseUrl"].to_s + + caption = Caption.new(name.to_s, languageCode, baseUrl) + caption.name = caption.name.split(" - ")[0] caption end captions ||= [] of Caption @@ -782,18 +786,19 @@ struct Video end end -struct CaptionName - include JSON::Serializable +class Caption + property name + property languageCode + property baseUrl - property simpleText : String -end + getter name : String + getter languageCode : String + getter baseUrl : String -struct Caption - include JSON::Serializable + setter name - property name : CaptionName - property baseUrl : String - property languageCode : String + def initialize(@name, @languageCode, @baseUrl) + end end class VideoRedirect < Exception diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index cff3e60a..c37d20d5 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -25,13 +25,13 @@ <% end %> <% preferred_captions.each do |caption| %> - " - label="<%= caption.name.simpleText %>"> + " + label="<%= caption.name %>"> <% end %> <% captions.each do |caption| %> - " - label="<%= caption.name.simpleText %>"> + " + label="<%= caption.name %>"> <% end %> <% end %> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 91e03725..f21fdb04 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -178,8 +178,8 @@ we're going to need to do it here in order to allow for translations. <% end %> <% captions.each do |caption| %> - <% end %> From 57bb8c610a5489a7159a29004fdd56ec8bff50e5 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 1 Jul 2021 23:55:29 -0700 Subject: [PATCH 5/6] Use embed stream pull as fallback for gated videos --- src/invidious/videos.cr | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 1c9f5d03..77fa6ecb 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -994,23 +994,33 @@ def fetch_video(id, region) # Try to pull streams from embed URL if info["reason"]? - # The html5, c and cver parameters are required in order to extract age-restricted videos - # See https://github.com/yt-dlp/yt-dlp/commit/4e6767b5f2e2523ebd3dd1240584ead53e8c8905 - required_parameters = URI::Params.encode({ + required_parameters = { "video_id" => id, "eurl" => "https://youtube.googleapis.com/v/#{id}", "html5" => "1", - "c" => "TVHTML5", - "cver" => "6.20180913", - }) + "gl" => "US", + "hl" => "en", + } + if info["reason"].as_s.includes?("inappropriate") + # The html5, c and cver parameters are required in order to extract age-restricted videos + # See https://github.com/yt-dlp/yt-dlp/commit/4e6767b5f2e2523ebd3dd1240584ead53e8c8905 + required_parameters.merge!({ + "c" => "TVHTML5", + "cver" => "6.20180913", + }) - # In order to actually extract video info without error, the `x-youtube-client-version` - # has to be set to the same version as `cver` above. - embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{required_parameters}", - headers: HTTP::Headers{ - "x-youtube-client-version" => "6.20180913", - }).body - ) + # In order to actually extract video info without error, the `x-youtube-client-version` + # has to be set to the same version as `cver` above. + additional_headers = HTTP::Headers{"x-youtube-client-version" => "6.20180913"} + else + embed_page = YT_POOL.client &.get("/embed/#{id}").body + sts = embed_page.match(/"sts"\s*:\s*(?\d+)/).try &.["sts"]? || "" + required_parameters["sts"] = sts + additional_headers = HTTP::Headers{} of String => String + end + + embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{URI::Params.encode(required_parameters)}", + headers: additional_headers).body) if embed_info["player_response"]? player_response = JSON.parse(embed_info["player_response"]) From 39110ad21c2a7a17138d2472942bf52f4540c9f4 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 11 Jul 2021 16:17:22 -0700 Subject: [PATCH 6/6] Use struct for caption object --- src/invidious/videos.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 77fa6ecb..27c54b14 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -786,7 +786,7 @@ struct Video end end -class Caption +struct Caption property name property languageCode property baseUrl