fetch with innertube api when video is unavailable (#2329)

+ rename some client type to better names
+ fix thirdParty hack
This commit is contained in:
Émilien Devos 2021-08-16 19:41:16 +02:00 committed by GitHub
parent 25362f16a0
commit b5d2eb5c70
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 63 deletions

View file

@ -8,12 +8,12 @@ module YoutubeAPI
# Enumerate used to select one of the clients supported by the API # Enumerate used to select one of the clients supported by the API
enum ClientType enum ClientType
Web Web
WebEmbed WebEmbeddedPlayer
WebMobile WebMobile
WebAgeBypass WebScreenEmbed
Android Android
AndroidEmbed AndroidEmbeddedPlayer
AndroidAgeBypass AndroidScreenEmbed
end end
# List of hard-coded values used by the different clients # List of hard-coded values used by the different clients
@ -24,7 +24,7 @@ module YoutubeAPI
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
screen: "WATCH_FULL_SCREEN", screen: "WATCH_FULL_SCREEN",
}, },
ClientType::WebEmbed => { ClientType::WebEmbeddedPlayer => {
name: "WEB_EMBEDDED_PLAYER", # 56 name: "WEB_EMBEDDED_PLAYER", # 56
version: "1.20210721.1.0", version: "1.20210721.1.0",
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
@ -36,7 +36,7 @@ module YoutubeAPI
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
screen: "", # None screen: "", # None
}, },
ClientType::WebAgeBypass => { ClientType::WebScreenEmbed => {
name: "WEB", name: "WEB",
version: "2.20210721.00.00", version: "2.20210721.00.00",
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
@ -48,13 +48,13 @@ module YoutubeAPI
api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
screen: "", # ?? screen: "", # ??
}, },
ClientType::AndroidEmbed => { ClientType::AndroidEmbeddedPlayer => {
name: "ANDROID_EMBEDDED_PLAYER", # 55 name: "ANDROID_EMBEDDED_PLAYER", # 55
version: "16.20", version: "16.20",
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
screen: "", # None? screen: "", # None?
}, },
ClientType::AndroidAgeBypass => { ClientType::AndroidScreenEmbed => {
name: "ANDROID", # 3 name: "ANDROID", # 3
version: "16.20", version: "16.20",
api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
@ -156,9 +156,6 @@ module YoutubeAPI
"gl" => client_config.region || "US", # Can't be empty! "gl" => client_config.region || "US", # Can't be empty!
"clientName" => client_config.name, "clientName" => client_config.name,
"clientVersion" => client_config.version, "clientVersion" => client_config.version,
"thirdParty" => {
"embedUrl" => "", # Placeholder
},
}, },
} }
@ -167,14 +164,10 @@ module YoutubeAPI
client_context["client"]["clientScreen"] = client_config.screen client_context["client"]["clientScreen"] = client_config.screen
end end
# Replacing/removing the placeholder is easier than trying to
# merge two different Hash structures.
if client_config.screen == "EMBED" if client_config.screen == "EMBED"
client_context["client"]["thirdParty"] = { client_context["thirdParty"] = {
"embedUrl" => "https://www.youtube.com/embed/dQw4w9WgXcQ", "embedUrl" => "https://www.youtube.com/embed/dQw4w9WgXcQ",
} }
else
client_context["client"].delete("thirdParty")
end end
return client_context return client_context

View file

@ -819,10 +819,14 @@ def parse_related(r : JSON::Any) : JSON::Any?
JSON::Any.new(rv) JSON::Any.new(rv)
end end
def extract_video_info(video_id : String, proxy_region : String? = nil) def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil)
params = {} of String => JSON::Any params = {} of String => JSON::Any
client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
if context_screen == "embed"
client_config.client_type = YoutubeAPI::ClientType::WebScreenEmbed
end
player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
if player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK" if player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK"
@ -844,7 +848,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil)
# maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs:
# https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
if !params["reason"]? if !params["reason"]?
client_config.client_type = YoutubeAPI::ClientType::Android if context_screen == "embed"
client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
else
client_config.client_type = YoutubeAPI::ClientType::Android
end
stream_data = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) stream_data = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
params["streamingData"] = stream_data["streamingData"]? || JSON::Any.new("") params["streamingData"] = stream_data["streamingData"]? || JSON::Any.new("")
end end
@ -972,52 +980,10 @@ def fetch_video(id, region)
end end
end end
# Try to pull streams from embed URL # Try to fetch video info using an embedded client
if info["reason"]? if info["reason"]?
required_parameters = { embed_info = extract_video_info(video_id: id, context_screen: "embed")
"video_id" => id, info = embed_info if !embed_info["reason"]?
"eurl" => "https://youtube.googleapis.com/v/#{id}",
"html5" => "1",
"gl" => "US",
"hl" => "en",
}
if info["reason"].as_s.includes?("inappropriate")
# The html5, c and cver parameters are required in order to extract age-restricted videos
# See https://github.com/yt-dlp/yt-dlp/commit/4e6767b5f2e2523ebd3dd1240584ead53e8c8905
required_parameters.merge!({
"c" => "TVHTML5",
"cver" => "6.20180913",
})
# In order to actually extract video info without error, the `x-youtube-client-version`
# has to be set to the same version as `cver` above.
additional_headers = HTTP::Headers{"x-youtube-client-version" => "6.20180913"}
else
embed_page = YT_POOL.client &.get("/embed/#{id}").body
sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]? || ""
required_parameters["sts"] = sts
additional_headers = HTTP::Headers{} of String => String
end
embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{URI::Params.encode(required_parameters)}",
headers: additional_headers).body)
if embed_info["player_response"]?
player_response = JSON.parse(embed_info["player_response"])
{"captions", "microformat", "playabilityStatus", "streamingData", "videoDetails", "storyboards"}.each do |f|
info[f] = player_response[f] if player_response[f]?
end
end
initial_data = JSON.parse(embed_info["watch_next_response"]) if embed_info["watch_next_response"]?
info["relatedVideos"] = initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]?
.try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r|
parse_related r
}.try { |a| JSON::Any.new(a) } || embed_info["rvs"]?.try &.split(",").map { |r|
r = HTTP::Params.parse(r).to_h
JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) }))
}.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any)
end end
raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]?