From 31244cbcc89fa816e38afad1b4962fbe46497326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos?= Date: Tue, 30 Aug 2022 14:20:08 +0000 Subject: [PATCH] replicate headers and params made by yt apps --- src/invidious/yt_backend/connection_pool.cr | 14 +-- src/invidious/yt_backend/youtube_api.cr | 110 ++++++++++++++++---- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 3feb9233..23e98ae3 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -7,17 +7,19 @@ {% end %} def add_yt_headers(request) - request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" - request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["accept-language"] ||= "en-us,en;q=0.5" + if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" + end + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" return if request.resource.starts_with? "/sorry/index" request.headers["x-youtube-client-name"] ||= "1" request.headers["x-youtube-client-version"] ||= "2.20200609" # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" if !CONFIG.cookies.empty? - request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" end end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 30d7613b..c014dc0e 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -7,9 +7,12 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_APP_VERSION = "17.29.35" - private ANDROID_SDK_VERSION = 30_i64 - private IOS_APP_VERSION = "17.30.1" + private ANDROID_APP_VERSION = "17.33.42" + private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US)" + private ANDROID_SDK_VERSION = 31_i64 + private ANDROID_VERSION = "12" + private IOS_APP_VERSION = "17.33.2" + private WINDOWS_VERSION = "10.0" # Enumerate used to select one of the clients supported by the API enum ClientType @@ -33,27 +36,39 @@ module YoutubeAPI # List of hard-coded values used by the different clients HARDCODED_CLIENTS = { ClientType::Web => { - name: "WEB", - version: "2.20220804.07.00", - api_key: DEFAULT_API_KEY, - screen: "WATCH_FULL_SCREEN", + name: "WEB", + version: "2.20220804.07.00", + api_key: DEFAULT_API_KEY, + screen: "WATCH_FULL_SCREEN", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, ClientType::WebEmbeddedPlayer => { - name: "WEB_EMBEDDED_PLAYER", # 56 - version: "1.20220803.01.00", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "WEB_EMBEDDED_PLAYER", # 56 + version: "1.20220803.01.00", + api_key: DEFAULT_API_KEY, + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, ClientType::WebMobile => { - name: "MWEB", - version: "2.20220805.01.00", - api_key: DEFAULT_API_KEY, + name: "MWEB", + version: "2.20220805.01.00", + api_key: DEFAULT_API_KEY, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, ClientType::WebScreenEmbed => { - name: "WEB", - version: "2.20220804.00.00", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "WEB", + version: "2.20220804.00.00", + api_key: DEFAULT_API_KEY, + screen: "EMBED", + os_name: "Windows", + os_version: WINDOWS_VERSION, + platform: "DESKTOP", }, # Android @@ -63,6 +78,10 @@ module YoutubeAPI version: ANDROID_APP_VERSION, api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, ClientType::AndroidEmbeddedPlayer => { name: "ANDROID_EMBEDDED_PLAYER", # 55 @@ -75,6 +94,10 @@ module YoutubeAPI api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, # IOS @@ -179,6 +202,22 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:android_sdk_version]? end + def user_agent : String? + HARDCODED_CLIENTS[@client_type][:user_agent]? + end + + def os_name : String? + HARDCODED_CLIENTS[@client_type][:os_name]? + end + + def os_version : String? + HARDCODED_CLIENTS[@client_type][:os_version]? + end + + def platform : String? + HARDCODED_CLIENTS[@client_type][:platform]? + end + # Convert to string, for logging purposes def to_s return { @@ -226,6 +265,18 @@ module YoutubeAPI client_context["client"]["androidSdkVersion"] = android_sdk_version end + if os_name = client_config.os_name + client_context["client"]["osName"] = os_name + end + + if os_version = client_config.os_version + client_context["client"]["osVersion"] = os_version + end + + if platform = client_config.platform + client_context["client"]["platform"] = platform + end + return client_context end @@ -361,8 +412,18 @@ module YoutubeAPI ) # JSON Request data, required by the API data = { - "videoId" => video_id, - "context" => self.make_context(client_config), + "contentCheckOk" => true, + "videoId" => video_id, + "context" => self.make_context(client_config), + "racyCheckOk" => true, + "user" => { + "lockedSafetyMode" => false, + }, + "playbackContext" => { + "contentPlaybackContext" => { + "html5Preference": "HTML5_PREF_WANTS", + }, + }, } # Append the additional parameters if those were provided @@ -460,10 +521,15 @@ module YoutubeAPI url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip, deflate", + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip, deflate", + "x-goog-api-format-version" => "2", } + if user_agent = client_config.user_agent + headers["User-Agent"] = user_agent + end + # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")