From c658fd27cced47c438eb148f1a1aedf482be8f46 Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Fri, 2 Sep 2022 21:18:56 +0200 Subject: [PATCH] better spoof requests --- src/invidious/yt_backend/connection_pool.cr | 3 - src/invidious/yt_backend/youtube_api.cr | 103 +++++++++++++++----- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 23e98ae3..46e5bf85 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -13,9 +13,6 @@ def add_yt_headers(request) request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - return if request.resource.starts_with? "/sorry/index" - request.headers["x-youtube-client-name"] ||= "1" - request.headers["x-youtube-client-version"] ||= "2.20200609" # Preserve original cookies and add new YT consent cookie for EU servers request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+" if !CONFIG.cookies.empty? diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index c014dc0e..02327025 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -8,11 +8,16 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" private ANDROID_APP_VERSION = "17.33.42" - private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US)" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308 + private ANDROID_USER_AGENT = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US) gzip" private ANDROID_SDK_VERSION = 31_i64 private ANDROID_VERSION = "12" private IOS_APP_VERSION = "17.33.2" - private WINDOWS_VERSION = "10.0" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330 + private IOS_USER_AGENT = "com.google.ios.youtube/17.33.2 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)" + # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224 + private IOS_VERSION = "15.6.0.19G71" + private WINDOWS_VERSION = "10.0" # Enumerate used to select one of the clients supported by the API enum ClientType @@ -37,6 +42,7 @@ module YoutubeAPI HARDCODED_CLIENTS = { ClientType::Web => { name: "WEB", + name_proto: "1", version: "2.20220804.07.00", api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", @@ -45,7 +51,8 @@ module YoutubeAPI platform: "DESKTOP", }, ClientType::WebEmbeddedPlayer => { - name: "WEB_EMBEDDED_PLAYER", # 56 + name: "WEB_EMBEDDED_PLAYER", + name_proto: "56", version: "1.20220803.01.00", api_key: DEFAULT_API_KEY, screen: "EMBED", @@ -55,6 +62,7 @@ module YoutubeAPI }, ClientType::WebMobile => { name: "MWEB", + name_proto: "2", version: "2.20220805.01.00", api_key: DEFAULT_API_KEY, os_name: "Android", @@ -63,6 +71,7 @@ module YoutubeAPI }, ClientType::WebScreenEmbed => { name: "WEB", + name_proto: "1", version: "2.20220804.00.00", api_key: DEFAULT_API_KEY, screen: "EMBED", @@ -75,6 +84,7 @@ module YoutubeAPI ClientType::Android => { name: "ANDROID", + name_proto: "3", version: ANDROID_APP_VERSION, api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", android_sdk_version: ANDROID_SDK_VERSION, @@ -84,12 +94,14 @@ module YoutubeAPI platform: "MOBILE", }, ClientType::AndroidEmbeddedPlayer => { - name: "ANDROID_EMBEDDED_PLAYER", # 55 - version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, + name: "ANDROID_EMBEDDED_PLAYER", + name_proto: "55", + version: ANDROID_APP_VERSION, + api_key: DEFAULT_API_KEY, }, ClientType::AndroidScreenEmbed => { - name: "ANDROID", # 3 + name: "ANDROID", + name_proto: "3", version: ANDROID_APP_VERSION, api_key: DEFAULT_API_KEY, screen: "EMBED", @@ -103,33 +115,56 @@ module YoutubeAPI # IOS ClientType::IOS => { - name: "IOS", # 5 - version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", + name: "IOS", + name_proto: "5", + version: IOS_APP_VERSION, + api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, ClientType::IOSEmbedded => { - name: "IOS_MESSAGES_EXTENSION", # 66 - version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, + name: "IOS_MESSAGES_EXTENSION", + name_proto: "66", + version: IOS_APP_VERSION, + api_key: DEFAULT_API_KEY, + user_agent: IOS_USER_AGENT, + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, ClientType::IOSMusic => { - name: "IOS_MUSIC", # 26 - version: "4.32", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", + name: "IOS_MUSIC", + name_proto: "26", + version: "5.21", + api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", + user_agent: "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)", + device_make: "Apple", + device_model: "iPhone14,5", + os_name: "iPhone", + os_version: IOS_VERSION, + platform: "MOBILE", }, # TV app ClientType::TvHtml5 => { - name: "TVHTML5", # 7 - version: "7.20220325", - api_key: DEFAULT_API_KEY, + name: "TVHTML5", + name_proto: "7", + version: "7.20220325", + api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { - name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", # 85 - version: "2.0", - api_key: DEFAULT_API_KEY, - screen: "EMBED", + name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", + name_proto: "85", + version: "2.0", + api_key: DEFAULT_API_KEY, + screen: "EMBED", }, } @@ -183,6 +218,10 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:name] end + def name_proto : String + HARDCODED_CLIENTS[@client_type][:name_proto] + end + # :ditto: def version : String HARDCODED_CLIENTS[@client_type][:version] @@ -210,6 +249,14 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:os_name]? end + def device_make : String? + HARDCODED_CLIENTS[@client_type][:device_make]? + end + + def device_model : String? + HARDCODED_CLIENTS[@client_type][:device_model]? + end + def os_version : String? HARDCODED_CLIENTS[@client_type][:os_version]? end @@ -265,6 +312,14 @@ module YoutubeAPI client_context["client"]["androidSdkVersion"] = android_sdk_version end + if device_make = client_config.device_make + client_context["client"]["deviceMake"] = device_make + end + + if device_model = client_config.device_model + client_context["client"]["deviceModel"] = device_model + end + if os_name = client_config.os_name client_context["client"]["osName"] = os_name end @@ -524,6 +579,8 @@ module YoutubeAPI "Content-Type" => "application/json; charset=UTF-8", "Accept-Encoding" => "gzip, deflate", "x-goog-api-format-version" => "2", + "x-youtube-client-name" => client_config.name_proto, + "x-youtube-client-version" => client_config.version, } if user_agent = client_config.user_agent