From f345f667e2bd7306a1c54df1c7cf26d26707067a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 26 Jul 2020 12:00:56 +0200 Subject: [PATCH] Extract YouTube's key --- .../youtube/YoutubeParsingHelper.java | 80 ++++++++++++------- .../extractors/YoutubeSearchExtractor.java | 8 +- 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index b329be28..f9d896d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -64,6 +64,8 @@ public class YoutubeParsingHelper { private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; + private static String key; + private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"}; private static String[] youtubeMusicKeys; @@ -214,39 +216,31 @@ public class YoutubeParsingHelper { return response.length() > 50; // ensure to have a valid response } - /** - * Get the client version from a page - * @return - * @throws ParsingException - */ - public static String getClientVersion() throws IOException, ExtractionException { - if (!isNullOrEmpty(clientVersion)) return clientVersion; - if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION; - + private static void getClientVersionAndKey() throws IOException, ExtractionException { final String url = "https://www.youtube.com/results?search_query=test"; final String html = getDownloader().get(url).responseBody(); - JsonObject initialData = getInitialData(html); - JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + final JsonObject initialData = getInitialData(html); + final JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); String shortClientVersion = null; // try to get version from initial data first - for (Object service : serviceTrackingParams) { - JsonObject s = (JsonObject) service; + for (final Object service : serviceTrackingParams) { + final JsonObject s = (JsonObject) service; if (s.getString("service").equals("CSI")) { - JsonArray params = s.getArray("params"); - for (Object param : params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); + final JsonArray params = s.getArray("params"); + for (final Object param : params) { + final JsonObject p = (JsonObject) param; + final String key = p.getString("key"); if (key != null && key.equals("cver")) { - return clientVersion = p.getString("value"); + clientVersion = p.getString("value"); } } } else if (s.getString("service").equals("ECATCHER")) { // fallback to get a shortened client version which does not contain the last two digits - JsonArray params = s.getArray("params"); - for (Object param : params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); + final JsonArray params = s.getArray("params"); + for (final Object param : params) { + final JsonObject p = (JsonObject) param; + final String key = p.getString("key"); if (key != null && key.equals("client.version")) { shortClientVersion = p.getString("value"); } @@ -255,26 +249,54 @@ public class YoutubeParsingHelper { } String contextClientVersion; - String[] patterns = { + final String[] patterns = { "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", "innertube_context_client_version\":\"([0-9\\.]+?)\"", "client.version=([0-9\\.]+)" }; - for (String pattern : patterns) { + for (final String pattern : patterns) { try { contextClientVersion = Parser.matchGroup1(pattern, html); if (!isNullOrEmpty(contextClientVersion)) { - return clientVersion = contextClientVersion; + clientVersion = contextClientVersion; } - } catch (Exception ignored) { - } + } catch (Parser.RegexException ignored) { } } if (shortClientVersion != null) { - return clientVersion = shortClientVersion; + clientVersion = shortClientVersion; } - throw new ParsingException("Could not get client version"); + try { + key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + } catch (Parser.RegexException e) { + try { + key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html); + } catch (Parser.RegexException ignored) { } + } + } + + /** + * Get the client version + */ + public static String getClientVersion() throws IOException, ExtractionException { + if (!isNullOrEmpty(clientVersion)) return clientVersion; + if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION; + + getClientVersionAndKey(); + if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version"); + return clientVersion; + } + + /** + * Get the key + */ + public static String getKey() throws IOException, ExtractionException { + if (!isNullOrEmpty(key)) return key; + + getClientVersionAndKey(); + if (isNullOrEmpty(key)) throw new ParsingException("Could not extract key"); + return key; } public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index df86b5d3..6e7d41c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -28,6 +28,7 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -107,7 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException { + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") @@ -221,7 +222,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { + "&itct=" + clickTrackingParams); } - private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) { + private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException { if (isNullOrEmpty(continuationItemRenderer)) { return null; } @@ -229,8 +230,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final String token = continuationItemRenderer.getObject("continuationEndpoint") .getObject("continuationCommand").getString("token"); - // FIXME: Key needs to be extracted - final String url = "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; + final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey(); return new Page(url, token); }