From 5842b9ad37f760289f74856832e1db0076a5502e Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 24 Feb 2020 19:03:54 +0100 Subject: [PATCH] Add getClientVersion() and HARDCODED_CLIENT_VERSION to YouTubeParsingHelper Prefer hardcoded client version above the current one when making requests to retrieve the same JSON structure for each request. --- .../extractors/YoutubeChannelExtractor.java | 26 +++++++-- .../extractors/YoutubePlaylistExtractor.java | 26 +++++++-- .../linkHandler/YoutubeParsingHelper.java | 58 +++++++++++++++++++ 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 27678657..0cdddde7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -211,14 +211,30 @@ public class YoutubeChannelExtractor extends ChannelExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); try { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + // Use the hardcoded client version first to get JSON with a structure we know + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException pe) { - throw new ParsingException("Could not parse json data for next streams", pe); + } catch (Exception e) { + try { + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException ignored) { + throw new ParsingException("Could not parse json data for next streams", e); + } } JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 8abee5f1..73a2044b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -198,14 +198,30 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); try { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + // Use the hardcoded client version first to get JSON with a structure we know + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException pe) { - throw new ParsingException("Could not parse json data for next streams", pe); + } catch (Exception e) { + try { + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException ignored) { + throw new ParsingException("Could not parse json data for next streams", e); + } } JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 78516f6d..81e4feb3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; @@ -42,6 +43,8 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -157,4 +160,59 @@ public class YoutubeParsingHelper { } } + /** + * Get the client version from a page + * @param initialData + * @param html The page HTML + * @return + * @throws ParsingException + */ + public static String getClientVersion(JsonObject initialData, String html) throws ParsingException { + if (initialData == null) initialData = getInitialData(html); + JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + String shortClientVersion = null; + + // try to get version from initial data first + for (Object service : serviceTrackingParams) { + JsonObject s = (JsonObject) service; + if (s.getString("service").equals("CSI")) { + JsonArray params = s.getArray("params"); + for (Object param: params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("cver")) { + return p.getString("value"); + } + } + } else if (s.getString("service").equals("ECATCHER")) { + // fallback to get a shortened client version which does not contain the last do digits + JsonArray params = s.getArray("params"); + for (Object param: params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("client.version")) { + shortClientVersion = p.getString("value"); + } + } + } + } + + String clientVersion; + String[] patterns = { + "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", + "innertube_context_client_version\":\"([0-9\\.]+?)\"", + "client.version=([0-9\\.]+)" + }; + for (String pattern: patterns) { + try { + clientVersion = Parser.matchGroup1(pattern, html); + if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; + } catch (Exception ignored) {} + } + + if (shortClientVersion != null) return shortClientVersion; + + throw new ParsingException("Could not get client version"); + } + }