From 157055fbfd7d9b125bdbcf286c7e8b33eb736def Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 16:42:04 +0100 Subject: [PATCH] Create getJsonResponse() function --- .../extractors/YoutubeChannelExtractor.java | 41 ++---------------- .../extractors/YoutubePlaylistExtractor.java | 42 ++---------------- .../extractors/YoutubeSearchExtractor.java | 43 ++----------------- .../extractors/YoutubeStreamExtractor.java | 17 +------- .../extractors/YoutubeTrendingExtractor.java | 25 +---------- .../linkHandler/YoutubeParsingHelper.java | 20 +++++++++ 6 files changed, 33 insertions(+), 155 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index c051e31f..f28c0586 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -19,14 +17,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* @@ -62,22 +57,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); } @@ -214,22 +194,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(pageUrl); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("gridContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index fdd75868..2405df12 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,20 +10,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -42,22 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); playlistInfo = getPlaylistInfo(); @@ -207,22 +186,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(pageUrl); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("playlistVideoListContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 222c411f..62d1b3f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; @@ -14,16 +12,12 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* @@ -57,22 +51,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); } @@ -125,23 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } - + final JsonArray ajaxJson = getJsonResponse(pageUrl); JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("itemSectionContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 92096d01..75aa9284 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; @@ -53,6 +52,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -580,20 +580,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - initialAjaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + initialAjaxJson = getJsonResponse(url); final String playerUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 25aaa4db..30eceb9b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -22,8 +22,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -32,18 +30,14 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; public class YoutubeTrendingExtractor extends KioskExtractor { @@ -60,22 +54,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final String url = getUrl() + "?pbj=1&gl=" + getExtractorContentCountry().getCountryCode(); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 5a33e4ce..02306c02 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -9,11 +9,13 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLDecoder; @@ -343,4 +345,22 @@ public class YoutubeParsingHelper { return thumbnailUrl; } + + public static JsonArray getJsonResponse(String url) throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); + final String response = getDownloader().get(url, headers).responseBody(); + + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + try { + return JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } }