Create getJsonResponse() function

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@@ -19,14 +17,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.annotation.Nonnull;
 
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
 
 /*
@@ -62,22 +57,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
 
-        JsonArray ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(url);
-
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
 
         initialData = ajaxJson.getObject(1).getObject("response");
     }
@@ -214,22 +194,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         fetchPage();
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        JsonArray ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(pageUrl);
-
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
 
         JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
                 .getObject("continuationContents").getObject("gridContinuation");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -12,20 +10,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.annotation.Nonnull;
 
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 
@@ -42,22 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         final String url = getUrl() + "&pbj=1";
 
-        JsonArray ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(url);
-
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
 
         initialData = ajaxJson.getObject(1).getObject("response");
         playlistInfo = getPlaylistInfo();
@@ -207,22 +186,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         }
 
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        JsonArray ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(pageUrl);
-
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
 
         JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
                 .getObject("continuationContents").getObject("playlistVideoListContinuation");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -14,16 +12,12 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.annotation.Nonnull;
 
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
 
 /*
@@ -57,22 +51,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         final String url = getUrl() + "&pbj=1";
 
-        JsonArray ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(url);
-
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
 
         initialData = ajaxJson.getObject(1).getObject("response");
     }
@@ -125,23 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         }
 
         InfoItemsSearchCollector collector = getInfoItemSearchCollector();
-        JsonArray ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(pageUrl);
-
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
-
 
         JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
                 .getObject("continuationContents").getObject("itemSectionContinuation");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
@@ -53,6 +52,7 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 
@@ -580,20 +580,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         final String url = getUrl() + "&pbj=1";
 
-        Map<String, List<String>> headers = new HashMap<>();
+        initialAjaxJson = getJsonResponse(url);
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            initialAjaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
 
         final String playerUrl;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -22,8 +22,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -32,18 +30,14 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
-import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.annotation.Nonnull;
 
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
 
 public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@@ -60,22 +54,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
         final String url = getUrl() + "?pbj=1&gl="
                 + getExtractorContentCountry().getCountryCode();
 
-        JsonArray ajaxJson;
+        final JsonArray ajaxJson = getJsonResponse(url);
-
-        Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
-        final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
-        if (response.length() < 50) { // ensure to have a valid response
-            throw new ParsingException("Could not parse json data for next streams");
-        }
-
-        try {
-            ajaxJson = JsonParser.array().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json data for next streams", e);
-        }
 
         initialData = ajaxJson.getObject(1).getObject("response");
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java

@@ -9,11 +9,13 @@ import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URL;
 import java.net.URLDecoder;
@@ -343,4 +345,22 @@ public class YoutubeParsingHelper {
 
         return thumbnailUrl;
     }
+
+    public static JsonArray getJsonResponse(String url) throws IOException, ExtractionException {
+        Map<String, List<String>> headers = new HashMap<>();
+        headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
+        headers.put("X-YouTube-Client-Version",
+                Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
+        final String response = getDownloader().get(url, headers).responseBody();
+
+        if (response.length() < 50) { // ensure to have a valid response
+            throw new ParsingException("JSON response is too short");
+        }
+
+        try {
+            return JsonParser.array().from(response);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse JSON", e);
+        }
+    }
 }