Don't call fetch page in constructor

src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -33,12 +33,16 @@ public abstract class Extractor {
      */
     @Nullable
     private String cleanUrl;
+    private boolean pageFetched = false;
+    private final Downloader downloader;
 
     public Extractor(StreamingService service, String url) throws ExtractionException {
         if(service == null) throw new NullPointerException("service is null");
         if(url == null) throw new NullPointerException("url is null");
         this.service = service;
         this.originalUrl = url;
+        this.downloader = NewPipe.getDownloader();
+        if(downloader == null) throw new NullPointerException("downloader is null");
     }
 
     /**
@@ -49,8 +53,26 @@ public abstract class Extractor {
 
     /**
      * Fetch the current page.
+     * @throws IOException if the page can not be loaded
+     * @throws ExtractionException if the pages content is not understood
      */
-    public abstract void fetchPage() throws IOException, ExtractionException;
+    public void fetchPage() throws IOException, ExtractionException {
+        if(pageFetched) return;
+        onFetchPage(downloader);
+        pageFetched = true;
+    }
+
+    protected void assertPageFetched() {
+        if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
+    }
+
+    /**
+     * Fetch the current page.
+     * @param downloader the download to use
+     * @throws IOException if the page can not be loaded
+     * @throws ExtractionException if the pages content is not understood
+     */
+    public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
 
     @Nonnull
     public abstract String getId() throws ParsingException;

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -24,14 +24,13 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
 
         userId = getUrlIdHandler().getId(getOriginalUrl());
         String apiUrl = "https://api.soundcloud.com/users/" + userId +
                 "?client_id=" + SoundcloudParsingHelper.clientId();
 
-        String response = dl.download(apiUrl);
+        String response = downloader.download(apiUrl);
         try {
             user = JsonParser.object().from(response);
         } catch (JsonParserException e) {

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
+import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -23,7 +24,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
     }
 
     @Override
-    public void fetchPage() {
+    public void onFetchPage(@Nonnull Downloader downloader) {
     }
 
     @Nonnull

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -66,13 +66,13 @@ public class SoundcloudParsingHelper {
      * Call the endpoint "/resolve" of the api.<br/>
      * See https://developers.soundcloud.com/docs/api/reference#resolve
      */
-    public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
+    public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
         String apiUrl = "https://api.soundcloud.com/resolve"
                 + "?url=" + URLEncoder.encode(url, "UTF-8")
                 + "&client_id=" + clientId();
 
         try {
-            return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl));
+            return JsonParser.object().from(downloader.download(apiUrl));
         } catch (JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -24,15 +24,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
 
         playlistId = getUrlIdHandler().getId(getOriginalUrl());
         String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
                 "?client_id=" + SoundcloudParsingHelper.clientId() +
                 "&representation=compact";
 
-        String response = dl.download(apiUrl);
+        String response = downloader.download(apiUrl);
         try {
             playlist = JsonParser.object().from(response);
         } catch (JsonParserException e) {

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -21,12 +21,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
     public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
         super(service, url);
-        fetchPage();
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
+        track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
 
         String policy = track.getString("policy", "");
         if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java

@@ -59,9 +59,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        Downloader downloader = NewPipe.getDownloader();
-
         String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
         String pageContent = downloader.download(channelUrl);
         doc = Jsoup.parse(pageContent, channelUrl);

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java

@@ -36,9 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        Downloader downloader = NewPipe.getDownloader();
-
         String pageContent = downloader.download(getCleanUrl());
         doc = Jsoup.parse(pageContent, getCleanUrl());
 

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -86,7 +86,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
         super(service, url);
-        fetchPage();
     }
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -415,7 +414,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             // If the video is age restricted getPlayerConfig will fail
             return null;
         }
-        JsonObject playerConfig = getPlayerConfig(getPageHtml());
+        // TODO: This should be done in onFetchPage()
+        JsonObject playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
         String playerResponse = playerConfig.getObject("args").getString("player_response");
 
         JsonObject captions;
@@ -530,26 +530,23 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     private static String pageHtml = null;
 
-    private String getPageHtml() throws IOException, ExtractionException{
+    private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{
         if (pageHtml == null) {
-            Downloader dl = NewPipe.getDownloader();
+            pageHtml = downloader.download(getCleanUrl());
-            pageHtml = dl.download(getCleanUrl());
         }
         return pageHtml;
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
+        String pageContent = getPageHtml(downloader);
-
-        String pageContent = getPageHtml();
         doc = Jsoup.parse(pageContent, getCleanUrl());
 
 
         String playerUrl;
         // Check if the video is age restricted
         if (pageContent.contains("<meta property=\"og:restrictions:age")) {
-            String infoPageResponse = dl.download(String.format(GET_VIDEO_INFO_URL, getId()));
+            String infoPageResponse = downloader.download(String.format(GET_VIDEO_INFO_URL, getId()));
             videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
             playerUrl = getPlayerUrlFromRestrictedVideo();
             isAgeRestricted = true;

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java

@@ -42,9 +42,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        Downloader downloader = NewPipe.getDownloader();
-
         final String contentCountry = getContentCountry();
         String url = getCleanUrl();
         if(contentCountry != null && !contentCountry.isEmpty()) {

src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -236,7 +236,8 @@ public class StreamInfo extends Info {
      * Fills out the video info fields which are common to all services.
      * Probably needs to be overridden by subclasses
      */
-    public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException {
+    private static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
+        extractor.fetchPage();
         StreamInfo streamInfo;
         try {
             streamInfo = extractImportantData(extractor);

src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java

@@ -29,6 +29,7 @@ public class SoundcloudStreamExtractorDefaultTest {
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
         extractor = (SoundcloudStreamExtractor) SoundCloud.getService().getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
+        extractor.fetchPage();
     }
 
     @Test

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java

@@ -41,12 +41,14 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
  */
 public class YoutubeStreamExtractorDefaultTest {
     public static final String HTTPS = "https://";
-    private StreamExtractor extractor;
+    private YoutubeStreamExtractor extractor;
 
     @Before
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
-        extractor = YouTube.getService().getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
+        extractor = (YoutubeStreamExtractor) YouTube.getService()
+                .getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
+        extractor.fetchPage();
     }
 
     @Test

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java

@@ -28,6 +28,7 @@ public class YoutubeStreamExtractorRestrictedTest {
         NewPipe.init(Downloader.getInstance());
         extractor = (YoutubeStreamExtractor) YouTube.getService()
                 .getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0");
+        extractor.fetchPage();
     }
 
     @Test