diff --git a/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/src/main/java/org/schabi/newpipe/extractor/Extractor.java index 3e0456a9..e7132183 100644 --- a/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -33,12 +33,16 @@ public abstract class Extractor { */ @Nullable private String cleanUrl; + private boolean pageFetched = false; + private final Downloader downloader; public Extractor(StreamingService service, String url) throws ExtractionException { if(service == null) throw new NullPointerException("service is null"); if(url == null) throw new NullPointerException("url is null"); this.service = service; this.originalUrl = url; + this.downloader = NewPipe.getDownloader(); + if(downloader == null) throw new NullPointerException("downloader is null"); } /** @@ -49,8 +53,26 @@ public abstract class Extractor { /** * Fetch the current page. + * @throws IOException if the page can not be loaded + * @throws ExtractionException if the pages content is not understood */ - public abstract void fetchPage() throws IOException, ExtractionException; + public void fetchPage() throws IOException, ExtractionException { + if(pageFetched) return; + onFetchPage(downloader); + pageFetched = true; + } + + protected void assertPageFetched() { + if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()"); + } + + /** + * Fetch the current page. + * @param downloader the download to use + * @throws IOException if the page can not be loaded + * @throws ExtractionException if the pages content is not understood + */ + public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException; @Nonnull public abstract String getId() throws ParsingException; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 845d7721..e81fe2cb 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -24,14 +24,13 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { userId = getUrlIdHandler().getId(getOriginalUrl()); String apiUrl = "https://api.soundcloud.com/users/" + userId + "?client_id=" + SoundcloudParsingHelper.clientId(); - String response = dl.download(apiUrl); + String response = downloader.download(apiUrl); try { user = JsonParser.object().from(response); } catch (JsonParserException e) { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 24e51b1a..ac5f933b 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -23,7 +24,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public void fetchPage() { + public void onFetchPage(@Nonnull Downloader downloader) { } @Nonnull diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 77030537..2c766d01 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -66,13 +66,13 @@ public class SoundcloudParsingHelper { * Call the endpoint "/resolve" of the api.
* See https://developers.soundcloud.com/docs/api/reference#resolve */ - public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException { + public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException { String apiUrl = "https://api.soundcloud.com/resolve" + "?url=" + URLEncoder.encode(url, "UTF-8") + "&client_id=" + clientId(); try { - return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl)); + return JsonParser.object().from(downloader.download(apiUrl)); } catch (JsonParserException e) { throw new ParsingException("Could not parse json response", e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 3f51be7c..35da7642 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -24,15 +24,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { playlistId = getUrlIdHandler().getId(getOriginalUrl()); String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + "?client_id=" + SoundcloudParsingHelper.clientId() + "&representation=compact"; - String response = dl.download(apiUrl); + String response = downloader.download(apiUrl); try { playlist = JsonParser.object().from(response); } catch (JsonParserException e) { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index d323a262..ca02b7a0 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -21,12 +21,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { super(service, url); - fetchPage(); } @Override - public void fetchPage() throws IOException, ExtractionException { - track = SoundcloudParsingHelper.resolveFor(getOriginalUrl()); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl()); String policy = track.getString("policy", ""); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 054e6d79..d1b9ae16 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -59,9 +59,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader downloader = NewPipe.getDownloader(); - + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS; String pageContent = downloader.download(channelUrl); doc = Jsoup.parse(pageContent, channelUrl); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index 02619fc6..c6e5c064 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -36,9 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader downloader = NewPipe.getDownloader(); - + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { String pageContent = downloader.download(getCleanUrl()); doc = Jsoup.parse(pageContent, getCleanUrl()); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 515af748..33c94972 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -86,7 +86,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { super(service, url); - fetchPage(); } /*////////////////////////////////////////////////////////////////////////// @@ -415,7 +414,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { // If the video is age restricted getPlayerConfig will fail return null; } - JsonObject playerConfig = getPlayerConfig(getPageHtml()); + // TODO: This should be done in onFetchPage() + JsonObject playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader())); String playerResponse = playerConfig.getObject("args").getString("player_response"); JsonObject captions; @@ -530,26 +530,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static String pageHtml = null; - private String getPageHtml() throws IOException, ExtractionException{ + private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{ if (pageHtml == null) { - Downloader dl = NewPipe.getDownloader(); - pageHtml = dl.download(getCleanUrl()); + pageHtml = downloader.download(getCleanUrl()); } return pageHtml; } @Override - public void fetchPage() throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); - - String pageContent = getPageHtml(); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String pageContent = getPageHtml(downloader); doc = Jsoup.parse(pageContent, getCleanUrl()); String playerUrl; // Check if the video is age restricted if (pageContent.contains("