CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Don't call fetch page in constructor

Browse source
This commit is contained in:
Coffeemakr 2017-11-28 13:37:01 +01:00
parent 5272468898
commit 9a46992285
No known key found for this signature in database
GPG key ID: 3F35676D8FF6E743
14 changed files with 51 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/main/java/org/schabi/newpipe/extractor/Extractor.java
View file

@ -33,12 +33,16 @@ public abstract class Extractor {
*/
@Nullable
private String cleanUrl;
private boolean pageFetched = false;
private final Downloader downloader;
public Extractor(StreamingService service, String url) throws ExtractionException {
if(service == null) throw new NullPointerException("service is null");
if(url == null) throw new NullPointerException("url is null");
this.service = service;
this.originalUrl = url;
this.downloader = NewPipe.getDownloader();
if(downloader == null) throw new NullPointerException("downloader is null");
}
/**
@ -49,8 +53,26 @@ public abstract class Extractor {
/**
* Fetch the current page.
* @throws IOException if the page can not be loaded
* @throws ExtractionException if the pages content is not understood
*/
public abstract void fetchPage() throws IOException, ExtractionException;
public void fetchPage() throws IOException, ExtractionException {
if(pageFetched) return;
onFetchPage(downloader);
pageFetched = true;
}
protected void assertPageFetched() {
if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
}
/**
* Fetch the current page.
* @param downloader the download to use
* @throws IOException if the page can not be loaded
* @throws ExtractionException if the pages content is not understood
*/
public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
@Nonnull
public abstract String getId() throws ParsingException;

5
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View file

@ -24,14 +24,13 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
userId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
String response = downloader.download(apiUrl);
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {

3
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View file

@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -23,7 +24,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
}
@Override
public void fetchPage() {
public void onFetchPage(@Nonnull Downloader downloader) {
}
@Nonnull

4
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file

@ -66,13 +66,13 @@ public class SoundcloudParsingHelper {
* Call the endpoint "/resolve" of the api.<br/>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();
try {
return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl));
return JsonParser.object().from(downloader.download(apiUrl));
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

5
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View file

@ -24,15 +24,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
playlistId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
String response = dl.download(apiUrl);
String response = downloader.download(apiUrl);
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {

5
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View file

@ -21,12 +21,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
fetchPage();
}
@Override
public void fetchPage() throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
String policy = track.getString("policy", "");
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {

4
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View file

@ -59,9 +59,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
String pageContent = downloader.download(channelUrl);
doc = Jsoup.parse(pageContent, channelUrl);

4
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
View file

@ -36,9 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl());

17
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View file

@ -86,7 +86,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
@ -415,7 +414,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// If the video is age restricted getPlayerConfig will fail
return null;
}
JsonObject playerConfig = getPlayerConfig(getPageHtml());
// TODO: This should be done in onFetchPage()
JsonObject playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
String playerResponse = playerConfig.getObject("args").getString("player_response");
JsonObject captions;
@ -530,26 +530,23 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static String pageHtml = null;
private String getPageHtml() throws IOException, ExtractionException{
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{
if (pageHtml == null) {
Downloader dl = NewPipe.getDownloader();
pageHtml = dl.download(getCleanUrl());
pageHtml = downloader.download(getCleanUrl());
}
return pageHtml;
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
String pageContent = getPageHtml();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String pageContent = getPageHtml(downloader);
doc = Jsoup.parse(pageContent, getCleanUrl());
String playerUrl;
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
String infoPageResponse = dl.download(String.format(GET_VIDEO_INFO_URL, getId()));
String infoPageResponse = downloader.download(String.format(GET_VIDEO_INFO_URL, getId()));
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = getPlayerUrlFromRestrictedVideo();
isAgeRestricted = true;

4
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java
View file

@ -42,9 +42,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
}
@Override
public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String contentCountry = getContentCountry();
String url = getCleanUrl();
if(contentCountry != null && !contentCountry.isEmpty()) {

3
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View file

@ -236,7 +236,8 @@ public class StreamInfo extends Info {
* Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses
*/
public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException {
private static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
extractor.fetchPage();
StreamInfo streamInfo;
try {
streamInfo = extractImportantData(extractor);

1
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View file

@ -29,6 +29,7 @@ public class SoundcloudStreamExtractorDefaultTest {
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = (SoundcloudStreamExtractor) SoundCloud.getService().getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
extractor.fetchPage();
}
@Test

6
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View file

@ -41,12 +41,14 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
*/
public class YoutubeStreamExtractorDefaultTest {
public static final String HTTPS = "https://";
private StreamExtractor extractor;
private YoutubeStreamExtractor extractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = YouTube.getService().getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
extractor = (YoutubeStreamExtractor) YouTube.getService()
.getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
extractor.fetchPage();
}
@Test

1
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java
View file

@ -28,6 +28,7 @@ public class YoutubeStreamExtractorRestrictedTest {
NewPipe.init(Downloader.getInstance());
extractor = (YoutubeStreamExtractor) YouTube.getService()
.getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0");
extractor.fetchPage();
}
@Test