Fix bug and some re-structure
This commit is contained in:
parent
5e34556ac3
commit
11216f361f
18 changed files with 134 additions and 134 deletions
|
@ -11,52 +11,46 @@ import java.util.List;
|
|||
*/
|
||||
public abstract class ListExtractor extends Extractor {
|
||||
|
||||
/**
|
||||
* Get a new ListExtractor with the given nextPageUrl set.
|
||||
*/
|
||||
public ListExtractor(StreamingService service, String url) {
|
||||
super(service, url);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException;
|
||||
|
||||
public abstract InfoItemsCollector<? extends InfoItem, ?> getInfoItems() throws IOException, ExtractionException;
|
||||
public abstract String getNextPageUrl() throws IOException, ExtractionException;
|
||||
|
||||
public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException;
|
||||
public abstract InfoItemPage<? extends InfoItem> getPage(final String nextPageUrl) throws IOException, ExtractionException;
|
||||
|
||||
public boolean hasNextPage() throws IOException, ExtractionException {
|
||||
return getNextPageUrl() != null && !getNextPageUrl().isEmpty();
|
||||
final String nextPageUrl = getNextPageUrl();
|
||||
return nextPageUrl != null && !nextPageUrl.isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Inner
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
public static class InfoItemPage {
|
||||
public static class InfoItemPage<T extends InfoItem> {
|
||||
/**
|
||||
* The current list of items to this result
|
||||
*/
|
||||
public final List<InfoItem> infoItemList;
|
||||
private final List<T> itemsList;
|
||||
|
||||
/**
|
||||
* Next url to fetch more items
|
||||
*/
|
||||
public final String nextPageUrl;
|
||||
private final String nextPageUrl;
|
||||
|
||||
/**
|
||||
* Errors that happened during the extraction
|
||||
*/
|
||||
public final List<Throwable> errors;
|
||||
private final List<Throwable> errors;
|
||||
|
||||
public InfoItemPage(InfoItemsCollector collector, String nextPageUrl) {
|
||||
public InfoItemPage(InfoItemsCollector<T, ?> collector, String nextPageUrl) {
|
||||
this(collector.getItemList(), nextPageUrl, collector.getErrors());
|
||||
}
|
||||
|
||||
public InfoItemPage(List<InfoItem> infoItemList, String nextPageUrl, List<Throwable> errors) {
|
||||
this.infoItemList = infoItemList;
|
||||
public InfoItemPage(List<T> itemsList, String nextPageUrl, List<Throwable> errors) {
|
||||
this.itemsList = itemsList;
|
||||
this.nextPageUrl = nextPageUrl;
|
||||
this.errors = errors;
|
||||
}
|
||||
|
@ -65,8 +59,8 @@ public abstract class ListExtractor extends Extractor {
|
|||
return nextPageUrl != null && !nextPageUrl.isEmpty();
|
||||
}
|
||||
|
||||
public List<InfoItem> getItemsList() {
|
||||
return infoItemList;
|
||||
public List<T> getItemsList() {
|
||||
return itemsList;
|
||||
}
|
||||
|
||||
public String getNextPageUrl() {
|
||||
|
|
|
@ -1,9 +1,12 @@
|
|||
package org.schabi.newpipe.extractor.channel;
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.NonNull;
|
||||
import org.schabi.newpipe.extractor.*;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
@ -43,12 +46,10 @@ public abstract class ChannelExtractor extends ListExtractor {
|
|||
|
||||
@NonNull
|
||||
@Override
|
||||
public InfoItemsCollector getInfoItems()
|
||||
throws IOException, ExtractionException {
|
||||
return getStreams();
|
||||
}
|
||||
public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException;
|
||||
@Override
|
||||
public abstract InfoItemPage<StreamInfoItem> getPage(String nextPageUrl) throws IOException, ExtractionException;
|
||||
|
||||
public abstract StreamInfoItemsCollector getStreams() throws IOException, ExtractionException;
|
||||
public abstract String getAvatarUrl() throws ParsingException;
|
||||
public abstract String getBannerUrl() throws ParsingException;
|
||||
public abstract String getFeedUrl() throws ParsingException;
|
||||
|
|
|
@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.ListInfo;
|
|||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -37,7 +37,7 @@ public class ChannelInfo extends ListInfo {
|
|||
}
|
||||
|
||||
|
||||
public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl)
|
||||
public static InfoItemPage<StreamInfoItem> getMoreItems(StreamingService service, String url, String pageUrl)
|
||||
throws IOException, ExtractionException {
|
||||
return service.getChannelExtractor(url).getPage(pageUrl);
|
||||
}
|
||||
|
|
|
@ -20,10 +20,13 @@ package org.schabi.newpipe.extractor.kiosk;
|
|||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.NonNull;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
|
@ -40,6 +43,12 @@ public abstract class KioskExtractor extends ListExtractor {
|
|||
this.id = kioskId;
|
||||
}
|
||||
|
||||
@NonNull
|
||||
@Override
|
||||
public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException;
|
||||
@Override
|
||||
public abstract InfoItemPage<StreamInfoItem> getPage(String nextPageUrl) throws IOException, ExtractionException;
|
||||
|
||||
/**
|
||||
* For certain Websites the content of a kiosk will be different depending
|
||||
* on the country you want to poen the website in. Therefore you should
|
||||
|
|
|
@ -25,6 +25,7 @@ import org.schabi.newpipe.extractor.ListInfo;
|
|||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -35,10 +36,10 @@ public class KioskInfo extends ListInfo {
|
|||
super(serviceId, id, url, name);
|
||||
}
|
||||
|
||||
public static ListExtractor.InfoItemPage getMoreItems(StreamingService service,
|
||||
String url,
|
||||
String pageUrl,
|
||||
String contentCountry) throws IOException, ExtractionException {
|
||||
public static ListExtractor.InfoItemPage<StreamInfoItem> getMoreItems(StreamingService service,
|
||||
String url,
|
||||
String pageUrl,
|
||||
String contentCountry) throws IOException, ExtractionException {
|
||||
KioskList kl = service.getKioskList();
|
||||
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
|
||||
extractor.setContentCountry(contentCountry);
|
||||
|
|
|
@ -1,12 +1,12 @@
|
|||
package org.schabi.newpipe.extractor.playlist;
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.NonNull;
|
||||
import org.schabi.newpipe.extractor.InfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
@ -26,12 +26,10 @@ public abstract class PlaylistExtractor extends ListExtractor {
|
|||
|
||||
@NonNull
|
||||
@Override
|
||||
public InfoItemsCollector getInfoItems()
|
||||
throws IOException, ExtractionException {
|
||||
return getStreams();
|
||||
}
|
||||
public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException;
|
||||
@Override
|
||||
public abstract InfoItemPage<StreamInfoItem> getPage(String nextPageUrl) throws IOException, ExtractionException;
|
||||
|
||||
public abstract StreamInfoItemsCollector getStreams() throws IOException, ExtractionException;
|
||||
public abstract String getThumbnailUrl() throws ParsingException;
|
||||
public abstract String getBannerUrl() throws ParsingException;
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.ListInfo;
|
|||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
@ -16,7 +17,7 @@ public class PlaylistInfo extends ListInfo {
|
|||
super(serviceId, id, url, name);
|
||||
}
|
||||
|
||||
public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException {
|
||||
public static InfoItemPage<StreamInfoItem> getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException {
|
||||
return service.getPlaylistExtractor(url).getPage(pageUrl);
|
||||
}
|
||||
|
||||
|
|
|
@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService;
|
|||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
@ -89,7 +90,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||
|
||||
@Nonnull
|
||||
@Override
|
||||
public StreamInfoItemsCollector getStreams() throws ExtractionException {
|
||||
public StreamInfoItemsCollector getInfoItems() throws ExtractionException {
|
||||
if(streamInfoItemsCollector == null) {
|
||||
computeNextPageAndGetStreams();
|
||||
}
|
||||
|
@ -120,14 +121,14 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
|
||||
if (!hasNextPage()) {
|
||||
throw new ExtractionException("Channel doesn't have more streams");
|
||||
public InfoItemPage<StreamInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
|
||||
if (pageUrl == null || pageUrl.isEmpty()) {
|
||||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
|
||||
|
||||
return new InfoItemPage(collector, nextPageUrl);
|
||||
return new InfoItemPage<>(collector, nextPageUrl);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,19 +1,17 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.schabi.newpipe.extractor.Collector;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class SoundcloudChartsExtractor extends KioskExtractor {
|
||||
private String url;
|
||||
|
@ -44,15 +42,15 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
|
||||
if (!hasNextPage()) {
|
||||
throw new ExtractionException("Chart doesn't have more streams");
|
||||
public InfoItemPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
|
||||
if (pageUrl == null || pageUrl.isEmpty()) {
|
||||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true);
|
||||
|
||||
return new InfoItemPage(collector, nextPageUrl);
|
||||
return new InfoItemPage<>(collector, nextPageUrl);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService;
|
|||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
@ -91,7 +92,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
@Nonnull
|
||||
@Override
|
||||
public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
|
||||
public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
|
||||
if(streamInfoItemsCollector == null) {
|
||||
computeStreamsAndNextPageUrl();
|
||||
}
|
||||
|
@ -119,14 +120,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
|
||||
if (!hasNextPage()) {
|
||||
throw new ExtractionException("Playlist doesn't have more streams");
|
||||
public InfoItemPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
|
||||
if (pageUrl == null || pageUrl.isEmpty()) {
|
||||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
|
||||
|
||||
return new InfoItemPage(collector, nextPageUrl);
|
||||
return new InfoItemPage<>(collector, nextPageUrl);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -13,7 +13,7 @@ import org.schabi.newpipe.extractor.StreamingService;
|
|||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
@ -150,7 +150,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
|
||||
@Nonnull
|
||||
@Override
|
||||
public StreamInfoItemsCollector getStreams() throws ExtractionException {
|
||||
public StreamInfoItemsCollector getInfoItems() throws ExtractionException {
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
|
||||
collectStreamsFrom(collector, ul);
|
||||
|
@ -158,29 +158,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException {
|
||||
public InfoItemPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
|
||||
if (pageUrl == null || pageUrl.isEmpty()) {
|
||||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
||||
// Unfortunately, we have to fetch the page even if we are only getting next streams,
|
||||
// as they don't deliver enough information on their own (the channel name, for example).
|
||||
fetchPage();
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
JsonObject ajaxJson;
|
||||
try {
|
||||
|
||||
if (!hasNextPage()) {
|
||||
throw new ExtractionException("Channel doesn't have more streams");
|
||||
}
|
||||
|
||||
fetchPage();
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonObject ajaxJson = JsonParser.object().from(
|
||||
NewPipe.getDownloader()
|
||||
.download(pageUrl));
|
||||
|
||||
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
|
||||
|
||||
collectStreamsFrom(collector, ajaxHtml.select("body").first());
|
||||
|
||||
return new InfoItemPage(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
|
||||
ajaxJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl));
|
||||
} catch (JsonParserException pe) {
|
||||
throw new ParsingException("Could not parse json data for next streams", pe);
|
||||
}
|
||||
|
||||
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
|
||||
collectStreamsFrom(collector, ajaxHtml.select("body").first());
|
||||
|
||||
return new InfoItemPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
|
||||
}
|
||||
|
||||
private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
|
||||
|
|
|
@ -12,8 +12,8 @@ import org.schabi.newpipe.extractor.StreamingService;
|
|||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
|
@ -26,10 +26,6 @@ import java.io.IOException;
|
|||
public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
private Document doc;
|
||||
/**
|
||||
* It's lazily initialized (when getInfoItemPage is called)
|
||||
*/
|
||||
private Document nextPageAjax;
|
||||
|
||||
public YoutubePlaylistExtractor(StreamingService service, String url) {
|
||||
super(service, url);
|
||||
|
@ -39,8 +35,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
String pageContent = downloader.download(getCleanUrl());
|
||||
doc = Jsoup.parse(pageContent, getCleanUrl());
|
||||
|
||||
nextPageAjax = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -135,7 +129,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
@Nonnull
|
||||
@Override
|
||||
public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException {
|
||||
public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException {
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
|
||||
collectStreamsFrom(collector, tbody);
|
||||
|
@ -143,28 +137,26 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException {
|
||||
public InfoItemPage<StreamInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
|
||||
if (pageUrl == null || pageUrl.isEmpty()) {
|
||||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
JsonObject pageJson;
|
||||
try {
|
||||
if (!hasNextPage()) {
|
||||
throw new ExtractionException("Playlist doesn't have more streams");
|
||||
}
|
||||
|
||||
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
// setupNextStreamsAjax(NewPipe.getDownloader());
|
||||
final JsonObject pageJson = JsonParser.object().from(NewPipe.getDownloader()
|
||||
.download(pageUrl));
|
||||
final Document pageHtml = Jsoup.parse("<table><tbody id=\"pl-load-more-destination\">"
|
||||
+ pageJson.getString("content_html")
|
||||
+ "</tbody></table>", pageUrl);
|
||||
|
||||
collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first());
|
||||
|
||||
|
||||
|
||||
return new InfoItemPage(collector, getNextPageUrlFromAjax(pageJson, pageUrl));
|
||||
pageJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl));
|
||||
} catch (JsonParserException pe) {
|
||||
throw new ParsingException("Could not parse ajax json", pe);
|
||||
}
|
||||
|
||||
final Document pageHtml = Jsoup.parse("<table><tbody id=\"pl-load-more-destination\">"
|
||||
+ pageJson.getString("content_html")
|
||||
+ "</tbody></table>", pageUrl);
|
||||
|
||||
collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first());
|
||||
|
||||
return new InfoItemPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl));
|
||||
}
|
||||
|
||||
private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl)
|
||||
|
|
|
@ -24,10 +24,14 @@ import org.jsoup.Jsoup;
|
|||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.schabi.newpipe.extractor.*;
|
||||
import org.schabi.newpipe.extractor.Downloader;
|
||||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
@ -66,7 +70,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public ListExtractor.InfoItemPage getPage(String pageUrl) {
|
||||
public ListExtractor.InfoItemPage<StreamInfoItem> getPage(String pageUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
|
|||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
|
@ -49,18 +50,18 @@ public class SoundcloudChannelExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testGetStreams() throws Exception {
|
||||
assertFalse("no streams are received", extractor.getStreams().getItemList().isEmpty());
|
||||
assertFalse("no streams are received", extractor.getInfoItems().getItemList().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreamsErrors() throws Exception {
|
||||
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
|
||||
assertTrue("errors during stream list extraction", extractor.getInfoItems().getErrors().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasMoreStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
extractor.getInfoItems();
|
||||
assertTrue("don't have more streams", extractor.hasNextPage());
|
||||
}
|
||||
|
||||
|
@ -77,10 +78,10 @@ public class SoundcloudChannelExtractorTest {
|
|||
@Test
|
||||
public void testGetPage() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
|
||||
assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
|
||||
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
|
||||
extractor.getInfoItems();
|
||||
ListExtractor.InfoItemPage<StreamInfoItem> nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
|
||||
assertTrue("extractor didn't have next streams", !nextItemsResult.getItemsList().isEmpty());
|
||||
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.getErrors().isEmpty());
|
||||
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());
|
||||
}
|
||||
|
||||
|
|
|
@ -80,9 +80,9 @@ public class SoundcloudChartsExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testGetNextPage() throws Exception {
|
||||
extractor.getInfoItems();
|
||||
extractor.getInfoItems().getItemList();
|
||||
assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
|
||||
|| extractor.getPage(extractor.getNextPageUrl()).infoItemList.isEmpty());
|
||||
|| extractor.getPage(extractor.getNextPageUrl()).getItemsList().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
@ -69,25 +69,25 @@ public class SoundcloudPlaylistExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testGetStreams() throws Exception {
|
||||
assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty());
|
||||
assertTrue("no streams are received", !extractor.getInfoItems().getItemList().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreamsErrors() throws Exception {
|
||||
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
|
||||
assertTrue("errors during stream list extraction", extractor.getInfoItems().getErrors().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasMoreStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
extractor.getInfoItems();
|
||||
assertTrue("extractor didn't have more streams", !extractor.hasNextPage());
|
||||
}
|
||||
|
||||
@Test(expected = ExtractionException.class)
|
||||
public void testGetNextPageNonExistent() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
extractor.getInfoItems();
|
||||
|
||||
// This playlist don't have more streams, it should throw an error
|
||||
extractor.getPage(extractor.getNextPageUrl());
|
||||
|
|
|
@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
|
|||
import org.schabi.newpipe.extractor.ListExtractor;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors;
|
||||
|
@ -88,18 +89,18 @@ public class YoutubeChannelExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testGetStreams() throws Exception {
|
||||
assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty());
|
||||
assertTrue("no streams are received", !extractor.getInfoItems().getItemList().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetStreamsErrors() throws Exception {
|
||||
assertEmptyErrors("errors during stream list extraction", extractor.getStreams().getErrors());
|
||||
assertEmptyErrors("errors during stream list extraction", extractor.getInfoItems().getErrors());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasMoreStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
extractor.getInfoItems();
|
||||
assertTrue("don't have more streams", extractor.hasNextPage());
|
||||
}
|
||||
|
||||
|
@ -116,10 +117,10 @@ public class YoutubeChannelExtractorTest {
|
|||
@Test
|
||||
public void testGetPage() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
|
||||
assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty());
|
||||
assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors);
|
||||
extractor.getInfoItems();
|
||||
ListExtractor.InfoItemPage<StreamInfoItem> nextItemsResult = extractor.getPage(extractor.getNextPageUrl());
|
||||
assertTrue("extractor didn't have next streams", !nextItemsResult.getItemsList().isEmpty());
|
||||
assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.getErrors());
|
||||
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -81,7 +81,7 @@ public class YoutubePlaylistExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testGetStreams() throws Exception {
|
||||
List<StreamInfoItem> streams = extractor.getStreams().getItemList();
|
||||
List<StreamInfoItem> streams = extractor.getInfoItems().getItemList();
|
||||
assertFalse("no streams are received", streams.isEmpty());
|
||||
assertTrue(streams.size() > 60);
|
||||
assertFalse(streams.contains(null));
|
||||
|
@ -96,13 +96,13 @@ public class YoutubePlaylistExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testGetStreamsErrors() throws Exception {
|
||||
assertEmptyErrors("errors during stream list extraction", extractor.getStreams().getErrors());
|
||||
assertEmptyErrors("errors during stream list extraction", extractor.getInfoItems().getErrors());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasMoreStreams() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
extractor.getInfoItems();
|
||||
assertTrue("extractor didn't have more streams", extractor.hasNextPage());
|
||||
}
|
||||
|
||||
|
@ -110,10 +110,10 @@ public class YoutubePlaylistExtractorTest {
|
|||
@Test @Ignore
|
||||
public void testGetNextPage() throws Exception {
|
||||
// Setup the streams
|
||||
extractor.getStreams();
|
||||
ListExtractor.InfoItemPage infoItemPage = extractor.getPage(extractor.getNextPageUrl());
|
||||
assertTrue("extractor didn't have next streams", !infoItemPage.infoItemList.isEmpty());
|
||||
assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.errors);
|
||||
extractor.getInfoItems();
|
||||
ListExtractor.InfoItemPage<StreamInfoItem> infoItemPage = extractor.getPage(extractor.getNextPageUrl());
|
||||
assertTrue("extractor didn't have next streams", !infoItemPage.getItemsList().isEmpty());
|
||||
assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.getErrors());
|
||||
assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage());
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue