From 4cc312086afc6d6772ba657ddf96c5f5b14f41bd Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 15 Apr 2020 14:09:46 +0200 Subject: [PATCH] Introduce Page class --- .../newpipe/extractor/ListExtractor.java | 41 +++++----- .../schabi/newpipe/extractor/ListInfo.java | 13 ++-- .../org/schabi/newpipe/extractor/Page.java | 45 +++++++++++ .../extractor/channel/ChannelInfo.java | 7 +- .../extractor/comments/CommentsInfo.java | 11 +-- .../newpipe/extractor/feed/FeedInfo.java | 2 +- .../newpipe/extractor/kiosk/KioskInfo.java | 10 +-- .../newpipe/extractor/kiosk/KioskList.java | 27 +++---- .../extractor/playlist/PlaylistInfo.java | 7 +- .../newpipe/extractor/search/SearchInfo.java | 9 +-- .../MediaCCCConferenceExtractor.java | 6 +- .../extractors/MediaCCCConferenceKiosk.java | 6 +- .../extractors/MediaCCCSearchExtractor.java | 3 +- .../peertube/PeertubeParsingHelper.java | 29 ++++--- .../extractors/PeertubeAccountExtractor.java | 30 ++++---- .../extractors/PeertubeChannelExtractor.java | 22 +++--- .../extractors/PeertubeCommentsExtractor.java | 24 +++--- .../extractors/PeertubePlaylistExtractor.java | 63 ++++++++++------ .../extractors/PeertubeSearchExtractor.java | 24 +++--- .../extractors/PeertubeStreamExtractor.java | 6 +- .../extractors/PeertubeTrendingExtractor.java | 24 +++--- .../SoundcloudChannelExtractor.java | 13 ++-- .../extractors/SoundcloudChartsExtractor.java | 17 ++--- .../SoundcloudCommentsExtractor.java | 68 +++++++++-------- .../SoundcloudPlaylistExtractor.java | 75 ++++++++----------- .../extractors/SoundcloudSearchExtractor.java | 16 ++-- .../extractors/YoutubeChannelExtractor.java | 32 ++++---- .../extractors/YoutubeCommentsExtractor.java | 65 +++++----------- .../YoutubeCommentsInfoItemExtractor.java | 11 +-- .../extractors/YoutubeFeedExtractor.java | 6 +- .../YoutubeMusicSearchExtractor.java | 26 +++---- .../extractors/YoutubePlaylistExtractor.java | 27 +++---- .../extractors/YoutubeSearchExtractor.java | 25 +++---- .../extractors/YoutubeTrendingExtractor.java | 3 +- .../schabi/newpipe/extractor/utils/Utils.java | 13 ++++ .../extractor/services/DefaultTests.java | 9 +-- .../PeertubeCommentsExtractorTest.java | 48 ++++++------ .../search/PeertubeSearchExtractorTest.java | 2 +- .../SoundcloudPlaylistExtractorTest.java | 2 +- .../search/SoundcloudSearchExtractorTest.java | 2 +- .../youtube/YoutubeCommentsExtractorTest.java | 24 +++--- .../youtube/YoutubePlaylistExtractorTest.java | 2 +- .../search/YoutubeSearchExtractorTest.java | 11 ++- 43 files changed, 477 insertions(+), 429 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/Page.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index 4e14e8f3..cfc3f5fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -4,14 +4,14 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.List; -import javax.annotation.Nonnull; - import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + /** * Base class to extractors that have a list (e.g. playlists, users). */ @@ -37,8 +37,8 @@ public abstract class ListExtractor extends Extractor { } /** - * A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page where the items are from the initial request and - * the nextPageUrl relative to it. + * A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page + * where the items are from the initial request and the nextPage relative to it. * * @return a {@link InfoItemsPage} corresponding to the initial page */ @@ -48,11 +48,11 @@ public abstract class ListExtractor extends Extractor { /** * Get a list of items corresponding to the specific requested page. * - * @param pageUrl any page url got from the exclusive implementation of the list extractor + * @param page any page got from the exclusive implementation of the list extractor * @return a {@link InfoItemsPage} corresponding to the requested page - * @see InfoItemsPage#getNextPageUrl() + * @see InfoItemsPage#getNextPage() */ - public abstract InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException; + public abstract InfoItemsPage getPage(final Page page) throws IOException, ExtractionException; @Override public ListLinkHandler getLinkHandler() { @@ -65,23 +65,22 @@ public abstract class ListExtractor extends Extractor { /** * A class that is used to wrap a list of gathered items and eventual errors, it - * also contains a field that points to the next available page ({@link #nextPageUrl}). + * also contains a field that points to the next available page ({@link #nextPage}). */ public static class InfoItemsPage { private static final InfoItemsPage EMPTY = - new InfoItemsPage<>(Collections.emptyList(), "", Collections.emptyList()); + new InfoItemsPage<>(Collections.emptyList(), null, Collections.emptyList()); /** * A convenient method that returns a representation of an empty page. * - * @return a type-safe page with the list of items and errors empty and the nextPageUrl set to an empty string. + * @return a type-safe page with the list of items and errors empty and the nextPage set to {@code null}. */ public static InfoItemsPage emptyPage() { //noinspection unchecked return (InfoItemsPage) EMPTY; } - /** * The current list of items of this page */ @@ -90,35 +89,37 @@ public abstract class ListExtractor extends Extractor { /** * Url pointing to the next page relative to this one * - * @see ListExtractor#getPage(String) + * @see ListExtractor#getPage(Page) + * @see Page */ - private final String nextPageUrl; + private final Page nextPage; /** * Errors that happened during the extraction */ private final List errors; - public InfoItemsPage(InfoItemsCollector collector, String nextPageUrl) { - this(collector.getItems(), nextPageUrl, collector.getErrors()); + public InfoItemsPage(InfoItemsCollector collector, Page nextPage) { + this(collector.getItems(), nextPage, collector.getErrors()); } - public InfoItemsPage(List itemsList, String nextPageUrl, List errors) { + public InfoItemsPage(List itemsList, Page nextPage, List errors) { this.itemsList = itemsList; - this.nextPageUrl = nextPageUrl; + this.nextPage = nextPage; this.errors = errors; } public boolean hasNextPage() { - return !isNullOrEmpty(nextPageUrl); + return nextPage != null && (!isNullOrEmpty(nextPage.getUrl()) + || !isNullOrEmpty(nextPage.getIds())); } public List getItems() { return itemsList; } - public String getNextPageUrl() { - return nextPageUrl; + public Page getNextPage() { + return nextPage; } public List getErrors() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java index 38177e1e..84010205 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java @@ -8,7 +8,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public abstract class ListInfo extends Info { private List relatedItems; - private String nextPageUrl = null; + private Page nextPage = null; private final List contentFilters; private final String sortFilter; @@ -39,15 +39,16 @@ public abstract class ListInfo extends Info { } public boolean hasNextPage() { - return !isNullOrEmpty(nextPageUrl); + return nextPage != null && (!isNullOrEmpty(nextPage.getUrl()) + || !isNullOrEmpty(nextPage.getIds())); } - public String getNextPageUrl() { - return nextPageUrl; + public Page getNextPage() { + return nextPage; } - public void setNextPageUrl(String pageUrl) { - this.nextPageUrl = pageUrl; + public void setNextPage(Page page) { + this.nextPage = page; } public List getContentFilters() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java new file mode 100644 index 00000000..445fa3f0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java @@ -0,0 +1,45 @@ +package org.schabi.newpipe.extractor; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +public class Page implements Serializable { + private final String url; + private final List ids; + private final Map cookies; + + public Page(final String url, final List ids, final Map cookies) { + this.url = url; + this.ids = ids; + this.cookies = cookies; + } + + public Page(final String url) { + this(url, null, null); + } + + public Page(final String url, final Map cookies) { + this(url, null, cookies); + } + + public Page(final List ids) { + this(null, ids, null); + } + + public Page(final List ids, final Map cookies) { + this(null, ids, cookies); + } + + public String getUrl() { + return url; + } + + public List getIds() { + return ids; + } + + public Map getCookies() { + return cookies; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 52f18ead..6863e602 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.channel; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -49,8 +50,8 @@ public class ChannelInfo extends ListInfo { public static InfoItemsPage getMoreItems(StreamingService service, String url, - String pageUrl) throws IOException, ExtractionException { - return service.getChannelExtractor(url).getPage(pageUrl); + Page page) throws IOException, ExtractionException { + return service.getChannelExtractor(url).getPage(page); } public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException { @@ -81,7 +82,7 @@ public class ChannelInfo extends ListInfo { final InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(itemsPage.getItems()); - info.setNextPageUrl(itemsPage.getNextPageUrl()); + info.setNextPage(itemsPage.getNextPage()); try { info.setSubscriberCount(extractor.getSubscriberCount()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java index e07c6746..e5f25527 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.comments; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -39,23 +40,23 @@ public class CommentsInfo extends ListInfo { InfoItemsPage initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(commentsInfo, commentsExtractor); commentsInfo.setRelatedItems(initialCommentsPage.getItems()); - commentsInfo.setNextPageUrl(initialCommentsPage.getNextPageUrl()); + commentsInfo.setNextPage(initialCommentsPage.getNextPage()); return commentsInfo; } - public static InfoItemsPage getMoreItems(CommentsInfo commentsInfo, String pageUrl) + public static InfoItemsPage getMoreItems(CommentsInfo commentsInfo, Page page) throws ExtractionException, IOException { - return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl); + return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, page); } public static InfoItemsPage getMoreItems(StreamingService service, CommentsInfo commentsInfo, - String pageUrl) throws IOException, ExtractionException { + Page page) throws IOException, ExtractionException { if (null == commentsInfo.getCommentsExtractor()) { commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl())); commentsInfo.getCommentsExtractor().fetchPage(); } - return commentsInfo.getCommentsExtractor().getPage(pageUrl); + return commentsInfo.getCommentsExtractor().getPage(page); } private transient CommentsExtractor commentsExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java index f361cec7..03fe78f6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java @@ -45,7 +45,7 @@ public class FeedInfo extends ListInfo { final InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(itemsPage.getItems()); - info.setNextPageUrl(itemsPage.getNextPageUrl()); + info.setNextPage(itemsPage.getNextPage()); return info; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 521f6b9e..e054214e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -23,6 +23,7 @@ package org.schabi.newpipe.extractor.kiosk; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -33,18 +34,17 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; public class KioskInfo extends ListInfo { - private KioskInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException { super(serviceId, linkHandler, name); } public static ListExtractor.InfoItemsPage getMoreItems(StreamingService service, String url, - String pageUrl) + Page page) throws IOException, ExtractionException { KioskList kl = service.getKioskList(); - KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl); - return extractor.getPage(pageUrl); + KioskExtractor extractor = kl.getExtractorByUrl(url, page); + return extractor.getPage(page); } public static KioskInfo getInfo(String url) throws IOException, ExtractionException { @@ -71,7 +71,7 @@ public class KioskInfo extends ListInfo { final ListExtractor.InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(itemsPage.getItems()); - info.setNextPageUrl(itemsPage.getNextPageUrl()); + info.setNextPage(itemsPage.getNextPage()); return info; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 878fa8ce..5eef4adb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.kiosk; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; @@ -59,23 +60,23 @@ public class KioskList { public KioskExtractor getDefaultKioskExtractor() throws ExtractionException, IOException { - return getDefaultKioskExtractor(""); + return getDefaultKioskExtractor(null); } - public KioskExtractor getDefaultKioskExtractor(String nextPageUrl) + public KioskExtractor getDefaultKioskExtractor(Page nextPage) throws ExtractionException, IOException { - return getDefaultKioskExtractor(nextPageUrl, NewPipe.getPreferredLocalization()); + return getDefaultKioskExtractor(nextPage, NewPipe.getPreferredLocalization()); } - public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization) + public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization) throws ExtractionException, IOException { if (defaultKiosk != null && !defaultKiosk.equals("")) { - return getExtractorById(defaultKiosk, nextPageUrl, localization); + return getExtractorById(defaultKiosk, nextPage, localization); } else { if (!kioskList.isEmpty()) { // if not set get any entry Object[] keySet = kioskList.keySet().toArray(); - return getExtractorById(keySet[0].toString(), nextPageUrl, localization); + return getExtractorById(keySet[0].toString(), nextPage, localization); } else { return null; } @@ -86,12 +87,12 @@ public class KioskList { return defaultKiosk; } - public KioskExtractor getExtractorById(String kioskId, String nextPageUrl) + public KioskExtractor getExtractorById(String kioskId, Page nextPage) throws ExtractionException, IOException { - return getExtractorById(kioskId, nextPageUrl, NewPipe.getPreferredLocalization()); + return getExtractorById(kioskId, nextPage, NewPipe.getPreferredLocalization()); } - public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization) + public KioskExtractor getExtractorById(String kioskId, Page nextPage, Localization localization) throws ExtractionException, IOException { KioskEntry ke = kioskList.get(kioskId); if (ke == null) { @@ -111,17 +112,17 @@ public class KioskList { return kioskList.keySet(); } - public KioskExtractor getExtractorByUrl(String url, String nextPageUrl) + public KioskExtractor getExtractorByUrl(String url, Page nextPage) throws ExtractionException, IOException { - return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization()); + return getExtractorByUrl(url, nextPage, NewPipe.getPreferredLocalization()); } - public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization) + public KioskExtractor getExtractorByUrl(String url, Page nextPage, Localization localization) throws ExtractionException, IOException { for (Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); if (ke.handlerFactory.acceptUrl(url)) { - return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization); + return getExtractorById(ke.handlerFactory.getId(url), nextPage, localization); } } throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index 089b1de5..873d879e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.playlist; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -32,8 +33,8 @@ public class PlaylistInfo extends ListInfo { public static InfoItemsPage getMoreItems(StreamingService service, String url, - String pageUrl) throws IOException, ExtractionException { - return service.getPlaylistExtractor(url).getPage(pageUrl); + Page page) throws IOException, ExtractionException { + return service.getPlaylistExtractor(url).getPage(page); } /** @@ -112,7 +113,7 @@ public class PlaylistInfo extends ListInfo { final InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(itemsPage.getItems()); - info.setNextPageUrl(itemsPage.getNextPageUrl()); + info.setNextPage(itemsPage.getNextPage()); return info; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index b2e072cc..8967b0a8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.search; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; @@ -10,9 +11,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; - public class SearchInfo extends ListInfo { - private String searchString; private String searchSuggestion; private boolean isCorrectedSearch; @@ -55,7 +54,7 @@ public class SearchInfo extends ListInfo { ListExtractor.InfoItemsPage page = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(page.getItems()); - info.setNextPageUrl(page.getNextPageUrl()); + info.setNextPage(page.getNextPage()); return info; } @@ -63,9 +62,9 @@ public class SearchInfo extends ListInfo { public static ListExtractor.InfoItemsPage getMoreItems(StreamingService service, SearchQueryHandler query, - String pageUrl) + Page page) throws IOException, ExtractionException { - return service.getSearchExtractor(query).getPage(pageUrl); + return service.getSearchExtractor(query).getPage(page); } // Getter diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index d2e204c3..b933f179 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -79,8 +80,9 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { } @Override - public InfoItemsPage getPage(final String pageUrl) { - return null; + + public InfoItemsPage getPage(final Page page) { + return InfoItemsPage.emptyPage(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java index 622a3697..8ca27292 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; @@ -37,11 +38,12 @@ public class MediaCCCConferenceKiosk extends KioskExtractor { collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i))); } - return new InfoItemsPage<>(collector, ""); + return new InfoItemsPage<>(collector, null); } @Override - public InfoItemsPage getPage(final String pageUrl) { + + public InfoItemsPage getPage(final Page page) { return InfoItemsPage.emptyPage(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 2ee6c08b..676a89e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; @@ -80,7 +81,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } @Override - public InfoItemsPage getPage(final String pageUrl) { + public InfoItemsPage getPage(final Page page) { return InfoItemsPage.emptyPage(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index f752ecfe..fa47a828 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; @@ -14,7 +15,6 @@ import java.util.Date; import java.util.TimeZone; public class PeertubeParsingHelper { - public static final String START_KEY = "start"; public static final String COUNT_KEY = "count"; public static final int ITEMS_PER_PAGE = 12; @@ -23,17 +23,17 @@ public class PeertubeParsingHelper { private PeertubeParsingHelper() { } - public static void validate(JsonObject json) throws ContentNotAvailableException { - String error = json.getString("error"); + public static void validate(final JsonObject json) throws ContentNotAvailableException { + final String error = json.getString("error"); if (!Utils.isBlank(error)) { throw new ContentNotAvailableException(error); } } - public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { - Date date; + public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { + final Date date; try { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); sdf.setTimeZone(TimeZone.getTimeZone("GMT")); date = sdf.parse(textualUploadDate); } catch (ParseException e) { @@ -45,26 +45,25 @@ public class PeertubeParsingHelper { return uploadDate; } - public static String getNextPageUrl(String prevPageUrl, long total) { - String prevStart; + public static Page getNextPage(final String prevPageUrl, final long total) { + final String prevStart; try { prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); } catch (Parser.RegexException e) { - return ""; + return null; } - if (Utils.isBlank(prevStart)) return ""; - long nextStart = 0; + if (Utils.isBlank(prevStart)) return null; + final long nextStart; try { nextStart = Long.parseLong(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { - return ""; + return null; } if (nextStart >= total) { - return ""; + return null; } else { - return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + nextStart); + return new Page(prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + nextStart)); } } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index af627811..12588b49 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -87,15 +88,15 @@ public class PeertubeAccountExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(pageUrl); + return getPage(new Page(pageUrl)); } - private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { + private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonObject json) throws ParsingException { final JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); } catch (Exception e) { - throw new ParsingException("unable to extract channel streams", e); + throw new ParsingException("Unable to extract account streams", e); } for (final Object c : contents) { @@ -108,27 +109,28 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - final Response response = getDownloader().get(pageUrl); + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + final Response response = getDownloader().get(page.getUrl()); JsonObject json = null; if (response != null && !Utils.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { - throw new ParsingException("Could not parse json data for kiosk info", e); + throw new ParsingException("Could not parse json data for account info", e); } } - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final long total; if (json != null) { PeertubeParsingHelper.validate(json); - total = JsonUtils.getNumber(json, "total").longValue(); + final long total = JsonUtils.getNumber(json, "total").longValue(); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, json); + + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); } else { - throw new ExtractionException("Unable to get PeerTube kiosk info"); + throw new ExtractionException("Unable to get PeerTube account info"); } - return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override @@ -137,7 +139,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor { if (response != null && response.responseBody() != null) { setInitialData(response.responseBody()); } else { - throw new ExtractionException("Unable to extract PeerTube channel data"); + throw new ExtractionException("Unable to extract PeerTube account data"); } } @@ -145,9 +147,9 @@ public class PeertubeAccountExtractor extends ChannelExtractor { try { json = JsonParser.object().from(responseBody); } catch (JsonParserException e) { - throw new ExtractionException("Unable to extract PeerTube channel data", e); + throw new ExtractionException("Unable to extract PeerTube account data", e); } - if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); + if (json == null) throw new ExtractionException("Unable to extract PeerTube account data"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index cb8a5f78..d4a30c73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -93,7 +94,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(pageUrl); + return getPage(new Page(pageUrl)); } private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonObject json) throws ParsingException { @@ -101,7 +102,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { try { contents = (JsonArray) JsonUtils.getValue(json, "data"); } catch (Exception e) { - throw new ParsingException("unable to extract channel streams", e); + throw new ParsingException("Unable to extract channel streams", e); } for (final Object c : contents) { @@ -114,27 +115,28 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - final Response response = getDownloader().get(pageUrl); + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + final Response response = getDownloader().get(page.getUrl()); JsonObject json = null; if (response != null && !Utils.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { - throw new ParsingException("Could not parse json data for kiosk info", e); + throw new ParsingException("Could not parse json data for channel info", e); } } - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final long total; if (json != null) { PeertubeParsingHelper.validate(json); - total = JsonUtils.getNumber(json, "total").longValue(); + final long total = JsonUtils.getNumber(json, "total").longValue(); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, json); + + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); } else { - throw new ExtractionException("Unable to get PeerTube kiosk info"); + throw new ExtractionException("Unable to get PeerTube channel info"); } - return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index b8949329..d1b4378e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -24,8 +25,6 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelp import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; public class PeertubeCommentsExtractor extends CommentsExtractor { - private long total; - public PeertubeCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { super(service, uiHandler); } @@ -33,7 +32,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(pageUrl); + return getPage(new Page(pageUrl)); } private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject json) throws ParsingException { @@ -51,8 +50,8 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - final Response response = getDownloader().get(pageUrl); + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + final Response response = getDownloader().get(page.getUrl()); JsonObject json = null; if (response != null && !Utils.isBlank(response.responseBody())) { try { @@ -62,16 +61,19 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } } - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); if (json != null) { - final Number number = JsonUtils.getNumber(json, "total"); - if (number != null) this.total = number.longValue(); + PeertubeParsingHelper.validate(json); + final long total = JsonUtils.getNumber(json, "total").longValue(); + + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); collectCommentsFrom(collector, json); + + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); } else { - throw new ExtractionException("Unable to get peertube comments info"); + throw new ExtractionException("Unable to get PeerTube kiosk info"); } - return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } - public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException { } + @Override + public void onFetchPage(Downloader downloader) { } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 5020fd94..59043f9d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -27,7 +29,6 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelp public class PeertubePlaylistExtractor extends PlaylistExtractor { private JsonObject playlistInfo; - private String initialPageUrl; public PeertubePlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) { super(service, linkHandler); @@ -84,31 +85,50 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return getPage(initialPageUrl); + return getPage(new Page(getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); + } + + private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonObject json) throws ParsingException { + final JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + } catch (Exception e) { + throw new ParsingException("Unable to extract playlist streams", e); + } + + final String baseUrl = getBaseUrl(); + for (final Object c : contents) { + if (c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + collector.commit(extractor); + } + } } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - final Response response = getDownloader().get(pageUrl); - final JsonObject playlistVideos; - try { - playlistVideos = JsonParser.object().from(response.responseBody()); - } catch (JsonParserException jpe) { - throw new ExtractionException("Could not parse json", jpe); - } - PeertubeParsingHelper.validate(playlistVideos); - - final long total = JsonUtils.getNumber(playlistVideos, "total").longValue(); - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - final JsonArray videos = playlistVideos.getArray("data"); - for (final Object o : videos) { - final JsonObject video = ((JsonObject) o).getObject("video"); - collector.commit(new PeertubeStreamInfoItemExtractor(video, getBaseUrl())); + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + final Response response = getDownloader().get(page.getUrl()); + JsonObject json = null; + if (response != null && !Utils.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for playlist info", e); + } } - return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); + if (json != null) { + PeertubeParsingHelper.validate(json); + final long total = JsonUtils.getNumber(json, "total").longValue(); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, json); + + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); + } else { + throw new ExtractionException("Unable to get PeerTube playlist info"); + } } @Override @@ -120,7 +140,6 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { throw new ExtractionException("Could not parse json", jpe); } PeertubeParsingHelper.validate(playlistInfo); - initialPageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 680f99c1..9d96a49f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -5,8 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.InfoItemExtractor; -import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -46,12 +45,10 @@ public class PeertubeSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(pageUrl); + return getPage(new Page(pageUrl)); } - private InfoItemsCollector collectStreamsFrom(final JsonObject json) throws ParsingException { - final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - + private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonObject json) throws ParsingException { final JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); @@ -67,13 +64,11 @@ public class PeertubeSearchExtractor extends SearchExtractor { collector.commit(extractor); } } - - return collector; } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - final Response response = getDownloader().get(pageUrl); + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + final Response response = getDownloader().get(page.getUrl()); JsonObject json = null; if (response != null && !Utils.isBlank(response.responseBody())) { try { @@ -84,10 +79,15 @@ public class PeertubeSearchExtractor extends SearchExtractor { } if (json != null) { + PeertubeParsingHelper.validate(json); final long total = JsonUtils.getNumber(json, "total").longValue(); - return new InfoItemsPage<>(collectStreamsFrom(json), PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); + + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + collectStreamsFrom(collector, json); + + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); } else { - throw new ExtractionException("Unable to get peertube search info"); + throw new ExtractionException("Unable to get PeerTube search info"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 232556ec..d001296c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -341,7 +341,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { if (response != null && response.responseBody() != null) { setInitialData(response.responseBody()); } else { - throw new ExtractionException("Unable to extract peertube channel data"); + throw new ExtractionException("Unable to extract PeerTube channel data"); } loadSubtitles(); @@ -351,9 +351,9 @@ public class PeertubeStreamExtractor extends StreamExtractor { try { json = JsonParser.object().from(responseBody); } catch (JsonParserException e) { - throw new ExtractionException("Unable to extract peertube stream data", e); + throw new ExtractionException("Unable to extract PeerTube stream data", e); } - if (json == null) throw new ExtractionException("Unable to extract peertube stream data"); + if (json == null) throw new ExtractionException("Unable to extract PeerTube stream data"); PeertubeParsingHelper.validate(json); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index 37c22aa4..8aebc91d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -19,8 +20,6 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import javax.annotation.Nonnull; - import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; @@ -38,7 +37,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(pageUrl); + return getPage(new Page(pageUrl)); } private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonObject json) throws ParsingException { @@ -60,8 +59,8 @@ public class PeertubeTrendingExtractor extends KioskExtractor { } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - final Response response = getDownloader().get(pageUrl); + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + final Response response = getDownloader().get(page.getUrl()); JsonObject json = null; if (response != null && !Utils.isBlank(response.responseBody())) { try { @@ -71,18 +70,19 @@ public class PeertubeTrendingExtractor extends KioskExtractor { } } - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final long total; if (json != null) { - final Number number = JsonUtils.getNumber(json, "total"); - total = number.longValue(); + PeertubeParsingHelper.validate(json); + final long total = JsonUtils.getNumber(json, "total").longValue(); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, json); + + return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); } else { - throw new ExtractionException("Unable to get peertube kiosk info"); + throw new ExtractionException("Unable to get PeerTube kiosk info"); } - return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total)); } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { } + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index 2dcc81b1..b1841755 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -109,21 +110,17 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); - return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl); + return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl)); } catch (Exception e) { throw new ExtractionException("Could not get next page", e); } } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); + String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl()); - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, new Page(nextPageUrl)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java index 7cd0797b..71a5a936 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -33,19 +34,11 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true); + String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true); - return new InfoItemsPage<>(collector, nextPageUrl); - } - - - private void computeNextPageAndStreams() throws IOException, ExtractionException { + return new InfoItemsPage<>(collector, new Page(nextPageUrl)); } @Nonnull @@ -68,6 +61,6 @@ public class SoundcloudChartsExtractor extends KioskExtractor { String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, new Page(nextPageUrl)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index 1b9f060b..7f99a8b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -4,7 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -15,53 +17,57 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + public class SoundcloudCommentsExtractor extends CommentsExtractor { - - private JsonObject json; - - public SoundcloudCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { + public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { super(service, uiHandler); } @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + final Downloader downloader = NewPipe.getDownloader(); + final Response response = downloader.get(getUrl()); - collectStreamsFrom(collector, json.getArray("collection")); - - return new InfoItemsPage<>(collector, json.getString("next_href")); - } - - @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); - Response rp = dl.get(pageUrl); - try { - json = JsonParser.object().from(rp.responseBody()); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json", e); - } - - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, json.getArray("collection")); - - return new InfoItemsPage<>(collector, json.getString("next_href")); - } - - @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - Response response = downloader.get(getUrl()); + final JsonObject json; try { json = JsonParser.object().from(response.responseBody()); } catch (JsonParserException e) { throw new ParsingException("Could not parse json", e); } + + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + + collectStreamsFrom(collector, json.getArray("collection")); + + return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); } + @Override + public InfoItemsPage getPage(final Page page) throws ExtractionException, IOException { + final Downloader downloader = NewPipe.getDownloader(); + final Response response = downloader.get(page.getUrl()); + + final JsonObject json; + try { + json = JsonParser.object().from(response.responseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json", e); + } + + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + + collectStreamsFrom(collector, json.getArray("collection")); + + return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) { } + private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException { final String url = getUrl(); for (Object comment : entries) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index de46b43a..9e4fdeda 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -15,17 +16,19 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; -@SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { - private static final int streamsPerRequestedPage = 15; + private static final int STREAMS_PER_REQUESTED_PAGE = 15; private String playlistId; private JsonObject playlist; @@ -115,79 +118,65 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public String getSubChannelName() throws ParsingException { + public String getSubChannelName() { return ""; } @Nonnull @Override - public String getSubChannelUrl() throws ParsingException { + public String getSubChannelUrl() { return ""; } @Nonnull @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() { return ""; } - @Nonnull - @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); - StringBuilder nextPageUrlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id="); - nextPageUrlBuilder.append(SoundcloudParsingHelper.clientId()); - nextPageUrlBuilder.append("&ids="); + public InfoItemsPage getInitialPage() { + final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); + final List ids = new ArrayList<>(); - JsonArray tracks = playlist.getArray("tracks"); + final JsonArray tracks = playlist.getArray("tracks"); for (Object o : tracks) { if (o instanceof JsonObject) { - JsonObject track = (JsonObject) o; + final JsonObject track = (JsonObject) o; if (track.has("title")) { // i.e. if full info is available streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); } else { // %09d would be enough, but a 0 before the number does not create problems, so let's be sure - nextPageUrlBuilder.append(String.format("%010d,", track.getInt("id"))); + ids.add(String.format("%010d", track.getInt("id"))); } } } - nextPageUrlBuilder.setLength(nextPageUrlBuilder.length() - 1); // remove trailing , - String nextPageUrl = nextPageUrlBuilder.toString(); - if (nextPageUrl.endsWith("&ids")) { - // there are no other videos - nextPageUrl = ""; - } - return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl); + return new InfoItemsPage<>(streamInfoItemsCollector, new Page(ids)); } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - // see computeInitialTracksAndNextPageUrl - final int lengthFirstPartOfUrl = ("https://api-v2.soundcloud.com/tracks?client_id=" - + SoundcloudParsingHelper.clientId() - + "&ids=").length(); - final int lengthOfEveryStream = 11; - - String currentPageUrl, nextUrl; - int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage; - if (pageUrl.length() <= lengthMaxStreams) { - currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max - nextUrl = ""; // afterwards the list is complete + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + final List currentIds; + final List nextIds; + if (page.getIds().size() <= STREAMS_PER_REQUESTED_PAGE) { + // Fetch every remaining stream, there are less than the max + currentIds = page.getIds(); + nextIds = null; } else { - currentPageUrl = pageUrl.substring(0, lengthMaxStreams); - nextUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams); + currentIds = page.getIds().subList(0, STREAMS_PER_REQUESTED_PAGE); + nextIds = page.getIds().subList(STREAMS_PER_REQUESTED_PAGE, page.getIds().size()); } - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody(); + final String currentPageUrl = "https://api-v2.soundcloud.com/tracks?client_id=" + + SoundcloudParsingHelper.clientId() + + "&ids=" + Utils.join(",", currentIds); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody(); try { - JsonArray tracks = JsonParser.array().from(response); + final JsonArray tracks = JsonParser.array().from(response); for (Object track : tracks) { if (track instanceof JsonObject) { collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track)); @@ -197,6 +186,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { throw new ParsingException("Could not parse json response", e); } - return new InfoItemsPage<>(collector, nextUrl); + return new InfoItemsPage<>(collector, new Page(nextIds)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index d2c78b39..50a52a31 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -46,20 +47,20 @@ public class SoundcloudSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return new InfoItemsPage<>(collectItems(searchCollection), getNextPageUrlFromCurrentUrl(getUrl())); + return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(getUrl())); } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { final Downloader dl = getDownloader(); try { - final String response = dl.get(pageUrl, getExtractorLocalization()).responseBody(); + final String response = dl.get(page.getUrl(), getExtractorLocalization()).responseBody(); searchCollection = JsonParser.object().from(response).getArray("collection"); } catch (JsonParserException e) { throw new ParsingException("Could not parse json response", e); } - return new InfoItemsPage<>(collectItems(searchCollection), getNextPageUrlFromCurrentUrl(pageUrl)); + return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page.getUrl())); } @Override @@ -102,7 +103,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor { return collector; } - private String getNextPageUrlFromCurrentUrl(String currentUrl) + private Page getNextPageFromCurrentUrl(String currentUrl) throws MalformedURLException, UnsupportedEncodingException { final int pageOffset = Integer.parseInt( Parser.compatParseMap( @@ -110,8 +111,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor { .getQuery()) .get("offset")); - return currentUrl.replace("&offset=" + - Integer.toString(pageOffset), - "&offset=" + Integer.toString(pageOffset + ITEMS_PER_PAGE)); + return new Page(currentUrl.replace("&offset=" + pageOffset, + "&offset=" + (pageOffset + ITEMS_PER_PAGE))); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index cb898618..e2d12274 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -224,7 +225,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getInitialPage() throws ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String nextPageUrl = null; + Page nextPage = null; if (getVideoTab() != null) { final JsonObject gridRenderer = getVideoTab().getObject("content").getObject("sectionListRenderer") @@ -233,44 +234,39 @@ public class YoutubeChannelExtractor extends ChannelExtractor { collectStreamsFrom(collector, gridRenderer.getArray("items")); - nextPageUrl = getNextPageUrlFrom(gridRenderer.getArray("continuations")); + nextPage = getNextPageFrom(gridRenderer.getArray("continuations")); } - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextPage); } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { // Unfortunately, we have to fetch the page even if we are only getting next streams, // as they don't deliver enough information on their own (the channel name, for example). fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization()); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("gridContinuation"); collectStreamsFrom(collector, sectionListContinuation.getArray("items")); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); + return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations"))); } - - private String getNextPageUrlFrom(JsonArray continuations) { + private Page getNextPageFrom(final JsonArray continuations) { if (isNullOrEmpty(continuations)) { - return ""; + return null; } - JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - String continuation = nextContinuationData.getString("continuation"); - String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + + "&continuation=" + continuation + "&itct=" + clickTrackingParams); } private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 9ab90efb..62d1eb97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; + +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -17,7 +19,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; -import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -26,19 +27,18 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; -import static java.util.Collections.singletonList; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; +import static java.util.Collections.singletonList; public class YoutubeCommentsExtractor extends CommentsExtractor { - // using the mobile site for comments because it loads faster and uses get requests instead of post private static final String USER_AGENT = "Mozilla/5.0 (Android 8.1.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0"; private static final Pattern YT_CLIENT_NAME_PATTERN = Pattern.compile("INNERTUBE_CONTEXT_CLIENT_NAME\\\":(.*?)[,}]"); private String ytClientVersion; private String ytClientName; - private InfoItemsPage initPage; + private String responseBody; public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { super(service, uiHandler); @@ -46,48 +46,45 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - // initial page does not load any comments but is required to get comments token - super.fetchPage(); - return initPage; + String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}"); + String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\""); + return getPage(getNextPage(commentsToken)); } - private String getNextPageUrl(JsonObject ajaxJson) throws ParsingException { + private Page getNextPage(JsonObject ajaxJson) throws ParsingException { JsonArray arr; try { arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); } catch (Exception e) { - return ""; + return null; } if (arr.isEmpty()) { - return ""; + return null; } String continuation; try { continuation = JsonUtils.getString(arr.getObject(0), "nextContinuationData.continuation"); } catch (Exception e) { - return ""; + return null; } - return getNextPageUrl(continuation); + return getNextPage(continuation); } - private String getNextPageUrl(String continuation) throws ParsingException { + private Page getNextPage(String continuation) throws ParsingException { Map params = new HashMap<>(); params.put("action_get_comments", "1"); params.put("pbj", "1"); params.put("ctoken", continuation); try { - return "https://m.youtube.com/watch_comment?" + getDataString(params); + return new Page("https://m.youtube.com/watch_comment?" + getDataString(params)); } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not get next page url", e); } } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - String ajaxResponse = makeAjaxRequest(pageUrl); + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + String ajaxResponse = makeAjaxRequest(page.getUrl()); JsonObject ajaxJson; try { ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1); @@ -96,11 +93,10 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); collectCommentsFrom(collector, ajaxJson); - return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson)); + return new InfoItemsPage<>(collector, getNextPage(ajaxJson)); } private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException { - JsonArray contents; try { contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items"); @@ -128,16 +124,13 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { final Map> requestHeaders = new HashMap<>(); requestHeaders.put("User-Agent", singletonList(USER_AGENT)); final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization()); - String responseBody = response.responseBody(); + responseBody = response.responseBody(); ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\""); ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody); - String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}"); - String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\""); - initPage = getPage(getNextPageUrl(commentsToken)); } - private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException { + private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException { Map> requestHeaders = new HashMap<>(); requestHeaders.put("Accept", singletonList("*/*")); requestHeaders.put("User-Agent", singletonList(USER_AGENT)); @@ -166,22 +159,4 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { int endIndex = doc.indexOf(end, beginIndex); return doc.substring(beginIndex, endIndex); } - - public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException { - try { - return JsonUtils.getString(object, "simpleText"); - } catch (Exception e1) { - try { - JsonArray arr = JsonUtils.getArray(object, "runs"); - String result = ""; - for (int i = 0; i < arr.size(); i++) { - result = result + JsonUtils.getString(arr.getObject(i), "text"); - } - return result; - } catch (Exception e2) { - return ""; - } - } - } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 9d659f5d..10413c7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -11,7 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @@ -43,7 +44,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getName() throws ParsingException { try { - return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText")); + return getTextFromObject(JsonUtils.getObject(json, "authorText")); } catch (Exception e) { return ""; } @@ -52,7 +53,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getTextualUploadDate() throws ParsingException { try { - return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText")); + return getTextFromObject(JsonUtils.getObject(json, "publishedTimeText")); } catch (Exception e) { throw new ParsingException("Could not get publishedTimeText", e); } @@ -81,7 +82,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getCommentText() throws ParsingException { try { - String commentText = YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "contentText")); + String commentText = getTextFromObject(JsonUtils.getObject(json, "contentText")); // youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io return Utils.removeUTF8BOM(commentText); } catch (Exception e) { @@ -111,7 +112,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getUploaderName() throws ParsingException { try { - return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText")); + return getTextFromObject(JsonUtils.getObject(json, "authorText")); } catch (Exception e) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java index b68e5e14..c5c566d4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java @@ -5,6 +5,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -15,9 +16,10 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + public class YoutubeFeedExtractor extends FeedExtractor { public YoutubeFeedExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -66,7 +68,7 @@ public class YoutubeFeedExtractor extends FeedExtractor { } @Override - public InfoItemsPage getPage(String pageUrl) { + public InfoItemsPage getPage(Page page) { return null; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 3d0ee898..6028035f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -169,7 +170,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); - String nextPageUrl = null; + Page nextPage = null; for (Object content : contents) { if (((JsonObject) content).has("musicShelfRenderer")) { @@ -177,19 +178,15 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { collectMusicStreamsFrom(collector, musicShelfRenderer.getArray("contents")); - nextPageUrl = getNextPageUrlFrom(musicShelfRenderer.getArray("continuations")); + nextPage = getNextPageFrom(musicShelfRenderer.getArray("continuations")); } } - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextPage); } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); @@ -229,7 +226,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidJsonResponseBody(getDownloader().post(pageUrl, headers, json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json)); final JsonObject ajaxJson; try { @@ -243,7 +240,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); final JsonArray continuations = musicShelfContinuation.getArray("continuations"); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) { @@ -488,16 +485,17 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } } - private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { + private Page getNextPageFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { if (isNullOrEmpty(continuations)) { - return ""; + return null; } final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); final String continuation = nextContinuationData.getString("continuation"); final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; + return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + + "&continuation=" + continuation + "&itct=" + clickTrackingParams + "&alt=json" + + "&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 3a6d8d11..c23b6dcd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -169,7 +170,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String nextPageUrl = null; + Page nextPage = null; final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") @@ -191,39 +192,35 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer"); collectStreamsFrom(collector, videos.getArray("contents")); - nextPageUrl = getNextPageUrlFrom(videos.getArray("continuations")); + nextPage = getNextPageFrom(videos.getArray("continuations")); } - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextPage); } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization()); final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("playlistVideoListContinuation"); collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); + return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations"))); } - private String getNextPageUrlFrom(final JsonArray continuations) { + private Page getNextPageFrom(final JsonArray continuations) { if (isNullOrEmpty(continuations)) { - return ""; + return null; } JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; + return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams); } private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 430af92c..5eaafbde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -101,27 +102,23 @@ public class YoutubeSearchExtractor extends SearchExtractor { final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - String nextPageUrl = null; + Page nextPage = null; for (Object section : sections) { final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer"); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - nextPageUrl = getNextPageUrlFrom(itemSectionRenderer.getArray("continuations")); + nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations")); } - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextPage); } @Override - public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (isNullOrEmpty(pageUrl)) { - throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); - } - + public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization()); final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("itemSectionContinuation"); @@ -129,7 +126,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); final JsonArray continuations = itemSectionRenderer.getArray("continuations"); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException { @@ -149,16 +146,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { + private Page getNextPageFrom(final JsonArray continuations) throws ParsingException { if (isNullOrEmpty(continuations)) { - return ""; + return null; } final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); final String continuation = nextContinuationData.getString("continuation"); final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; + return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 3b2dd843..91188f37 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -23,6 +23,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -61,7 +62,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } @Override - public InfoItemsPage getPage(String pageUrl) { + public InfoItemsPage getPage(Page page) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 5b70ce59..288e401c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -7,6 +7,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -222,4 +223,16 @@ public class Utils { return true; } + + public static String join(final CharSequence delimiter, final Iterable elements) { + final StringBuilder stringBuilder = new StringBuilder(); + final Iterator iterator = elements.iterator(); + while (iterator.hasNext()) { + stringBuilder.append(iterator.next()); + if (iterator.hasNext()) { + stringBuilder.append(delimiter); + } + } + return stringBuilder.toString(); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index 3d9c2395..a335b0ec 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -86,8 +87,6 @@ public final class DefaultTests { public static void assertNoMoreItems(ListExtractor extractor) throws Exception { final ListExtractor.InfoItemsPage initialPage = extractor.getInitialPage(); assertFalse("More items available when it shouldn't", initialPage.hasNextPage()); - final String nextPageUrl = initialPage.getNextPageUrl(); - assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl)); } public static void assertNoDuplicatedItems(StreamingService expectedService, @@ -121,7 +120,7 @@ public final class DefaultTests { public static ListExtractor.InfoItemsPage defaultTestMoreItems(ListExtractor extractor) throws Exception { final ListExtractor.InfoItemsPage initialPage = extractor.getInitialPage(); assertTrue("Doesn't have more items", initialPage.hasNextPage()); - ListExtractor.InfoItemsPage nextPage = extractor.getPage(initialPage.getNextPageUrl()); + ListExtractor.InfoItemsPage nextPage = extractor.getPage(initialPage.getNextPage()); final List items = nextPage.getItems(); assertFalse("Next page is empty", items.isEmpty()); assertEmptyErrors("Next page have errors", nextPage.getErrors()); @@ -131,9 +130,9 @@ public final class DefaultTests { } public static void defaultTestGetPageInNewExtractor(ListExtractor extractor, ListExtractor newExtractor) throws Exception { - final String nextPageUrl = extractor.getInitialPage().getNextPageUrl(); + final Page nextPage = extractor.getInitialPage().getNextPage(); - final ListExtractor.InfoItemsPage page = newExtractor.getPage(nextPageUrl); + final ListExtractor.InfoItemsPage page = newExtractor.getPage(nextPage); defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index d80416db..af156917 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -14,7 +15,9 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; public class PeertubeCommentsExtractorTest { @@ -31,37 +34,38 @@ public class PeertubeCommentsExtractorTest { @Test public void testGetComments() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); - assertTrue(comments.getErrors().isEmpty()); - boolean result = findInComments(comments, "@root A great documentary on a great guy."); + while (comments.hasNextPage() && !result) { - comments = extractor.getPage(comments.getNextPageUrl()); + comments = extractor.getPage(comments.getNextPage()); result = findInComments(comments, "@root A great documentary on a great guy."); } - + assertTrue(result); } - + @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { - final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); - assertTrue(commentsInfo.getErrors().isEmpty()); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); assertEquals("Comments", commentsInfo.getName()); - + boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); - String nextPage = commentsInfo.getNextPageUrl(); - while (!Utils.isBlank(nextPage) && !result) { - final InfoItemsPage moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); + + Page nextPage = commentsInfo.getNextPage(); + InfoItemsPage moreItems = new InfoItemsPage<>(null, nextPage, null); + while (moreItems.hasNextPage() && !result) { + moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); result = findInComments(moreItems.getItems(), "Loved it!!!"); - nextPage = moreItems.getNextPageUrl(); + nextPage = moreItems.getNextPage(); } + assertTrue(result); } - + @Test public void testGetCommentsAllData() throws IOException, ExtractionException { - final InfoItemsPage comments = extractor.getInitialPage(); - for (final CommentsInfoItem c : comments.getItems()) { + InfoItemsPage comments = extractor.getInitialPage(); + for (CommentsInfoItem c : comments.getItems()) { assertFalse(Utils.isBlank(c.getUploaderUrl())); assertFalse(Utils.isBlank(c.getUploaderName())); assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); @@ -71,16 +75,16 @@ public class PeertubeCommentsExtractorTest { assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertFalse(Utils.isBlank(c.getThumbnailUrl())); assertFalse(Utils.isBlank(c.getUrl())); - assertEquals(-1, c.getLikeCount()); + assertFalse(c.getLikeCount() != -1); } } - - private boolean findInComments(final InfoItemsPage comments, final String comment) { + + private boolean findInComments(InfoItemsPage comments, String comment) { return findInComments(comments.getItems(), comment); } - - private boolean findInComments(final List comments, final String comment) { - for (final CommentsInfoItem c : comments) { + + private boolean findInComments(List comments, String comment) { + for (CommentsInfoItem c : comments) { if (c.getCommentText().contains(comment)) { return true; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java index 5ea11633..80654a40 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java @@ -51,7 +51,7 @@ public class PeertubeSearchExtractorTest { extractor.fetchPage(); final InfoItemsPage page1 = extractor.getInitialPage(); - final InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + final InfoItemsPage page2 = extractor.getPage(page1.getNextPage()); assertNoDuplicatedItems(PeerTube, page1, page2); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 6a366fd3..de9094c0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -268,7 +268,7 @@ public class SoundcloudPlaylistExtractorTest { ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); // Test for 2 more levels for (int i = 0; i < 2; i++) { - currentPage = extractor.getPage(currentPage.getNextPageUrl()); + currentPage = extractor.getPage(currentPage.getNextPage()); defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java index c99bf2a5..4c320012 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -119,7 +119,7 @@ public class SoundcloudSearchExtractorTest { extractor.fetchPage(); final InfoItemsPage page1 = extractor.getInitialPage(); - final InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + final InfoItemsPage page2 = extractor.getPage(page1.getNextPage()); assertNoDuplicatedItems(SoundCloud, page1, page2); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 89ca1b11..bb2b17be 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -22,7 +23,6 @@ import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeCommentsExtractorTest { - private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o"; private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o"; private static YoutubeCommentsExtractor extractorYT; @@ -33,6 +33,7 @@ public class YoutubeCommentsExtractorTest { NewPipe.init(DownloaderTestImpl.getInstance()); extractorYT = (YoutubeCommentsExtractor) YouTube .getCommentsExtractor(urlYT); + extractorYT.fetchPage(); extractorInvidious = (YoutubeCommentsExtractor) YouTube .getCommentsExtractor(urlInvidious); } @@ -44,12 +45,11 @@ public class YoutubeCommentsExtractorTest { } private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException { - boolean result; InfoItemsPage comments = extractor.getInitialPage(); - result = findInComments(comments, "s1ck m3m3"); + boolean result = findInComments(comments, "s1ck m3m3"); while (comments.hasNextPage() && !result) { - comments = extractor.getPage(comments.getNextPageUrl()); + comments = extractor.getPage(comments.getNextPage()); result = findInComments(comments, "s1ck m3m3"); } @@ -63,16 +63,18 @@ public class YoutubeCommentsExtractorTest { } private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { - boolean result = false; CommentsInfo commentsInfo = CommentsInfo.getInfo(url); - result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); - /* String nextPage = commentsInfo.getNextPageUrl(); - while (!Utils.isBlank(nextPage) && !result) { - InfoItemsPage moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); + assertEquals("Comments", commentsInfo.getName()); + boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); + + Page nextPage = commentsInfo.getNextPage(); + InfoItemsPage moreItems = new InfoItemsPage<>(null, nextPage, null); + while (moreItems.hasNextPage() && !result) { + moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); result = findInComments(moreItems.getItems(), "s1ck m3m3"); - nextPage = moreItems.getNextPageUrl(); - }*/ + nextPage = moreItems.getNextPage(); + } return result; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index f2e15a3f..2b03579f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -209,7 +209,7 @@ public class YoutubePlaylistExtractorTest { // test for 2 more levels for (int i = 0; i < 2; i++) { - currentPage = extractor.getPage(currentPage.getNextPageUrl()); + currentPage = extractor.getPage(currentPage.getNextPage()); defaultTestListOfItems(YouTube, currentPage.getItems(), currentPage.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 85d98f81..6e07b021 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -19,8 +19,9 @@ import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; public class YoutubeSearchExtractorTest { public static class All extends DefaultSearchExtractorTest { @@ -189,13 +190,11 @@ public class YoutubeSearchExtractorTest { final ListExtractor.InfoItemsPage initialPage = extractor().getInitialPage(); // YouTube actually gives us an empty next page, but after that, no more pages. assertTrue(initialPage.hasNextPage()); - final ListExtractor.InfoItemsPage nextEmptyPage = extractor.getPage(initialPage.getNextPageUrl()); + final ListExtractor.InfoItemsPage nextEmptyPage = extractor.getPage(initialPage.getNextPage()); assertEquals(0, nextEmptyPage.getItems().size()); assertEmptyErrors("Empty page has errors", nextEmptyPage.getErrors()); assertFalse("More items available when it shouldn't", nextEmptyPage.hasNextPage()); - final String nextPageUrl = nextEmptyPage.getNextPageUrl(); - assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl)); } } @@ -207,7 +206,7 @@ public class YoutubeSearchExtractorTest { extractor.fetchPage(); final ListExtractor.InfoItemsPage page1 = extractor.getInitialPage(); - final ListExtractor.InfoItemsPage page2 = extractor.getPage(page1.getNextPageUrl()); + final ListExtractor.InfoItemsPage page2 = extractor.getPage(page1.getNextPage()); assertNoDuplicatedItems(YouTube, page1, page2); }