diff --git a/README.md b/README.md index 9a5928b3..bf3cab89 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`) 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`) - + > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project. ## Supported sites diff --git a/build.gradle b/build.gradle index 3d8ced48..97759443 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.0' + version 'v0.18.6' group 'com.github.TeamNewPipe' repositories { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index b6e90d53..df1c2261 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor; -import java.io.IOException; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -13,7 +8,11 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -public abstract class Extractor{ +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; + +public abstract class Extractor { /** * {@link StreamingService} currently related to this extractor.
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls). @@ -21,19 +20,21 @@ public abstract class Extractor{ private final StreamingService service; private final LinkHandler linkHandler; - @Nullable private Localization forcedLocalization = null; - @Nullable private ContentCountry forcedContentCountry = null; + @Nullable + private Localization forcedLocalization = null; + @Nullable + private ContentCountry forcedContentCountry = null; private boolean pageFetched = false; private final Downloader downloader; public Extractor(final StreamingService service, final LinkHandler linkHandler) { - if(service == null) throw new NullPointerException("service is null"); - if(linkHandler == null) throw new NullPointerException("LinkHandler is null"); + if (service == null) throw new NullPointerException("service is null"); + if (linkHandler == null) throw new NullPointerException("LinkHandler is null"); this.service = service; this.linkHandler = linkHandler; this.downloader = NewPipe.getDownloader(); - if(downloader == null) throw new NullPointerException("downloader is null"); + if (downloader == null) throw new NullPointerException("downloader is null"); } /** @@ -46,11 +47,12 @@ public abstract class Extractor{ /** * Fetch the current page. - * @throws IOException if the page can not be loaded + * + * @throws IOException if the page can not be loaded * @throws ExtractionException if the pages content is not understood */ public void fetchPage() throws IOException, ExtractionException { - if(pageFetched) return; + if (pageFetched) return; onFetchPage(downloader); pageFetched = true; } @@ -65,8 +67,9 @@ public abstract class Extractor{ /** * Fetch the current page. + * * @param downloader the download to use - * @throws IOException if the page can not be loaded + * @throws IOException if the page can not be loaded * @throws ExtractionException if the pages content is not understood */ public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException; @@ -78,6 +81,7 @@ public abstract class Extractor{ /** * Get the name + * * @return the name * @throws ParsingException if the name cannot be extracted */ @@ -93,10 +97,10 @@ public abstract class Extractor{ public String getUrl() throws ParsingException { return linkHandler.getUrl(); } - + @Nonnull public String getBaseUrl() throws ParsingException { - return linkHandler.getBaseUrl(); + return linkHandler.getBaseUrl(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java index 2a3cb5f3..72eca16e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java @@ -27,7 +27,7 @@ import java.util.List; * along with NewPipe. If not, see . */ -public abstract class InfoItemsCollector implements Collector { +public abstract class InfoItemsCollector implements Collector { private final List itemList = new ArrayList<>(); private final List errors = new ArrayList<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index c65283a5..6936568a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -115,26 +115,28 @@ public enum MediaFormat { } /** - * Get the media format by it's id. + * Get the media format by its id. + * * @param id the id * @return the id of the media format or null. */ public static MediaFormat getFormatById(int id) { - for (MediaFormat vf: values()) { + for (MediaFormat vf : values()) { if (vf.id == id) return vf; } return null; } public static MediaFormat getFromSuffix(String suffix) { - for (MediaFormat vf: values()) { + for (MediaFormat vf : values()) { if (vf.suffix.equals(suffix)) return vf; } return null; } - + /** * Get the name of the format + * * @return the name of the format */ public String getName() { @@ -143,6 +145,7 @@ public enum MediaFormat { /** * Get the filename extension + * * @return the filename extension */ public String getSuffix() { @@ -151,10 +154,11 @@ public enum MediaFormat { /** * Get the mime type + * * @return the mime type */ public String getMimeType() { return mimeType; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 742d342d..d91083bf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,15 +1,15 @@ package org.schabi.newpipe.extractor; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.services.bandcamp.BandcampService; import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /* * Copyright (C) Christian Schabesberger 2018 * ServiceList.java is part of NewPipe. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 68398506..e21b17f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -1,20 +1,12 @@ package org.schabi.newpipe.extractor; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -26,6 +18,8 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; /* * Copyright (C) Christian Schabesberger 2018 @@ -269,7 +263,7 @@ public abstract class StreamingService { public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException { ListLinkHandlerFactory llhf = getCommentsLHFactory(); - if(null == llhf) { + if (llhf == null) { return null; } return getCommentsExtractor(llhf.fromUrl(url)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java index a60636b2..19abb1c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java @@ -33,7 +33,7 @@ public class ChannelInfoItemsCollector extends InfoItemsCollector{ +import java.io.IOException; - private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { - super(serviceId, listUrlIdHandler, name); - // TODO Auto-generated constructor stub - } - - public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { +public class CommentsInfo extends ListInfo { + + private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { + super(serviceId, listUrlIdHandler, name); + } + + public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { return getInfo(NewPipe.getServiceByUrl(url), url); } - public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException { - return getInfo(serviceByUrl.getCommentsExtractor(url)); - } + public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException { + return getInfo(serviceByUrl.getCommentsExtractor(url)); + } private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException { // for services which do not have a comments extractor @@ -44,21 +43,21 @@ public class CommentsInfo extends ListInfo{ return commentsInfo; } - + public static InfoItemsPage getMoreItems(CommentsInfo commentsInfo, String pageUrl) throws ExtractionException, IOException { return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl); } - + public static InfoItemsPage getMoreItems(StreamingService service, CommentsInfo commentsInfo, - String pageUrl) throws IOException, ExtractionException { + String pageUrl) throws IOException, ExtractionException { if (null == commentsInfo.getCommentsExtractor()) { commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl())); commentsInfo.getCommentsExtractor().fetchPage(); } return commentsInfo.getCommentsExtractor().getPage(pageUrl); } - + private transient CommentsExtractor commentsExtractor; public CommentsExtractor getCommentsExtractor() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index 59c3911c..b252d463 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -15,12 +15,16 @@ public class Response { private final Map> responseHeaders; private final String responseBody; - public Response(int responseCode, String responseMessage, Map> responseHeaders, @Nullable String responseBody) { + private final String latestUrl; + + public Response(int responseCode, String responseMessage, Map> responseHeaders, + @Nullable String responseBody, @Nullable String latestUrl) { this.responseCode = responseCode; this.responseMessage = responseMessage; this.responseHeaders = responseHeaders != null ? responseHeaders : Collections.>emptyMap(); this.responseBody = responseBody == null ? "" : responseBody; + this.latestUrl = latestUrl; } public int responseCode() { @@ -40,6 +44,16 @@ public class Response { return responseBody; } + /** + * Used for detecting a possible redirection, limited to the latest one. + * + * @return latest url known right before this response object was created + */ + @Nonnull + public String latestUrl() { + return latestUrl; + } + /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ @@ -54,7 +68,8 @@ public class Response { @Nullable public String getHeader(String name) { for (Map.Entry> headerEntry : responseHeaders.entrySet()) { - if (headerEntry.getKey().equalsIgnoreCase(name)) { + final String key = headerEntry.getKey(); + if (key != null && key.equalsIgnoreCase(name)) { if (headerEntry.getValue().size() > 0) { return headerEntry.getValue().get(0); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 7aef494d..44890524 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -25,7 +25,6 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; import javax.annotation.Nonnull; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 5a3be079..878fa8ce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -13,27 +13,30 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -public class KioskList { +public class KioskList { public interface KioskExtractorFactory { KioskExtractor createNewKiosk(final StreamingService streamingService, final String url, final String kioskId) - throws ExtractionException, IOException; + throws ExtractionException, IOException; } private final StreamingService service; private final HashMap kioskList = new HashMap<>(); private String defaultKiosk = null; - @Nullable private Localization forcedLocalization; - @Nullable private ContentCountry forcedContentCountry; + @Nullable + private Localization forcedLocalization; + @Nullable + private ContentCountry forcedContentCountry; private class KioskEntry { public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) { extractorFactory = ef; handlerFactory = h; } + final KioskExtractorFactory extractorFactory; final ListLinkHandlerFactory handlerFactory; } @@ -43,8 +46,8 @@ public class KioskList { } public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id) - throws Exception { - if(kioskList.get(id) != null) { + throws Exception { + if (kioskList.get(id) != null) { throw new Exception("Kiosk with type " + id + " already exists."); } kioskList.put(id, new KioskEntry(extractorFactory, handlerFactory)); @@ -66,10 +69,10 @@ public class KioskList { public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization) throws ExtractionException, IOException { - if(defaultKiosk != null && !defaultKiosk.equals("")) { + if (defaultKiosk != null && !defaultKiosk.equals("")) { return getExtractorById(defaultKiosk, nextPageUrl, localization); } else { - if(!kioskList.isEmpty()) { + if (!kioskList.isEmpty()) { // if not set get any entry Object[] keySet = kioskList.keySet().toArray(); return getExtractorById(keySet[0].toString(), nextPageUrl, localization); @@ -91,7 +94,7 @@ public class KioskList { public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization) throws ExtractionException, IOException { KioskEntry ke = kioskList.get(kioskId); - if(ke == null) { + if (ke == null) { throw new ExtractionException("No kiosk found with the type: " + kioskId); } else { final KioskExtractor kioskExtractor = ke.extractorFactory.createNewKiosk(service, @@ -109,15 +112,15 @@ public class KioskList { } public KioskExtractor getExtractorByUrl(String url, String nextPageUrl) - throws ExtractionException, IOException{ + throws ExtractionException, IOException { return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization()); } public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization) throws ExtractionException, IOException { - for(Map.Entry e : kioskList.entrySet()) { + for (Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); - if(ke.handlerFactory.acceptUrl(url)) { + if (ke.handlerFactory.acceptUrl(url)) { return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java index 52510513..b7439c39 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.linkhandler; -import java.io.Serializable; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; +import java.io.Serializable; + public class LinkHandler implements Serializable { protected final String originalUrl; protected final String url; @@ -31,8 +31,8 @@ public class LinkHandler implements Serializable { public String getId() { return id; } - + public String getBaseUrl() throws ParsingException { - return Utils.getBaseUrl(url); + return Utils.getBaseUrl(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 3890c2b2..6bba7b4e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -35,9 +35,9 @@ public abstract class LinkHandlerFactory { public abstract boolean onAcceptUrl(final String url) throws ParsingException; public String getUrl(String id, String baseUrl) throws ParsingException { - return getUrl(id); + return getUrl(id); } - + /////////////////////////////////// // Logic /////////////////////////////////// @@ -47,7 +47,7 @@ public abstract class LinkHandlerFactory { final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } - + public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if (url == null) throw new IllegalArgumentException("url can not be null"); if (!acceptUrl(url)) { @@ -55,9 +55,9 @@ public abstract class LinkHandlerFactory { } final String id = getId(url); - return new LinkHandler(url, getUrl(id,baseUrl), id); + return new LinkHandler(url, getUrl(id, baseUrl), id); } - + public LinkHandler fromId(String id) throws ParsingException { if (id == null) throw new IllegalArgumentException("id can not be null"); final String url = getUrl(id); @@ -82,5 +82,5 @@ public abstract class LinkHandlerFactory { throw fe; } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 83955265..9ea478b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -1,29 +1,35 @@ package org.schabi.newpipe.extractor.linkhandler; -import java.util.ArrayList; -import java.util.List; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; +import java.util.ArrayList; +import java.util.List; + public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /////////////////////////////////// // To Override /////////////////////////////////// - public List getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);} - public String getSortFilter(String url) throws ParsingException {return ""; } + public List getContentFilter(String url) throws ParsingException { + return new ArrayList<>(0); + } + + public String getSortFilter(String url) throws ParsingException { + return ""; + } + public abstract String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException; - + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return getUrl(id, contentFilter, sortFilter); + return getUrl(id, contentFilter, sortFilter); } /////////////////////////////////// // Logic /////////////////////////////////// - + @Override public ListLinkHandler fromUrl(String url) throws ParsingException { String baseUrl = Utils.getBaseUrl(url); @@ -32,7 +38,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { @Override public ListLinkHandler fromUrl(String url, String baseUrl) throws ParsingException { - if(url == null) throw new IllegalArgumentException("url may not be null"); + if (url == null) throw new IllegalArgumentException("url may not be null"); return new ListLinkHandler(super.fromUrl(url, baseUrl), getContentFilter(url), getSortFilter(url)); } @@ -41,7 +47,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public ListLinkHandler fromId(String id) throws ParsingException { return new ListLinkHandler(super.fromId(id), new ArrayList(0), ""); } - + @Override public ListLinkHandler fromId(String id, String baseUrl) throws ParsingException { return new ListLinkHandler(super.fromId(id, baseUrl), new ArrayList(0), ""); @@ -53,7 +59,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { final String url = getUrl(id, contentFilters, sortFilter); return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } - + public ListLinkHandler fromQuery(String id, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { @@ -61,10 +67,11 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } - + /** - * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, + * For making ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, * however it should not be overridden by the actual implementation. + * * @param id * @return the url coresponding to id without any filters applied */ @@ -76,7 +83,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public String getUrl(String id, String baseUrl) throws ParsingException { return getUrl(id, new ArrayList(0), "", baseUrl); } - + /** * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java index 6c1893c7..67dd7e49 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java @@ -24,6 +24,7 @@ public class SearchQueryHandler extends ListLinkHandler { /** * Returns the search string. Since ListQIHandler is based on ListLinkHandler * getSearchString() is equivalent to calling getId(). + * * @return the search string */ public String getSearchString() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java index 997e2d09..50977e20 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java @@ -13,14 +13,19 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { @Override public abstract String getUrl(String querry, List contentFilter, String sortFilter) throws ParsingException; - public String getSearchString(String url) { return "";} + + public String getSearchString(String url) { + return ""; + } /////////////////////////////////// // Logic /////////////////////////////////// @Override - public String getId(String url) { return getSearchString(url); } + public String getId(String url) { + return getSearchString(url); + } @Override public SearchQueryHandler fromQuery(String querry, @@ -34,10 +39,13 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { } /** - * It's not mandatorry for NewPipe to handle the Url + * It's not mandatory for NewPipe to handle the Url + * * @param url * @return */ @Override - public boolean onAcceptUrl(String url) { return false; } + public boolean onAcceptUrl(String url) { + return false; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java index 7461462a..02b89087 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java @@ -3,7 +3,10 @@ package org.schabi.newpipe.extractor.localization; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; public class Localization implements Serializable { public static final Localization DEFAULT = new Localization("en", "GB"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index aae9b14c..fe20135f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -21,8 +21,9 @@ public class TimeAgoParser { /** * Creates a helper to parse upload dates in the format '2 days ago'. *

- * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items. + * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items. *

+ * * @param patternsHolder An object that holds the "time ago" patterns, special cases, and the language word separator. */ public TimeAgoParser(PatternsHolder patternsHolder) { @@ -164,6 +165,7 @@ public class TimeAgoParser { /** * Marks the time as approximated by setting minutes, seconds and milliseconds to 0. + * * @param calendarTime Time to be marked as approximated */ private void markApproximatedTime(Calendar calendarTime) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java index bad31ea2..206fb68f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; @@ -34,7 +33,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; /** * Collector for search results - * + *

* This collector can handle the following extractor types: *

    *
  • {@link StreamInfoItemExtractor}
    • @@ -59,11 +58,11 @@ public class InfoItemsSearchCollector extends InfoItemsCollector getInitialPage() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray events = conferenceData.getArray("events"); - for(int i = 0; i < events.size(); i++) { + for (int i = 0; i < events.size(); i++) { collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i))); } return new InfoItemsPage<>(collector, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java index 580f79b0..f9800b46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java @@ -32,7 +32,7 @@ public class MediaCCCConferenceKiosk extends KioskExtractor { public InfoItemsPage getInitialPage() throws IOException, ExtractionException { JsonArray conferences = doc.getArray("conferences"); ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId()); - for(int i = 0; i < conferences.size(); i++) { + for (int i = 0; i < conferences.size(); i++) { collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i))); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 429445f2..6d740337 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -50,7 +50,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); searchItems.reset(); - if(getLinkHandler().getContentFilters().contains(CONFERENCES) + if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { searchConferences(getSearchString(), @@ -58,7 +58,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { searchItems); } - if(getLinkHandler().getContentFilters().contains(EVENTS) + if (getLinkHandler().getContentFilters().contains(EVENTS) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { JsonArray events = doc.getArray("events"); @@ -82,8 +82,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - if(getLinkHandler().getContentFilters().contains(EVENTS) - || getLinkHandler().getContentFilters().contains(ALL) + if (getLinkHandler().getContentFilters().contains(EVENTS) + || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { final String site; final String url = getUrl(); @@ -94,17 +94,17 @@ public class MediaCCCSearchExtractor extends SearchExtractor { throw new ExtractionException("Could not parse json.", jpe); } } - if(getLinkHandler().getContentFilters().contains(CONFERENCES) + if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) - conferenceKiosk.fetchPage(); + conferenceKiosk.fetchPage(); } private void searchConferences(String searchString, - List channelItems, - InfoItemsSearchCollector collector) { - for(final ChannelInfoItem item : channelItems) { - if(item.getName().toUpperCase().contains( + List channelItems, + InfoItemsSearchCollector collector) { + for (final ChannelInfoItem item : channelItems) { + if (item.getName().toUpperCase().contains( searchString.toUpperCase())) { collector.commit(new ChannelInfoItemExtractor() { @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index a1dfd245..59bfe1f4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -16,7 +16,9 @@ import org.schabi.newpipe.extractor.stream.*; import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Locale; public class MediaCCCStreamExtractor extends StreamExtractor { @@ -47,8 +49,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() throws ParsingException { - return data.getString("description"); + public Description getDescription() throws ParsingException { + return new Description(data.getString("description"), Description.PLAIN_TEXT); } @Override @@ -103,30 +105,30 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public String getDashMpdUrl() throws ParsingException { - return null; + return ""; } @Nonnull @Override public String getHlsUrl() throws ParsingException { - return null; + return ""; } @Override public List getAudioStreams() throws IOException, ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List audioStreams = new ArrayList<>(); - for(int i = 0; i < recordings.size(); i++) { + for (int i = 0; i < recordings.size(); i++) { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); - if(mimeType.startsWith("audio")) { + if (mimeType.startsWith("audio")) { //first we need to resolve the actual video data from CDN final MediaFormat mediaFormat; - if(mimeType.endsWith("opus")) { + if (mimeType.endsWith("opus")) { mediaFormat = MediaFormat.OPUS; - } else if(mimeType.endsWith("mpeg")) { + } else if (mimeType.endsWith("mpeg")) { mediaFormat = MediaFormat.MP3; - } else if(mimeType.endsWith("ogg")){ + } else if (mimeType.endsWith("ogg")) { mediaFormat = MediaFormat.OGG; } else { throw new ExtractionException("Unknown media format: " + mimeType); @@ -142,16 +144,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getVideoStreams() throws IOException, ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List videoStreams = new ArrayList<>(); - for(int i = 0; i < recordings.size(); i++) { + for (int i = 0; i < recordings.size(); i++) { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); - if(mimeType.startsWith("video")) { + if (mimeType.startsWith("video")) { //first we need to resolve the actual video data from CDN final MediaFormat mediaFormat; - if(mimeType.endsWith("webm")) { + if (mimeType.endsWith("webm")) { mediaFormat = MediaFormat.WEBM; - } else if(mimeType.endsWith("mp4")) { + } else if (mimeType.endsWith("mp4")) { mediaFormat = MediaFormat.MPEG_4; } else { throw new ExtractionException("Unknown media format: " + mimeType); @@ -172,13 +174,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public List getSubtitlesDefault() throws IOException, ExtractionException { - return null; + return Collections.emptyList(); } @Nonnull @Override - public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { - return null; + public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { + return Collections.emptyList(); } @Override @@ -211,7 +213,6 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } catch (JsonParserException jpe) { throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe); } - } @Nonnull @@ -225,4 +226,41 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public String getOriginalUrl() throws ParsingException { return data.getString("frontend_link"); } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public Locale getLanguageInfo() throws ParsingException { + return null; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index b1207a9a..f4074c0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -15,18 +15,23 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory @Override public String getId(String url) throws ParsingException { - if(url.startsWith("https://api.media.ccc.de/public/conferences/")) { + if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { return url.replace("https://api.media.ccc.de/public/conferences/", ""); - } else if(url.startsWith("https://media.ccc.de/c/")) { + } else if (url.startsWith("https://media.ccc.de/c/")) { return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); - } else { - throw new ParsingException("Could not get id from url: " + url); + } else if (url.startsWith("https://media.ccc.de/b/")) { + return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url); } + throw new ParsingException("Could not get id from url: " + url); } @Override public boolean onAcceptUrl(String url) throws ParsingException { - return url.startsWith("https://api.media.ccc.de/public/conferences/") - || url.startsWith("https://media.ccc.de/c/"); + try { + getId(url); + return true; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java index 603f1ea2..d44dc538 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import java.util.List; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index fb6c705a..dcc0b29b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -15,7 +15,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, CONFERENCES, EVENTS diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index fc6bf39e..2c91d056 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -1,16 +1,39 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.MalformedURLException; +import java.net.URL; public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override - public String getId(String url) throws ParsingException { - if(url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q=")) { - return url.replace("https://api.media.ccc.de/public/events/", ""); + public String getId(String urlString) throws ParsingException { + if (urlString.startsWith("https://api.media.ccc.de/public/events/") && + !urlString.contains("?q=")) { + return urlString.substring(39); //remove api…/public/events part } + + URL url; + try { + url = Utils.stringToURL(urlString); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("The given URL is not valid"); + } + + String path = url.getPath(); + // remove leading "/" of URL-path if URL-path is given + if (!path.isEmpty()) { + path = path.substring(1); + } + + if (path.startsWith("v/")) { + return path.substring(2); + } + throw new ParsingException("Could not get id from url: " + url); } @@ -21,7 +44,11 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public boolean onAcceptUrl(String url) throws ParsingException { - return url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q="); + try { + getId(url); + return true; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 992b8eb6..82b67539 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube; -import java.io.IOException; - +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -10,22 +11,20 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import java.io.IOException; public class PeertubeInstance { - + private final String url; private String name; public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); - + public PeertubeInstance(String url) { this.url = url; this.name = "PeerTube"; } - - public PeertubeInstance(String url , String name) { + + public PeertubeInstance(String url, String name) { this.url = url; this.name = name; } @@ -33,22 +32,22 @@ public class PeertubeInstance { public String getUrl() { return url; } - + public void fetchInstanceMetaData() throws Exception { Downloader downloader = NewPipe.getDownloader(); Response response = null; - + try { response = downloader.get(url + "/api/v1/config"); } catch (ReCaptchaException | IOException e) { throw new Exception("unable to configure instance " + url, e); } - - if(null == response || StringUtil.isBlank(response.responseBody())) { + + if (response == null || StringUtil.isBlank(response.responseBody())) { throw new Exception("unable to configure instance " + url); } - - try { + + try { JsonObject json = JsonParser.object().from(response.responseBody()); this.name = JsonUtils.getString(json, "instance.name"); } catch (JsonParserException | ParsingException e) { @@ -59,5 +58,5 @@ public class PeertubeInstance { public String getName() { return name; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index f518faf9..7821aa29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -1,32 +1,34 @@ package org.schabi.newpipe.extractor.services.peertube; +import com.grack.nanojson.JsonObject; +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; - -import org.jsoup.helper.StringUtil; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import com.grack.nanojson.JsonObject; +import java.util.TimeZone; public class PeertubeParsingHelper { - + private PeertubeParsingHelper() { } public static void validate(JsonObject json) throws ContentNotAvailableException { String error = json.getString("error"); - if(!StringUtil.isBlank(error)) { + if (!StringUtil.isBlank(error)) { throw new ContentNotAvailableException(error); } } - + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 47271732..29cc2b1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -1,49 +1,35 @@ package org.schabi.newpipe.extractor.services.peertube; -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.extractors.*; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + public class PeertubeService extends StreamingService { - + private PeertubeInstance instance; - + public PeertubeService(int id) { this(id, PeertubeInstance.defaultInstance); } - + public PeertubeService(int id, PeertubeInstance instance) { super(id, "PeerTube", asList(VIDEO, COMMENTS)); - this.instance = instance; + this.instance = instance; } @Override @@ -117,15 +103,15 @@ public class PeertubeService extends StreamingService { public String getBaseUrl() { return instance.getUrl(); } - + public PeertubeInstance getInstance() { return this.instance; } - + public void setInstance(PeertubeInstance instance) { this.instance = instance; } - + @Override public KioskList getKioskList() throws ExtractionException { KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() { @@ -155,6 +141,6 @@ public class PeertubeService extends StreamingService { return list; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 8b214670..d53fb7ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -1,7 +1,9 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -17,21 +19,18 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import java.io.IOException; public class PeertubeChannelExtractor extends ChannelExtractor { - + private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + private JsonObject json; private final String baseUrl; @@ -45,7 +44,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { String value; try { value = JsonUtils.getString(json, "avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -71,7 +70,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public String getDescription() throws ParsingException { try { return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { + } catch (ParsingException e) { return "No description"; } } @@ -86,18 +85,18 @@ public class PeertubeChannelExtractor extends ChannelExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract channel streams", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + } @Override @@ -110,26 +109,26 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for kiosk info", e); } } - + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { PeertubeParsingHelper.validate(json); Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { - throw new ExtractionException("Unable to get peertube kiosk info"); + throw new ExtractionException("Unable to get PeerTube kiosk info"); } return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; @@ -138,30 +137,30 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } - + @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { - throw new ExtractionException("Unable to extract peertube channel data"); + } else { + throw new ExtractionException("Unable to extract PeerTube channel data"); } - + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } @@ -172,14 +171,14 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube channel data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube channel data"); + if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); } @Override public String getName() throws ParsingException { return JsonUtils.getString(json, "displayName"); } - + @Override public String getOriginalUrl() throws ParsingException { return baseUrl + "/accounts/" + getId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index b7aba4fa..34477459 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -16,9 +17,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeCommentsExtractor extends CommentsExtractor { @@ -26,10 +25,10 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { super(service, uiHandler); } @@ -38,7 +37,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public String getName() throws ParsingException { return "Comments"; } - + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); @@ -49,18 +48,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract comments info", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); collector.commit(extractor); } } - + } @Override @@ -73,18 +72,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for comments info", e); } } - + CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube comments info"); @@ -97,7 +96,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -105,17 +104,17 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index edb59e1f..b7cbcec7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.ServiceList; @@ -9,8 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; - public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @@ -34,7 +33,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -49,13 +48,13 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getTextualPublishedTime() throws ParsingException { return JsonUtils.getString(item, "createdAt"); } - + @Override public DateWrapper getPublishedTime() throws ParsingException { String textualUploadDate = getTextualPublishedTime(); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public int getLikeCount() throws ParsingException { return -1; @@ -67,7 +66,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac try { Document doc = Jsoup.parse(htmlText); return doc.body().text(); - }catch(Exception e) { + } catch (Exception e) { return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); } } @@ -83,7 +82,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -91,7 +90,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public String getAuthorName() throws ParsingException { - return JsonUtils.getString(item, "account.displayName"); + return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); } @Override @@ -100,5 +99,5 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String host = JsonUtils.getString(item, "account.host"); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 98d20b53..1a96111a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -1,7 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -10,7 +8,9 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -public class PeertubePlaylistExtractor extends PlaylistExtractor{ +import java.io.IOException; + +public class PeertubePlaylistExtractor extends PlaylistExtractor { public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -73,8 +73,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor{ @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { - // TODO Auto-generated method stub - + } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index ff021f73..3840bf76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemExtractor; @@ -18,9 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeSearchExtractor extends SearchExtractor { @@ -28,10 +27,10 @@ public class PeertubeSearchExtractor extends SearchExtractor { private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); } @@ -40,7 +39,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { public String getSearchSuggestion() throws ParsingException { return null; } - + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); @@ -48,27 +47,27 @@ public class PeertubeSearchExtractor extends SearchExtractor { } private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { - + final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - + JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract search info", e); } - + String baseUrl = getBaseUrl(); - for(Object c: contents) { - if(c instanceof JsonObject) { + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + return collector; - + } @Override @@ -81,17 +80,17 @@ public class PeertubeSearchExtractor extends SearchExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for search info", e); } } - - if(json != null) { + + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl)); } else { throw new ExtractionException("Unable to get peertube search info"); @@ -103,7 +102,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -111,20 +110,20 @@ public class PeertubeSearchExtractor extends SearchExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index d8d0de00..76ead5d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,15 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -20,34 +17,30 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; public class PeertubeStreamExtractor extends StreamExtractor { - + + private final String baseUrl; private JsonObject json; private List subtitles = new ArrayList<>(); - private final String baseUrl; - + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); this.baseUrl = getBaseUrl(); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(json, "publishedAt"); @@ -63,24 +56,42 @@ public class PeertubeStreamExtractor extends StreamExtractor { return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(json, "thumbnailPath"); + return baseUrl + JsonUtils.getString(json, "previewPath"); } @Override - public String getDescription() throws ParsingException { + public Description getDescription() throws ParsingException { + String text; try { - return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { - return "No description"; + text = JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return Description.emptyDescription; } + if (text.length() == 250 && text.substring(247).equals("...")) { + //if description is shortened, get full description + Downloader dl = NewPipe.getDownloader(); + try { + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + text = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); + } + } + return new Description(text, Description.MARKDOWN); } @Override public int getAgeLimit() throws ParsingException { - return NO_AGE_LIMIT; + boolean isNSFW = JsonUtils.getBoolean(json, "nsfw"); + if (isNSFW) { + return 18; + } else { + return NO_AGE_LIMIT; + } } @Override @@ -130,7 +141,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { String value; try { value = JsonUtils.getString(json, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -157,8 +168,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { List videoStreams = new ArrayList<>(); try { JsonArray streams = json.getArray("files", new JsonArray()); - for(Object s: streams) { - if(!(s instanceof JsonObject)) continue; + for (Object s : streams) { + if (!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; String url = JsonUtils.getString(stream, "fileUrl"); String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); @@ -192,8 +203,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { List filteredSubs = new ArrayList<>(); - for(SubtitlesStream sub: subtitles) { - if(sub.getFormat() == format) { + for (SubtitlesStream sub : subtitles) { + if (sub.getFormat() == format) { filteredSubs.add(sub); } } @@ -215,29 +226,40 @@ public class PeertubeStreamExtractor extends StreamExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); List tags = getTags(); String apiUrl = null; - if(!tags.isEmpty()) { + if (!tags.isEmpty()) { apiUrl = getRelatedStreamsUrl(tags); - - }else { + + } else { apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; } - if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); + if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } - - private List getTags(){ + + @Override + public List getTags() { try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { return Collections.emptyList(); } } - + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + try { + return JsonUtils.getString(json, "support"); + } catch (ParsingException e) { + return ""; + } + } + private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); - for(String tag : tags) { + for (String tag : tags) { params.append("&tagsOneOf="); params.append(URLEncoder.encode(tag, "UTF-8")); } @@ -247,38 +269,38 @@ public class PeertubeStreamExtractor extends StreamExtractor { private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { Response response = getDownloader().get(apiUrl); JsonObject relatedVideosJson = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { relatedVideosJson = JsonParser.object().from(response.responseBody()); } catch (JsonParserException e) { throw new ParsingException("Could not parse json data for related videos", e); } } - - if(relatedVideosJson != null) { + + if (relatedVideosJson != null) { collectStreamsFrom(collector, relatedVideosJson); } } - + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract related videos", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); //do not add the same stream in related streams - if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); + if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor); } } - + } - + @Override public String getErrorMessage() { @@ -288,12 +310,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { + } else { throw new ExtractionException("Unable to extract peertube channel data"); } - + loadSubtitles(); } @@ -303,24 +325,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube stream data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + if (null == json) throw new ExtractionException("Unable to extract peertube stream data"); PeertubeParsingHelper.validate(json); } - + private void loadSubtitles() { if (subtitles.isEmpty()) { try { - Response response = getDownloader().get(getUrl() + "/captions"); + Response response = getDownloader().get(getUrl() + "/captions"); JsonObject captionsJson = JsonParser.object().from(response.responseBody()); JsonArray captions = JsonUtils.getArray(captionsJson, "data"); - for(Object c: captions) { - if(c instanceof JsonObject) { - JsonObject caption = (JsonObject)c; + for (Object c : captions) { + if (c instanceof JsonObject) { + JsonObject caption = (JsonObject) c; String url = baseUrl + JsonUtils.getString(caption, "captionPath"); String languageCode = JsonUtils.getString(caption, "language.id"); String ext = url.substring(url.lastIndexOf(".") + 1); MediaFormat fmt = MediaFormat.getFromSuffix(ext); - if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + if (fmt != null && languageCode != null) + subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); } } } catch (Exception e) { @@ -339,4 +362,32 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + "/videos/watch/" + getId(); } + @Override + public String getHost() throws ParsingException { + return JsonUtils.getString(json, "account.host"); + } + + @Override + public String getPrivacy() throws ParsingException { + return JsonUtils.getString(json, "privacy.label"); + } + + @Override + public String getCategory() throws ParsingException { + return JsonUtils.getString(json, "category.label"); + } + + @Override + public String getLicence() throws ParsingException { + return JsonUtils.getString(json, "licence.label"); + } + + @Override + public Locale getLanguageInfo() throws ParsingException { + try { + return new Locale(JsonUtils.getString(json, "language.id")); + } catch (ParsingException e) { + return null; + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index ca85270a..c4ed7730 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -8,58 +9,56 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; - public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - + protected final JsonObject item; private final String baseUrl; - + public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) { this.item = item; this.baseUrl = baseUrl; } - + @Override public String getUrl() throws ParsingException { String uuid = JsonUtils.getString(item, "uuid"); return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl(); } - + @Override public String getThumbnailUrl() throws ParsingException { String value = JsonUtils.getString(item, "thumbnailPath"); return baseUrl + value; } - + @Override public String getName() throws ParsingException { return JsonUtils.getString(item, "name"); } - + @Override public boolean isAd() throws ParsingException { return false; } - + @Override public long getViewCount() throws ParsingException { Number value = JsonUtils.getNumber(item, "views"); return value.longValue(); } - + @Override public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } - + @Override public String getUploaderName() throws ParsingException { return JsonUtils.getString(item, "account.displayName"); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(item, "publishedAt"); @@ -75,12 +74,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public StreamType getStreamType() throws ParsingException { return StreamType.VIDEO_STREAM; } - + @Override public long getDuration() throws ParsingException { Number value = JsonUtils.getNumber(item, "duration"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java index a2b24614..17062900 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import java.util.List; + public class PeertubeSubscriptionExtractor extends SubscriptionExtractor { public PeertubeSubscriptionExtractor(StreamingService service, List supportedSources) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java index 7e2ed4d9..a028e77a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -public class PeertubeSuggestionExtractor extends SuggestionExtractor{ +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class PeertubeSuggestionExtractor extends SuggestionExtractor { public PeertubeSuggestionExtractor(StreamingService service) { super(service); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index 1cb42046..e4bc1dff 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -16,17 +17,15 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeTrendingExtractor extends KioskExtractor { - + private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; @@ -49,19 +48,19 @@ public class PeertubeTrendingExtractor extends KioskExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { - throw new ParsingException("unable to extract kiosk info", e); + } catch (Exception e) { + throw new ParsingException("Unable to extract kiosk info", e); } - + String baseUrl = getBaseUrl(); - for(Object c: contents) { - if(c instanceof JsonObject) { + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + } @Override @@ -74,18 +73,18 @@ public class PeertubeTrendingExtractor extends KioskExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for kiosk info", e); } } - + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube kiosk info"); @@ -98,7 +97,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -106,17 +105,17 @@ public class PeertubeTrendingExtractor extends KioskExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index 640f99e5..c1e3f570 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); @@ -27,7 +27,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, searchFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java index 9faa3492..9c833750 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory(); @@ -33,10 +33,10 @@ public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilter, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { return baseUrl + String.format(COMMENTS_ENDPOINT, id); } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java index 64e291a5..dd1bd77f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory(); @@ -23,12 +23,12 @@ public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; } - + @Override public String getId(String url) throws ParsingException { return Parser.matchGroup1(ID_PATTERN, url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 69591167..8acf3e80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String CHARSET_UTF_8 = "UTF-8"; @@ -21,9 +21,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { String baseUrl = ServiceList.PeerTube.getBaseUrl(); - return getUrl(searchString, contentFilters, sortFilter, baseUrl); + return getUrl(searchString, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String searchString, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { try { @@ -38,6 +38,6 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { VIDEOS }; + return new String[]{VIDEOS}; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index 3ee7744c..222dc129 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -24,7 +24,7 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, baseUrl); } - + @Override public String getUrl(String id, String baseUrl) { return baseUrl + VIDEO_ENDPOINT + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index c0773c20..1d5609c1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -1,26 +1,26 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.schabi.newpipe.extractor.ServiceList; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; - public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { - - + + private static final PeertubeTrendingLinkHandlerFactory instance = new PeertubeTrendingLinkHandlerFactory(); - + public static final Map KIOSK_MAP; public static final Map REVERSE_KIOSK_MAP; public static final String KIOSK_TRENDING = "Trending"; public static final String KIOSK_MOST_LIKED = "Most liked"; public static final String KIOSK_RECENT = "Recently added"; public static final String KIOSK_LOCAL = "Local"; - + static { Map map = new HashMap<>(); map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending"); @@ -28,24 +28,24 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); KIOSK_MAP = Collections.unmodifiableMap(map); - + Map reverseMap = new HashMap<>(); - for(Map.Entry entry : KIOSK_MAP.entrySet()){ + for (Map.Entry entry : KIOSK_MAP.entrySet()) { reverseMap.put(entry.getValue(), entry.getKey()); } REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap); } - + public static PeertubeTrendingLinkHandlerFactory getInstance() { return instance; } - + @Override public String getUrl(String id, List contentFilters, String sortFilter) { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return String.format(KIOSK_MAP.get(id), baseUrl); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 54541217..853b6fc1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -86,7 +86,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - if(streamInfoItemsCollector == null) { + if (streamInfoItemsCollector == null) { computeNextPageAndGetStreams(); } return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); @@ -94,7 +94,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getNextPageUrl() throws ExtractionException { - if(nextPageUrl == null) { + if (nextPageUrl == null) { computeNextPageAndGetStreams(); } return nextPageUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java index ed5d9a98..be1b9fcd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index b773a935..b9f4894d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -12,8 +12,8 @@ import javax.annotation.Nonnull; import java.io.IOException; public class SoundcloudChartsExtractor extends KioskExtractor { - private StreamInfoItemsCollector collector = null; - private String nextPageUrl = null; + private StreamInfoItemsCollector collector = null; + private String nextPageUrl = null; public SoundcloudChartsExtractor(StreamingService service, ListLinkHandler linkHandler, @@ -68,7 +68,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public String getNextPageUrl() throws IOException, ExtractionException { - if(nextPageUrl == null) { + if (nextPageUrl == null) { computNextPageAndStreams(); } return nextPageUrl; @@ -77,7 +77,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - if(collector == null) { + if (collector == null) { computNextPageAndStreams(); } return new InfoItemsPage<>(collector, getNextPageUrl()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 4617f916..b8f6af24 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,9 +31,9 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "r5ELVSy3RkcjX7ilaL7n2v1Z8irA9SL8"; // Updated on 31/12/19 + private static final String HARDCODED_CLIENT_ID = "t0h1jzYMsaZXy6ggnZO71gHK3Ms6CFwE"; // Updated on 14/03/20 private static String clientId; - + private SoundcloudParsingHelper() { } @@ -103,7 +103,7 @@ public class SoundcloudParsingHelper { /** * Call the endpoint "/resolve" of the api.

    - * + *

    * See https://developers.soundcloud.com/docs/api/reference#resolve */ public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 5bc30972..ca08f66c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java index 17e4e7f2..c7c882a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 9ed340fa..42cd8ae0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -4,7 +4,10 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java index 90057120..dffcc91f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java @@ -25,7 +25,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto try { String url = "https://api-v2.soundcloud.com/search"; - if(contentFilter.size() > 0) { + if (contentFilter.size() > 0) { switch (contentFilter.get(0)) { case TRACKS: url += "/tracks"; @@ -58,7 +58,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, TRACKS, USERS, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index def9c386..30f2fc87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,31 +1,26 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import static java.util.Collections.singletonList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; + public class SoundcloudService extends StreamingService { public SoundcloudService(int id) { super(id, "SoundCloud", singletonList(AUDIO)); } - + @Override public String getBaseUrl() { return "https://soundcloud.com"; @@ -110,15 +105,15 @@ public class SoundcloudService extends StreamingService { return new SoundcloudSubscriptionExtractor(this); } - @Override - public ListLinkHandlerFactory getCommentsLHFactory() { - return null; - } + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return null; + } - @Override + @Override public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { return null; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 14f7023f..2919c89b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -4,7 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -20,6 +22,7 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -73,10 +76,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return artworkUrlBetterResolution; } - @Nonnull @Override - public String getDescription() { - return track.getString("description"); + public Description getDescription() { + return new Description(track.getString("description"), Description.PLAIN_TEXT); } @Override @@ -183,7 +185,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { try { JsonObject mp3UrlObject = JsonParser.object().from(res); // Links in this file are also only valid for a short period. - audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), + audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), MediaFormat.MP3, 128)); } catch (JsonParserException e) { throw new ParsingException("Could not parse streamable url", e); @@ -254,4 +256,41 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public String getErrorMessage() { return null; } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public Locale getLanguageInfo() throws ParsingException { + return null; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java index 3476eab1..fc578411 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java index b12f0467..924c4698 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java @@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 6137f029..eae3bcb9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,13 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; - -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -15,28 +7,22 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import javax.annotation.Nonnull; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; /* * Created by Christian Schabesberger on 23.08.15. @@ -125,7 +111,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d675cb25..11dd8985 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,28 +1,29 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + /* * Created by Christian Schabesberger on 25.07.16. * @@ -45,10 +46,8 @@ import java.io.IOException; @SuppressWarnings("WeakerAccess") public class YoutubeChannelExtractor extends ChannelExtractor { - /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; - private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; - - private Document doc; + private JsonObject initialData; + private JsonObject videoTab; public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -56,21 +55,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; - final Response response = downloader.get(channelUrl, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); + final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; + + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + initialData = ajaxJson.getObject(1).getObject("response"); + YoutubeParsingHelper.defaultAlertsCheck(initialData); } + @Override public String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(doc); + if (getVideoTab() == null) return ""; + return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations")); } @Nonnull @Override public String getUrl() throws ParsingException { try { - return CHANNEL_URL_BASE + getId(); + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId()); } catch (ParsingException e) { return super.getUrl(); } @@ -80,15 +85,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getId() throws ParsingException { try { - return doc.select("meta[itemprop=\"channelId\"]").first().attr("content"); - } catch (Exception ignored) {} - - // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) - try { - Element element = doc.getElementsByClass("yt-uix-subscription-button").first(); - if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first(); - - return element.attr("data-channel-external-id"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("channelId"); } catch (Exception e) { throw new ParsingException("Could not get channel id", e); } @@ -98,7 +95,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getName() throws ParsingException { try { - return doc.select("meta[property=\"og:title\"]").first().attr("content"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title"); } catch (Exception e) { throw new ParsingException("Could not get channel name", e); } @@ -107,7 +104,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { try { - return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src"); + String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") + .getArray("thumbnails").getObject(0).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -116,13 +116,18 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); - String cssContent = el.html(); - String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent); + String url = null; + try { + url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner") + .getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception ignored) {} + if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { + return null; + } - return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url; + return fixThumbnailUrl(url); } catch (Exception e) { - throw new ParsingException("Could not get Banner", e); + throw new ParsingException("Could not get banner", e); } } @@ -137,25 +142,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - - final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); - if (el != null) { - String elTitle = el.attr("title"); + final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); + if (subscriberInfo != null) { try { - return Utils.mixedNumberWordToLong(elTitle); + return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo)); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } } else { - // If the element is null, the channel have the subscriber count disabled - return -1; + // If there's no subscribe button, the channel has the subscriber count disabled + if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) { + return -1; + } else { + return 0; + } } } @Override public String getDescription() throws ParsingException { try { - return doc.select("meta[name=\"description\"]").first().attr("content"); + return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); } catch (Exception e) { throw new ParsingException("Could not get channel description", e); } @@ -165,8 +172,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); - collectStreamsFrom(collector, ul); + + if (getVideoTab() != null) { + JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("gridRenderer").getArray("items"); + collectStreamsFrom(collector, videos); + } + return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -181,106 +194,81 @@ public class YoutubeChannelExtractor extends ChannelExtractor { fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonObject ajaxJson; - try { - final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); - ajaxJson = JsonParser.object().from(response); - } catch (JsonParserException pe) { - throw new ParsingException("Could not parse json data for next streams", pe); - } + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl); - collectStreamsFrom(collector, ajaxHtml.select("body").first()); + JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("gridContinuation"); - return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); + collectStreamsFrom(collector, sectionListContinuation.getArray("items")); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); } - private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) - throws ParsingException { - String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); - if (!loadMoreHtmlDataRaw.isEmpty()) { - return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl)); - } else { - return ""; - } + + private String getNextPageUrlFrom(JsonArray continuations) { + if (continuations == null) return ""; + + JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + String continuation = nextContinuationData.getString("continuation"); + String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } - private String getNextPageUrlFrom(Document d) throws ParsingException { - try { - Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); - if (button != null) { - return button.attr("abs:data-uix-load-more-href"); - } else { - // Sometimes channels are simply so small, they don't have a more streams/videos - return ""; - } - } catch (Exception e) { - throw new ParsingException("Could not get next page url", e); - } - } - - private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) throws ParsingException { + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException { collector.reset(); final String uploaderName = getName(); final String uploaderUrl = getUrl(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Element li : element.children()) { - if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { + for (Object video : videos) { + if (((JsonObject) video).getObject("gridVideoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor( + ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) { @Override - public String getUrl() throws ParsingException { - try { - Element el = li.select("div[class=\"feed-item-dismissable\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - Element el = li.select("div[class=\"feed-item-dismissable\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } - } - - @Override - public String getUploaderName() throws ParsingException { + public String getUploaderName() { return uploaderName; } @Override - public String getUploaderUrl() throws ParsingException { + public String getUploaderUrl() { return uploaderUrl; } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - String url; - Element te = li.select("span[class=\"yt-thumb-clip\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } }); } } } + + private JsonObject getVideoTab() throws ParsingException { + if (this.videoTab != null) return this.videoTab; + + JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs"); + JsonObject videoTab = null; + + for (Object tab : tabs) { + if (((JsonObject) tab).getObject("tabRenderer") != null) { + if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) { + videoTab = ((JsonObject) tab).getObject("tabRenderer"); + break; + } + } + } + + if (videoTab == null) { + throw new ParsingException("Could not find Videos tab"); + } + + try { + if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("messageRenderer") + .getObject("text")).equals("This channel has no videos.")) + return null; + } catch (Exception ignored) {} + + this.videoTab = videoTab; + return videoTab; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index a687c050..29aa045b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,12 +1,14 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import org.jsoup.nodes.Element; +import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* * Created by Christian Schabesberger on 12.02.17. @@ -29,87 +31,67 @@ import java.util.regex.Pattern; */ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private final Element el; + private JsonObject channelInfoItem; - public YoutubeChannelInfoItemExtractor(Element el) { - this.el = el; + public YoutubeChannelInfoItemExtractor(JsonObject channelInfoItem) { + this.channelInfoItem = channelInfoItem; } @Override public String getThumbnailUrl() throws ParsingException { - Element img = el.select("span[class*=\"yt-thumb-simple\"]").first() - .select("img").first(); + try { + String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - String url = img.attr("abs:src"); - - if (url.contains("gif")) { - url = img.attr("abs:data-thumb"); + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); } - return url; } @Override public String getName() throws ParsingException { - return el.select("a[class*=\"yt-uix-tile-link\"]").first() - .text(); + try { + return getTextFromObject(channelInfoItem.getObject("title")); + } catch (Exception e) { + throw new ParsingException("Could not get name", e); + } } @Override public String getUrl() throws ParsingException { try { - String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first() - .attr("abs:data-href"); - - Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)"); - Matcher match = channelIdPattern.matcher(buttonTrackingUrl); - - if (match.matches()) { - return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); - } - } catch(Exception ignored) {} - - // fallback method for channels without "Subscribe" button (or just in case yt changes things) - // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail - try { - return el.select("a[class*=\"yt-uix-tile-link\"]").first() - .attr("abs:href"); + String id = "channel/" + channelInfoItem.getString("channelId"); + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); } catch (Exception e) { - throw new ParsingException("Could not get channel url", e); + throw new ParsingException("Could not get url", e); } } @Override public long getSubscriberCount() throws ParsingException { - final Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first(); - if (subsEl != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text())); - } catch (NumberFormatException e) { - throw new ParsingException("Could not get subscriber count", e); - } - } else { - // If the element is null, the channel have the subscriber count disabled - return -1; + try { + String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText")); + return Utils.mixedNumberWordToLong(subscribers); + } catch (Exception e) { + throw new ParsingException("Could not get subscriber count", e); } } @Override public long getStreamCount() throws ParsingException { - Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first(); - if (metaEl == null) { - return 0; - } else { - return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text())); + try { + return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText")))); + } catch (Exception e) { + throw new ParsingException("Could not get stream count", e); } } @Override public String getDescription() throws ParsingException { - Element desEl = el.select("div[class*=\"yt-lockup-description\"]").first(); - if (desEl == null) { - return ""; - } else { - return desEl.text(); + try { + return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); + } catch (Exception e) { + throw new ParsingException("Could not get description", e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 1c377cc5..78eb59f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -22,7 +21,9 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import static java.util.Collections.singletonList; @@ -58,14 +59,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException { - + JsonArray arr; try { arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); } catch (Exception e) { return ""; } - if(arr.isEmpty()) { + if (arr.isEmpty()) { return ""; } String continuation; @@ -107,11 +108,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException { - + JsonArray contents; try { contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items"); - }catch(Exception e) { + } catch (Exception e) { //no comments return; } @@ -119,12 +120,12 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { List comments; try { comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to get parse youtube comments", e); } - - for(Object c: comments) { - if(c instanceof JsonObject) { + + for (Object c : comments) { + if (c instanceof JsonObject) { CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl(), getTimeAgoParser()); collector.commit(extractor); } @@ -132,7 +133,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void fetchTitle(JsonArray contents) { - if(null == title) { + if (title == null) { try { title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); } catch (Exception e) { @@ -190,7 +191,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { int endIndex = doc.indexOf(end, beginIndex); return doc.substring(beginIndex, endIndex); } - + public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException { try { return JsonUtils.getString(object, "simpleText"); @@ -198,7 +199,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { try { JsonArray arr = JsonUtils.getArray(object, "runs"); String result = ""; - for(int i=0; i getInitialPage() throws ExtractionException { + public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); - collectStreamsFrom(collector, tbody); + + JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("playlistVideoListRenderer").getArray("contents"); + + collectStreamsFrom(collector, videos); return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -139,155 +188,42 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonObject pageJson; - try { - final String responseBody = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); - pageJson = JsonParser.object().from(responseBody); - } catch (JsonParserException pe) { - throw new ParsingException("Could not parse ajax json", pe); - } + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - final Document pageHtml = Jsoup.parse("" - + pageJson.getString("content_html") - + "
    ", pageUrl); + JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("playlistVideoListContinuation"); - collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first()); + collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl)); + return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); } - private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl) - throws ParsingException{ - String nextPageHtml = pageJson.getString("load_more_widget_html"); - if (!nextPageHtml.isEmpty()) { - return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl)); - } else { + private String getNextPageUrlFrom(JsonArray continuations) { + if (continuations == null) { return ""; } + + JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + String continuation = nextContinuationData.getString("continuation"); + String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } - private String getNextPageUrlFrom(Document d) throws ParsingException { - try { - Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); - if (button != null) { - return button.attr("abs:data-uix-load-more-href"); - } else { - // Sometimes playlists are simply so small, they don't have a more streams/videos - return ""; - } - } catch (Exception e) { - throw new ParsingException("could not get next streams' url", e); - } - } - - private void collectStreamsFrom(@Nonnull StreamInfoItemsCollector collector, @Nullable Element element) { + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) { collector.reset(); - if (element == null) { - return; - } - - final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Element li : element.children()) { - if(isDeletedItem(li)) { - continue; + for (Object video : videos) { + if (((JsonObject) video).getObject("playlistVideoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { + @Override + public long getViewCount() { + return -1; + } + }); } - - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - public Element uploaderLink; - - @Override - public boolean isAd() { - return false; - } - - @Override - public String getUrl() throws ParsingException { - try { - return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - return li.attr("data-title"); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } - } - - @Override - public long getDuration() throws ParsingException { - try { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - - Element first = li.select("div[class=\"timestamp\"] span").first(); - if (first == null) { - // Video unavailable (private, deleted, etc.), this is a thing that happens specifically with playlists, - // because in other cases, those videos don't even show up - return -1; - } - - return YoutubeParsingHelper.parseDurationString(first.text()); - } catch (Exception e) { - throw new ParsingException("Could not get duration" + getUrl(), e); - } - } - - - private Element getUploaderLink() { - // should always be present since we filter deleted items - if(uploaderLink == null) { - uploaderLink = li.select("div[class=pl-video-owner] a").first(); - } - return uploaderLink; - } - - @Override - public String getUploaderName() throws ParsingException { - return getUploaderLink().text(); - } - - @Override - public String getUploaderUrl() throws ParsingException { - // this url is not always in the form "/channel/..." - // sometimes Youtube provides urls in the from "/user/..." - return getUploaderLink().attr("abs:href"); - } - - @Override - public String getTextualUploadDate() throws ParsingException { - return ""; - } - - @Override - public long getViewCount() throws ParsingException { - return -1; - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() + "/hqdefault.jpg"; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); } } - - /** - * Check if the playlist item is deleted - * @param li the list item - * @return true if the item is deleted - */ - private boolean isDeletedItem(Element li) { - return li.select("div[class=pl-video-owner] a").isEmpty(); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 8812391f..5b4e7afc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -1,97 +1,68 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import org.jsoup.nodes.Element; +import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { - private final Element el; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; - public YoutubePlaylistInfoItemExtractor(Element el) { - this.el = el; +public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private JsonObject playlistInfoItem; + + public YoutubePlaylistInfoItemExtractor(JsonObject playlistInfoItem) { + this.playlistInfoItem = playlistInfoItem; } @Override public String getThumbnailUrl() throws ParsingException { - String url; - try { - Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - url = te.attr("abs:src"); + String url = playlistInfoItem.getArray("thumbnails").getObject(0) + .getArray("thumbnails").getObject(0).getString("url"); - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } + return fixThumbnailUrl(url); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist thumbnail url", e); + throw new ParsingException("Could not get thumbnail url", e); } - - return url; } @Override public String getName() throws ParsingException { - String name; try { - final Element title = el.select("[class=\"yt-lockup-title\"]").first() - .select("a").first(); - - name = title == null ? "" : title.text(); + return getTextFromObject(playlistInfoItem.getObject("title")); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist name", e); + throw new ParsingException("Could not get name", e); } - - return name; } @Override public String getUrl() throws ParsingException { try { - final Element a = el.select("div[class=\"yt-lockup-meta\"]") - .select("ul[class=\"yt-lockup-meta-info\"]") - .select("li").select("a").first(); - - if(a != null) { - return a.attr("abs:href"); - } - - // this is for yt premium playlists - return el.select("h3[class=\"yt-lockup-title\"").first() - .select("a").first() - .attr("abs:href"); - + String id = playlistInfoItem.getString("playlistId"); + return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(id); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist url", e); + throw new ParsingException("Could not get url", e); } } @Override public String getUploaderName() throws ParsingException { - String name; - try { - final Element div = el.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first(); - - name = div.text(); + return getTextFromObject(playlistInfoItem.getObject("longBylineText")); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist uploader", e); + throw new ParsingException("Could not get uploader name", e); } - - return name; } @Override public long getStreamCount() throws ParsingException { try { - final Element count = el.select("span[class=\"formatted-video-count-label\"]").first() - .select("b").first(); - - return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text())); + return Long.parseLong(Utils.removeNonDigitCharacters(playlistInfoItem.getString("videoCount"))); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist stream count", e); + throw new ParsingException("Could not get stream count", e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 2e3aad32..6fec1164 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -1,26 +1,24 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.utils.Parser; + +import java.io.IOException; import javax.annotation.Nonnull; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* * Created by Christian Schabesberger on 22.07.2018 @@ -43,8 +41,7 @@ import java.net.URL; */ public class YoutubeSearchExtractor extends SearchExtractor { - - private Document doc; + private JsonObject initialData; public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); @@ -52,9 +49,11 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + final String url = getUrl() + "&pbj=1"; + + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); } @Nonnull @@ -64,81 +63,85 @@ public class YoutubeSearchExtractor extends SearchExtractor { } @Override - public String getSearchSuggestion() { - final Element el = doc.select("div[class*=\"spell-correction\"]").first(); - if (el != null) { - return el.select("a").first().text(); - } else { + public String getSearchSuggestion() throws ParsingException { + JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + if (showingResultsForRenderer == null) { return ""; + } else { + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } } @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - return new InfoItemsPage<>(collectItems(doc), getNextPageUrl()); + InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); + + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); + } + + return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override public String getNextPageUrl() throws ExtractionException { - return getUrl() + "&page=" + 2; + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); } @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); - doc = Jsoup.parse(response, pageUrl); - - return new InfoItemsPage<>(collectItems(doc), getNextPageUrlFromCurrentUrl(pageUrl)); - } - - private String getNextPageUrlFromCurrentUrl(String currentUrl) - throws MalformedURLException, UnsupportedEncodingException { - final int pageNr = Integer.parseInt( - Parser.compatParseMap( - new URL(currentUrl) - .getQuery()) - .get("page")); - - return currentUrl.replace("&page=" + pageNr, - "&page=" + Integer.toString(pageNr + 1)); - } - - private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException { - InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - collector.reset(); - - Element list = doc.select("ol[class=\"item-section\"]").first(); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - - for (Element item : list.children()) { - /* First we need to determine which kind of item we are working with. - Youtube depicts five different kinds of items on its search result page. These are - regular videos, playlists, channels, two types of video suggestions, and a "no video - found" item. Since we only want videos, we need to filter out all the others. - An example for this can be seen here: - https://www.youtube.com/results?search_query=asdf&page=1 - - We already applied a filter to the url, so we don't need to care about channels and - playlists now. - */ - - Element el; - - if ((el = item.select("div[class*=\"search-message\"]").first()) != null) { - throw new NothingFoundException(el.text()); - - // video item type - } else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(el, timeAgoParser)); - } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) { - collector.commit(new YoutubeChannelInfoItemExtractor(el)); - } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null && - item.select(".yt-pl-icon-mix").isEmpty()) { - collector.commit(new YoutubePlaylistInfoItemExtractor(el)); - } + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } - return collector; + InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); + + JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("itemSectionContinuation"); + + collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations"))); + } + + private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { + collector.reset(); + + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object item : videos) { + if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { + throw new NothingFoundException(getTextFromObject(((JsonObject) item) + .getObject("backgroundPromoRenderer").getObject("bodyText"))); + } else if (((JsonObject) item).getObject("videoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); + } else if (((JsonObject) item).getObject("channelRenderer") != null) { + collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer"))); + } else if (((JsonObject) item).getObject("playlistRenderer") != null) { + collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); + } + } + } + + private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException { + if (continuations == null) { + return ""; + } + + JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + String continuation = nextContinuationData.getString("continuation"); + String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index cee1a4b9..04818de0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,11 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; + import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -15,31 +11,52 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Request; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; /* * Created by Christian Schabesberger on 06.08.15. @@ -62,8 +79,6 @@ import java.util.regex.Pattern; */ public class YoutubeStreamExtractor extends StreamExtractor { - private static final String TAG = YoutubeStreamExtractor.class.getSimpleName(); - /*////////////////////////////////////////////////////////////////////////// // Exceptions //////////////////////////////////////////////////////////////////////////*/ @@ -74,26 +89,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - public class SubtitlesException extends ContentNotAvailableException { - SubtitlesException(String message, Throwable cause) { - super(message, cause); - } - } - /*//////////////////////////////////////////////////////////////////////////*/ - private Document doc; + private JsonArray initialAjaxJson; @Nullable private JsonObject playerArgs; @Nonnull private final Map videoInfoPage = new HashMap<>(); private JsonObject playerResponse; + private JsonObject initialData; + private JsonObject videoPrimaryInfoRenderer; + private JsonObject videoSecondaryInfoRenderer; + private int ageLimit; @Nonnull private List subtitlesInfos = new ArrayList<>(); - private boolean isAgeRestricted; - public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { super(service, linkHandler); } @@ -106,21 +117,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getName() throws ParsingException { assertPageFetched(); - try { - return playerResponse.getObject("videoDetails").getString("title"); + String title = null; - } catch (Exception e) { - // fallback HTML method - String name = null; + try { + title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); + } catch (Exception ignored) {} + + if (title == null) { try { - name = doc.select("meta[name=title]").attr(CONTENT); + title = playerResponse.getObject("videoDetails").getString("title"); } catch (Exception ignored) {} - if (name == null) { - throw new ParsingException("Could not get name", e); - } - return name; + if (title == null) throw new ParsingException("Could not get name"); } + + return title; } @Override @@ -130,18 +141,39 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); - } catch (Exception e) { - String uploadDate = null; - try { - uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT); - } catch (Exception ignored) {} - - if (uploadDate == null) { - throw new ParsingException("Could not get upload date", e); + JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); + if (micro.getString("uploadDate") != null && !micro.getString("uploadDate").isEmpty()) { + return micro.getString("uploadDate"); } - return uploadDate; - } + if (micro.getString("publishDate") != null && !micro.getString("publishDate").isEmpty()) { + return micro.getString("publishDate"); + } + } catch (Exception ignored) {} + + try { + if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { + String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); + + try { // Premiered 20 hours ago + TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); + Calendar parsedTime = timeAgoParser.parse(time).date(); + return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); + } catch (Exception ignored) {} + + try { // Premiered Feb 21, 2020 + Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); + return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); + } catch (Exception ignored) {} + } + } catch (Exception ignored) {} + + try { + // TODO this parses English formatted dates only, we need a better approach to parse the textual date + Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( + getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); + return new SimpleDateFormat("yyyy-MM-dd").format(d); + } catch (Exception ignored) {} + throw new ParsingException("Could not get upload date"); } @Override @@ -162,122 +194,38 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - return thumbnails.getObject(thumbnails.size() - 1).getString("url"); + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + return fixThumbnailUrl(url); } catch (Exception e) { - String url = null; - try { - url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); - } catch (Exception ignored) {} - - if (url == null) { - throw new ParsingException("Could not get thumbnail url", e); - } - return url; + throw new ParsingException("Could not get thumbnail url"); } } @Nonnull @Override - public String getDescription() throws ParsingException { + public Description getDescription() throws ParsingException { assertPageFetched(); + // description with more info on links try { - // first try to get html-formatted description - return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); - } catch (Exception e) { - try { - // fallback to raw non-html description - return playerResponse.getObject("videoDetails").getString("shortDescription"); - } catch (Exception ignored) { - throw new ParsingException("Could not get the description", e); - } + String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); + return new Description(description, Description.HTML); + } catch (Exception ignored) { } + + // raw non-html description + try { + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); + } catch (Exception ignored) { + throw new ParsingException("Could not get description"); } } - // onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;" - // :00 is NOT recognized as a timestamp in description or comments. - // 0:00 is recognized in both description and comments. - // https://www.youtube.com/watch?v=4cccfDXu1vA - private final static Pattern DESCRIPTION_TIMESTAMP_ONCLICK_REGEX = Pattern.compile( - "seekTo\\(" - + "(?:(\\d+)\\*3600\\+)?" // hours? - + "(\\d+)\\*60\\+" // minutes - + "(\\d+)" // seconds - + "\\)"); - - @SafeVarargs - private static T coalesce(T... args) { - for (T arg : args) { - if (arg != null) return arg; - } - throw new IllegalArgumentException("all arguments to coalesce() were null"); - } - - private String parseHtmlAndGetFullLinks(String descriptionHtml) - throws MalformedURLException, UnsupportedEncodingException, ParsingException { - final Document description = Jsoup.parse(descriptionHtml, getUrl()); - for(Element a : description.select("a")) { - final String rawUrl = a.attr("abs:href"); - final URL redirectLink = new URL(rawUrl); - - final Matcher onClickTimestamp; - final String queryString; - if ((onClickTimestamp = DESCRIPTION_TIMESTAMP_ONCLICK_REGEX.matcher(a.attr("onclick"))) - .find()) { - a.removeAttr("onclick"); - - String hours = coalesce(onClickTimestamp.group(1), "0"); - String minutes = onClickTimestamp.group(2); - String seconds = onClickTimestamp.group(3); - - int timestamp = 0; - timestamp += Integer.parseInt(hours) * 3600; - timestamp += Integer.parseInt(minutes) * 60; - timestamp += Integer.parseInt(seconds); - - String setTimestamp = "&t=" + timestamp; - - // Even after clicking https://youtu.be/...?t=6, - // getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=. - a.attr("href", getUrl() + setTimestamp); - - } else if((queryString = redirectLink.getQuery()) != null) { - // if the query string is null we are not dealing with a redirect link, - // so we don't need to override it. - final String link = - Parser.compatParseMap(queryString).get("q"); - - if(link != null) { - // if link is null the a tag is a hashtag. - // They refer to the youtube search. We do not handle them. - a.text(link); - a.attr("href", link); - } else if(redirectLink.toString().contains("https://www.youtube.com/")) { - a.text(redirectLink.toString()); - a.attr("href", redirectLink.toString()); - } - } else if(redirectLink.toString().contains("https://www.youtube.com/")) { - descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString()); - a.text(redirectLink.toString()); - a.attr("href", redirectLink.toString()); - } - } - return description.select("body").first().html(); - } - @Override - public int getAgeLimit() throws ParsingException { - assertPageFetched(); - if (!isAgeRestricted) { - return NO_AGE_LIMIT; - } - try { - return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]") - .attr(CONTENT).replace("+", "")); - } catch (Exception e) { - throw new ParsingException("Could not get age restriction"); - } + public int getAgeLimit() { + if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet"); + + return ageLimit; } @Override @@ -316,68 +264,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getViewCount() throws ParsingException { assertPageFetched(); + String views = null; + try { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { - return getLiveStreamWatchingCount(); - } else { - return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount")); - } - } catch (Exception e) { + views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") + .getObject("videoViewCountRenderer").getObject("viewCount")); + } catch (Exception ignored) {} + + if (views == null) { try { - return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT)); - } catch (Exception ignored) { - throw new ParsingException("Could not get view count", e); - } - } - } + views = playerResponse.getObject("videoDetails").getString("viewCount"); + } catch (Exception ignored) {} - private long getLiveStreamWatchingCount() throws ExtractionException, IOException, JsonParserException { - // https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key= - String innerTubeKey = null, clientVersion = null; - if (playerArgs != null && !playerArgs.isEmpty()) { - innerTubeKey = playerArgs.getString("innertube_api_key"); - clientVersion = playerArgs.getString("innertube_context_client_version"); - } else if (!videoInfoPage.isEmpty()) { - innerTubeKey = videoInfoPage.get("innertube_api_key"); - clientVersion = videoInfoPage.get("innertube_context_client_version"); + if (views == null) throw new ParsingException("Could not get view count"); } - if (innerTubeKey == null || innerTubeKey.isEmpty()) { - throw new ExtractionException("Couldn't get innerTube key"); - } - - if (clientVersion == null || clientVersion.isEmpty()) { - throw new ExtractionException("Couldn't get innerTube client version"); - } - - final String metadataUrl = "https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=" + innerTubeKey; - final byte[] dataBody = ("{\"context\":{\"client\":{\"clientName\":1,\"clientVersion\":\"" + clientVersion + "\"}}" + - ",\"videoId\":\"" + getId() + "\"}").getBytes("UTF-8"); - final Response response = getDownloader().execute(Request.newBuilder() - .post(metadataUrl, dataBody) - .addHeader("Content-Type", "application/json") - .build()); - final JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - - for (Object actionEntry : jsonObject.getArray("actions")) { - if (!(actionEntry instanceof JsonObject)) continue; - final JsonObject entry = (JsonObject) actionEntry; - - final JsonObject updateViewershipAction = entry.getObject("updateViewershipAction", null); - if (updateViewershipAction == null) continue; - - final JsonArray viewCountRuns = JsonUtils.getArray(updateViewershipAction, "viewership.videoViewCountRenderer.viewCount.runs"); - if (viewCountRuns.isEmpty()) continue; - - final JsonObject textObject = viewCountRuns.getObject(0); - if (!textObject.has("text")) { - throw new ExtractionException("Response don't have \"text\" element"); - } - - return Long.parseLong(Utils.removeNonDigitCharacters(textObject.getString("text"))); - } - - throw new ExtractionException("Could not find correct results in response"); + return Long.parseLong(Utils.removeNonDigitCharacters(views)); } @Override @@ -385,9 +287,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String likesString = ""; try { - Element button = doc.select("button.like-button-renderer-like-button").first(); try { - likesString = button.select("span.yt-uix-button-content").first().text(); + likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar") + .getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -408,9 +310,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String dislikesString = ""; try { - Element button = doc.select("button.like-button-renderer-dislike-button").first(); try { - dislikesString = button.select("span.yt-uix-button-content").first().text(); + dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar") + .getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -431,59 +333,52 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { assertPageFetched(); try { - return "https://www.youtube.com/channel/" + - playerResponse.getObject("videoDetails").getString("channelId"); - } catch (Exception e) { - String uploaderUrl = null; - try { - uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children() - .select("a").first().attr("abs:href"); - } catch (Exception ignored) {} - - if (uploaderUrl == null) { - throw new ParsingException("Could not get channel link", e); - } - return uploaderUrl; - } + String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer() + .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint")); + if (uploaderUrl != null) return uploaderUrl; + } catch (Exception ignored) {} + try { + String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); + if (uploaderId != null) + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); + } catch (Exception ignored) {} + throw new ParsingException("Could not get uploader url"); } @Nonnull @Override public String getUploaderName() throws ParsingException { assertPageFetched(); + String uploaderName = null; + try { - return playerResponse.getObject("videoDetails").getString("author"); - } catch (Exception e) { - String name = null; + uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") + .getObject("videoOwnerRenderer").getObject("title")); + } catch (Exception ignored) {} + + if (uploaderName == null) { try { - name = doc.select("div.yt-user-info").first().text(); + uploaderName = playerResponse.getObject("videoDetails").getString("author"); } catch (Exception ignored) {} - if (name == null) { - throw new ParsingException("Could not get uploader name"); - } - return name; + if (uploaderName == null) throw new ParsingException("Could not get uploader name"); } + + return uploaderName; } @Nonnull @Override public String getUploaderAvatarUrl() throws ParsingException { assertPageFetched(); - - String uploaderAvatarUrl = null; try { - uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first() - .select("img").first() - .attr("abs:data-thumb"); - } catch (Exception e) {//todo: add fallback method + String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { throw new ParsingException("Could not get uploader avatar url", e); } - - if (uploaderAvatarUrl == null) { - throw new ParsingException("Could not get uploader avatar url"); - } - return uploaderAvatarUrl; } @Nonnull @@ -592,13 +487,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitlesDefault() { return getSubtitles(MediaFormat.TTML); } @Override @Nonnull - public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { + public List getSubtitles(final MediaFormat format) { assertPageFetched(); List subtitles = new ArrayList<>(); for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) { @@ -622,18 +517,28 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItem getNextStream() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() throws ExtractionException { assertPageFetched(); - try { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - Elements watch = doc.select("div[class=\"watch-sidebar-section\"]"); - if (watch.size() < 1) { - return null;// prevent the snackbar notification "report error" on age-restricted videos + if (getAgeLimit() != NO_AGE_LIMIT) return null; + + try { + final JsonObject firstWatchNextItem = initialData.getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("secondaryResults") + .getObject("secondaryResults").getArray("results").getObject(0); + + if (!firstWatchNextItem.has("compactAutoplayRenderer")) { + // there is no "next" stream + return null; } - collector.commit(extractVideoPreviewInfo(watch.first().select("li").first(), timeAgoParser)); + final JsonObject videoInfo = firstWatchNextItem.getObject("compactAutoplayRenderer") + .getArray("contents").getObject(0).getObject("compactVideoRenderer"); + + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); return collector.getItems().get(0); } catch (Exception e) { throw new ParsingException("Could not get next video", e); @@ -641,20 +546,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); + + if (getAgeLimit() != NO_AGE_LIMIT) return null; + try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("secondaryResults").getObject("secondaryResults").getArray("results"); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); - Element ul = doc.select("ul[id=\"watch-related\"]").first(); - if (ul != null) { - for (Element li : ul.children()) { - // first check if we have a playlist. If so leave them out - if (li.select("a[class*=\"content-link\"]").first() != null) { - collector.commit(extractVideoPreviewInfo(li, timeAgoParser)); - } - } + for (Object ul : results) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer"); + + if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); } return collector; } catch (Exception e) { @@ -667,23 +574,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public String getErrorMessage() { - StringBuilder errorReason; - Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first(); - - if (errorElement == null) { - errorReason = null; - } else { - String errorMessage = errorElement.text(); - if (errorMessage == null || errorMessage.isEmpty()) { - errorReason = null; - } else { - errorReason = new StringBuilder(errorMessage); - errorReason.append(" "); - errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); - } + try { + return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus") + .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason")); + } catch (ParsingException e) { + return null; } - - return errorReason != null ? errorReason.toString() : ""; } /*////////////////////////////////////////////////////////////////////////// @@ -693,11 +589,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; - private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; - private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; - private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 = @@ -709,32 +602,41 @@ public class YoutubeStreamExtractor extends StreamExtractor { private volatile String decryptionCode = ""; - private String pageHtml = null; - @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; - final Response response = downloader.get(verifiedUrl, getExtractorLocalization()); - pageHtml = response.responseBody(); - doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response); + final String url = getUrl() + "&pbj=1"; + + initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); final String playerUrl; - // Check if the video is age restricted - if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) { + + if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos + initialData = initialAjaxJson.getObject(2).getObject("response"); + ageLimit = 18; + final EmbeddedInfo info = getEmbeddedInfo(); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); playerUrl = info.url; - isAgeRestricted = true; } else { - final JsonObject ytPlayerConfig = getPlayerConfig(); - playerArgs = getPlayerArgs(ytPlayerConfig); - playerUrl = getPlayerUrl(ytPlayerConfig); - isAgeRestricted = false; + initialData = initialAjaxJson.getObject(3).getObject("response"); + ageLimit = NO_AGE_LIMIT; + + playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player")); + playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player")); } + playerResponse = getPlayerResponse(); + final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY); + final String status = playabilityStatus.getString("status"); + // If status exist, and is not "OK", throw a ContentNotAvailableException with the reason. + if (status != null && !status.toLowerCase().equals("ok")) { + final String reason = playabilityStatus.getString("reason"); + throw new ContentNotAvailableException("Got error: \"" + reason + "\""); + } + if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); } @@ -744,23 +646,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JsonObject getPlayerConfig() throws ParsingException { - try { - String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml); - return JsonParser.object().from(ytPlayerConfigRaw); - } catch (Parser.RegexException e) { - String errorReason = getErrorMessage(); - switch (errorReason) { - case "": - throw new ContentNotAvailableException("Content not available: player config empty", e); - default: - throw new ContentNotAvailableException("Content not available", e); - } - } catch (Exception e) { - throw new ParsingException("Could not parse yt player config", e); - } - } - private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException { JsonObject playerArgs; @@ -910,9 +795,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Nonnull - private List getAvailableSubtitlesInfo() throws SubtitlesException { + private List getAvailableSubtitlesInfo() { // If the video is age restricted getPlayerConfig will fail - if (isAgeRestricted) return Collections.emptyList(); + if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList(); final JsonObject captions; if (!playerResponse.has("captions")) { @@ -924,7 +809,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject()); final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray()); // todo: use this to apply auto translation to different language from a source language - final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray()); +// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray()); // This check is necessary since there may be cases where subtitles metadata do not contain caption track info // e.g. https://www.youtube.com/watch?v=-Vpwatutnko @@ -981,6 +866,50 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Utils //////////////////////////////////////////////////////////////////////////*/ + private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { + if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer; + + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("results").getObject("results").getArray("contents"); + JsonObject videoPrimaryInfoRenderer = null; + + for (Object content : contents) { + if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) { + videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); + break; + } + } + + if (videoPrimaryInfoRenderer == null) { + throw new ParsingException("Could not find videoPrimaryInfoRenderer"); + } + + this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer; + return videoPrimaryInfoRenderer; + } + + private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { + if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer; + + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("results").getObject("results").getArray("contents"); + JsonObject videoSecondaryInfoRenderer = null; + + for (Object content : contents) { + if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) { + videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer"); + break; + } + } + + if (videoSecondaryInfoRenderer == null) { + throw new ParsingException("Could not find videoSecondaryInfoRenderer"); + } + + this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer; + return videoSecondaryInfoRenderer; + } + @Nonnull private static String getVideoInfoUrl(final String id, final String sts) { return "https://www.youtube.com/get_video_info?" + "video_id=" + id + @@ -1015,123 +944,98 @@ public class YoutubeStreamExtractor extends StreamExtractor { urlAndItags.put(streamUrl, itagItem); } - } catch (UnsupportedEncodingException ignored) { - - } + } catch (UnsupportedEncodingException ignored) {} } } return urlAndItags; } - /** - * Provides information about links to other videos on the video page, such as related videos. - * This is encapsulated in a StreamInfoItem object, which is a subset of the fields in a full StreamInfo. - */ - private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li, final TimeAgoParser timeAgoParser) { - return new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - - @Override - public String getUrl() throws ParsingException { - return li.select("a.content-link").first().attr("abs:href"); - } - - @Override - public String getName() throws ParsingException { - //todo: check NullPointerException causing - return li.select("span.title").first().text(); - //this page causes the NullPointerException, after finding it by searching for "tjvg": - //https://www.youtube.com/watch?v=Uqg0aEhLFAg - } - - @Override - public String getUploaderName() throws ParsingException { - return li.select("span[class*=\"attribution\"").first() - .select("span").first().text(); - } - - @Override - public String getUploaderUrl() throws ParsingException { - return ""; // The uploader is not linked - } - - @Override - public String getTextualUploadDate() throws ParsingException { - return ""; - } - - @Override - public String getThumbnailUrl() throws ParsingException { - Element img = li.select("img").first(); - String thumbnailUrl = img.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we caught such an item. - if (thumbnailUrl.contains(".gif")) { - thumbnailUrl = img.attr("data-thumb"); + @Nonnull + @Override + public List getFrames() throws ExtractionException { + try { + JsonObject jo = initialAjaxJson.getObject(2).getObject("player"); + final String resp = jo.getObject("args").getString("player_response"); + jo = JsonParser.object().from(resp); + final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); + final String url = spec[0]; + final ArrayList result = new ArrayList<>(spec.length - 1); + for (int i = 1; i < spec.length; ++i) { + final String[] parts = spec[i].split("#"); + if (parts.length != 8) { + continue; } - if (thumbnailUrl.startsWith("//")) { - thumbnailUrl = HTTPS + thumbnailUrl; + final int frameWidth = Integer.parseInt(parts[0]); + final int frameHeight = Integer.parseInt(parts[1]); + final int totalCount = Integer.parseInt(parts[2]); + final int framesPerPageX = Integer.parseInt(parts[3]); + final int framesPerPageY = Integer.parseInt(parts[4]); + final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; + final List urls; + if (baseUrl.contains("$M")) { + final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); + urls = new ArrayList<>(totalPages); + for (int j = 0; j < totalPages; j++) { + urls.add(baseUrl.replace("$M", String.valueOf(j))); + } + } else { + urls = Collections.singletonList(baseUrl); } - return thumbnailUrl; + result.add(new Frameset( + urls, + frameWidth, + frameHeight, + totalCount, + framesPerPageX, + framesPerPageY + )); } - }; + result.trimToSize(); + return result; + } catch (Exception e) { + throw new ExtractionException(e); + } } - @Nonnull - @Override - public List getFrames() throws ExtractionException { - try { - final String script = doc.select("#player-api").first().siblingElements().select("script").html(); - int p = script.indexOf("ytplayer.config"); - if (p == -1) { - return Collections.emptyList(); - } - p = script.indexOf('{', p); - int e = script.indexOf("ytplayer.load", p); - if (e == -1) { - return Collections.emptyList(); - } - JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); - final String resp = jo.getObject("args").getString("player_response"); - jo = JsonParser.object().from(resp); - final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); - final String url = spec[0]; - final ArrayList result = new ArrayList<>(spec.length - 1); - for (int i = 1; i < spec.length; ++i) { - final String[] parts = spec[i].split("#"); - if (parts.length != 8) { - continue; - } - final int frameWidth = Integer.parseInt(parts[0]); - final int frameHeight = Integer.parseInt(parts[1]); - final int totalCount = Integer.parseInt(parts[2]); - final int framesPerPageX = Integer.parseInt(parts[3]); - final int framesPerPageY = Integer.parseInt(parts[4]); - final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; - final List urls; - if (baseUrl.contains("$M")) { - final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); - urls = new ArrayList<>(totalPages); - for (int j = 0; j < totalPages; j++) { - urls.add(baseUrl.replace("$M", String.valueOf(j))); - } - } else { - urls = Collections.singletonList(baseUrl); - } - result.add(new Frameset( - urls, - frameWidth, - frameHeight, - totalCount, - framesPerPageX, - framesPerPageY - )); - } - result.trimToSize(); - return result; - } catch (Exception e) { - throw new ExtractionException(e); - } - } + @Nonnull + @Override + public String getHost() { + return ""; + } + + @Nonnull + @Override + public String getPrivacy() { + return ""; + } + + @Nonnull + @Override + public String getCategory() { + return ""; + } + + @Nonnull + @Override + public String getLicence() { + return ""; + } + + @Override + public Locale getLanguageInfo() { + return null; + } + + @Nonnull + @Override + public List getTags() { + return new ArrayList<>(); + } + + @Nonnull + @Override + public String getSupportInfo() { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 514b4fb0..2330130e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,11 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; @@ -15,6 +16,8 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; + /* * Copyright (C) Christian Schabesberger 2016 * YoutubeStreamInfoItemExtractor.java is part of NewPipe. @@ -35,111 +38,149 @@ import java.util.Date; public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - private final Element item; + private JsonObject videoInfo; private final TimeAgoParser timeAgoParser; - - private String cachedUploadDate; + private StreamType cachedStreamType; /** * Creates an extractor of StreamInfoItems from a YouTube page. - * @param item The page element + * + * @param videoInfoItem The JSON page element * @param timeAgoParser A parser of the textual dates or {@code null}. */ - public YoutubeStreamInfoItemExtractor(Element item, @Nullable TimeAgoParser timeAgoParser) { - this.item = item; + public YoutubeStreamInfoItemExtractor(JsonObject videoInfoItem, @Nullable TimeAgoParser timeAgoParser) { + this.videoInfo = videoInfoItem; this.timeAgoParser = timeAgoParser; } @Override - public StreamType getStreamType() throws ParsingException { - if (isLiveStream(item)) { - return StreamType.LIVE_STREAM; - } else { - return StreamType.VIDEO_STREAM; + public StreamType getStreamType() { + if (cachedStreamType != null) { + return cachedStreamType; } + + try { + JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { + return cachedStreamType = StreamType.LIVE_STREAM; + } + } + + } catch (Exception ignored) {} + + try { + final String style = videoInfo.getArray("thumbnailOverlays").getObject(0) + .getObject("thumbnailOverlayTimeStatusRenderer").getString("style"); + if (style.equalsIgnoreCase("LIVE")) { + return cachedStreamType = StreamType.LIVE_STREAM; + } + } catch (Exception ignored) {} + + return cachedStreamType = StreamType.VIDEO_STREAM; } @Override public boolean isAd() throws ParsingException { - return !item.select("span[class*=\"icon-not-available\"]").isEmpty() - || !item.select("span[class*=\"yt-badge-ad\"]").isEmpty() - || isPremiumVideo(); - } - - private boolean isPremiumVideo() { - Element premiumSpan = item.select("span[class=\"standalone-collection-badge-renderer-red-text\"]").first(); - if(premiumSpan == null) return false; - - // if this span has text it most likely says ("Free Video") so we can play this - if(premiumSpan.hasText()) return false; - return true; + return isPremium() || getName().equals("[Private video]") || getName().equals("[Deleted video]"); } @Override public String getUrl() throws ParsingException { try { - Element el = item.select("div[class*=\"yt-lockup-video\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); + String videoId = videoInfo.getString("videoId"); + return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); + throw new ParsingException("Could not get url", e); } } @Override public String getName() throws ParsingException { - try { - Element el = item.select("div[class*=\"yt-lockup-video\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } + String name = getTextFromObject(videoInfo.getObject("title")); + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); } @Override public long getDuration() throws ParsingException { - try { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - - final Element duration = item.select("span[class*=\"video-time\"]").first(); - // apparently on youtube, video-time element will not show up if the video has a duration of 00:00 - // see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf - return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text()); - } catch (Exception e) { - throw new ParsingException("Could not get Duration: " + getUrl(), e); + if (getStreamType() == StreamType.LIVE_STREAM || isPremiere()) { + return -1; } + + String duration = null; + + try { + duration = getTextFromObject(videoInfo.getObject("lengthText")); + } catch (Exception ignored) {} + + if (duration == null) { + try { + for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { + if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) { + duration = getTextFromObject(((JsonObject) thumbnailOverlay) + .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); + } + } + } catch (Exception ignored) {} + + if (duration == null) throw new ParsingException("Could not get duration"); + } + + return YoutubeParsingHelper.parseDurationString(duration); } @Override public String getUploaderName() throws ParsingException { + String name = null; + try { - return item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .text(); - } catch (Exception e) { - throw new ParsingException("Could not get uploader", e); + name = getTextFromObject(videoInfo.getObject("longBylineText")); + } catch (Exception ignored) {} + + if (name == null) { + try { + name = getTextFromObject(videoInfo.getObject("ownerText")); + } catch (Exception ignored) {} + + if (name == null) { + try { + name = getTextFromObject(videoInfo.getObject("shortBylineText")); + } catch (Exception ignored) {} + + if (name == null) throw new ParsingException("Could not get uploader name"); + } } + + return name; } @Override public String getUploaderUrl() throws ParsingException { - // this url is not always in the form "/channel/..." - // sometimes Youtube provides urls in the from "/user/..." - try { - try { - return item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .attr("abs:href"); - } catch (Exception e){} + String url = null; - // try this if the first didn't work - return item.select("span[class=\"title\"") - .text().split(" - ")[0]; - } catch (Exception e) { - System.out.println(item.html()); - throw new ParsingException("Could not get uploader url", e); + try { + url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); + } catch (Exception ignored) {} + + if (url == null) { + try { + url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); + } catch (Exception ignored) {} + + if (url == null) { + try { + url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); + } catch (Exception ignored) {} + + if (url == null) throw new ParsingException("Could not get uploader url"); + } } + + return url; } @Nullable @@ -149,29 +190,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return null; } - if (cachedUploadDate != null) { - return cachedUploadDate; + if (isPremiere()) { + final Date date = getDateFromPremiere().getTime(); + return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date); } try { - if (isVideoReminder()) { - final Calendar calendar = getDateFromReminder(); - if (calendar != null) { - return cachedUploadDate = new SimpleDateFormat("yyyy-MM-dd HH:mm") - .format(calendar.getTime()); - } - } - - - Element meta = item.select("div[class=\"yt-lockup-meta\"]").first(); - if (meta == null) return ""; - - final Elements li = meta.select("li"); - if (li.isEmpty()) return ""; - - return cachedUploadDate = li.first().text(); + return getTextFromObject(videoInfo.getObject("publishedTimeText")); } catch (Exception e) { - throw new ParsingException("Could not get upload date", e); + // upload date is not always available, e.g. in playlists + return null; } } @@ -182,115 +210,89 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return null; } - if (isVideoReminder()) { - return new DateWrapper(getDateFromReminder()); + if (isPremiere()) { + return new DateWrapper(getDateFromPremiere()); } - String textualUploadDate = getTextualUploadDate(); + final String textualUploadDate = getTextualUploadDate(); if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) { - return timeAgoParser.parse(textualUploadDate); - } else { - return null; + try { + return timeAgoParser.parse(textualUploadDate); + } catch (ParsingException e) { + throw new ParsingException("Could not get upload date", e); + } } + return null; } @Override public long getViewCount() throws ParsingException { - String input; - - final Element spanViewCount = item.select("span.view-count").first(); - if (spanViewCount != null) { - input = spanViewCount.text(); - - } else if (getStreamType().equals(StreamType.LIVE_STREAM)) { - Element meta = item.select("ul.yt-lockup-meta-info").first(); - if (meta == null) return 0; - - final Elements li = meta.select("li"); - if (li.isEmpty()) return 0; - - input = li.first().text(); - } else { - try { - Element meta = item.select("div.yt-lockup-meta").first(); - if (meta == null) return -1; - - // This case can happen if google releases a special video - if (meta.select("li").size() < 2) return -1; - - input = meta.select("li").get(1).text(); - } catch (IndexOutOfBoundsException e) { - throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e); - } - } - - if (input == null) { - throw new ParsingException("Input is null"); - } - try { - - return Long.parseLong(Utils.removeNonDigitCharacters(input)); - } catch (NumberFormatException e) { - // if this happens the video probably has no views - if (!input.isEmpty()){ - return 0; + if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { + return -1; } - throw new ParsingException("Could not handle input: " + input, e); + final JsonObject viewCountObject = videoInfo.getObject("viewCountText"); + if (viewCountObject == null) { + // This object is null when a video has its views hidden. + return -1; + } + + final String viewCount = getTextFromObject(viewCountObject); + + if (viewCount.toLowerCase().contains("no views")) { + return 0; + } else if (viewCount.toLowerCase().contains("recommended")) { + return -1; + } + + return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); + } catch (Exception e) { + throw new ParsingException("Could not get view count", e); } } @Override public String getThumbnailUrl() throws ParsingException { try { - String url; - Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + String url = videoInfo.getObject("thumbnail").getArray("thumbnails") + .getObject(0).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } } - - private boolean isVideoReminder() { - return !item.select("span.yt-uix-livereminder").isEmpty(); - } - - private Calendar getDateFromReminder() throws ParsingException { - final Element timeFuture = item.select("span.yt-badge.localized-date").first(); - - if (timeFuture == null) { - throw new ParsingException("Span timeFuture is null"); - } - - final String timestamp = timeFuture.attr("data-timestamp"); - if (!timestamp.isEmpty()) { - try { - final Calendar calendar = Calendar.getInstance(); - calendar.setTime(new Date(Long.parseLong(timestamp) * 1000L)); - return calendar; - } catch (Exception e) { - throw new ParsingException("Could not parse = \"" + timestamp + "\""); + private boolean isPremium() { + try { + JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("Premium")) { + return true; + } } + } catch (Exception ignored) { } - - throw new ParsingException("Could not parse date from reminder element: \"" + timeFuture + "\""); + return false; } - /** - * Generic method that checks if the element contains any clues that it's a livestream item - */ - protected static boolean isLiveStream(Element item) { - return !item.select("span[class*=\"yt-badge-live\"]").isEmpty() - || !item.select("span[class*=\"video-time-overlay-live\"]").isEmpty(); + private boolean isPremiere() { + return videoInfo.has("upcomingEventData"); + } + + private Calendar getDateFromPremiere() throws ParsingException { + final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData"); + final String startTime = upcomingEventData.getString("startTime"); + + try { + final long startTimeTimestamp = Long.parseLong(startTime); + final Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date(startTimeTimestamp * 1000L)); + return calendar; + } catch (Exception e) { + throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index d41d90c5..8ed35c19 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -3,9 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; @@ -57,12 +57,12 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { String response = dl.get(url, getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" - response = response.substring(3, response.length()-1); + response = response.substring(3, response.length() - 1); try { JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray()); for (Object suggestion : collection) { if (!(suggestion instanceof JsonArray)) continue; - String suggestionStr = ((JsonArray)suggestion).getString(0); + String suggestionStr = ((JsonArray) suggestion).getString(0); if (suggestionStr == null) continue; suggestions.add(suggestionStr); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index a4014191..e15463b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -20,27 +20,28 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; * along with NewPipe. If not, see . */ -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; -public class YoutubeTrendingExtractor extends KioskExtractor { +import javax.annotation.Nonnull; - private Document doc; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + +public class YoutubeTrendingExtractor extends KioskExtractor { + private JsonObject initialData; public YoutubeTrendingExtractor(StreamingService service, ListLinkHandler linkHandler, @@ -50,11 +51,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + - "?gl=" + getExtractorContentCountry().getCountryCode(); + final String url = getUrl() + "?pbj=1&gl=" + + getExtractorContentCountry().getCountryCode(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); } @Override @@ -70,99 +72,39 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Nonnull @Override public String getName() throws ParsingException { + String name; try { - Element a = doc.select("a[href*=\"/feed/trending\"]").first(); - Element span = a.select("span[class*=\"display-name\"]").first(); - Element nameSpan = span.select("span").first(); - return nameSpan.text(); + name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get Trending name", e); } + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get Trending name"); } @Nonnull @Override - public InfoItemsPage getInitialPage() throws ParsingException { + public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]"); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); + JsonArray itemSectionRenderers = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents"); - for(Element ul : uls) { - for(final Element li : ul.children()) { - final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - try { - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getUploaderUrl() throws ParsingException { - try { - String link = getUploaderLink().attr("abs:href"); - if (link.isEmpty()) { - throw new IllegalArgumentException("is empty"); - } - return link; - } catch (Exception e) { - throw new ParsingException("Could not get Uploader name"); - } - } - - private Element getUploaderLink() { - // this url is not always in the form "/channel/..." - // sometimes Youtube provides urls in the from "/user/..." - Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first(); - return uploaderEl.select("a").first(); - } - - @Override - public String getUploaderName() throws ParsingException { - try { - return getUploaderLink().text(); - } catch (Exception e) { - throw new ParsingException("Could not get Uploader name"); - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - String url; - Element te = li.select("span[class=\"yt-thumb-simple\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); + for (Object itemSectionRenderer : itemSectionRenderers) { + JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") + .getObject("expandedShelfContentsRenderer"); + if (expandedShelfContentsRenderer != null) { + for (Object ul : expandedShelfContentsRenderer.getArray("items")) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); + } } } - return new InfoItemsPage<>(collector, getNextPageUrl()); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index eb34cf06..77eaf069 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -35,6 +35,14 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { return instance; } + /** + * Returns URL to channel from an ID + * + * @param id Channel ID including e.g. 'channel/' + * @param contentFilters + * @param searchFilter + * @return URL to channel + */ @Override public String getUrl(String id, List contentFilters, String searchFilter) { return "https://www.youtube.com/" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java index 421fc13f..43baf6a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java @@ -1,7 +1,10 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; + import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import java.util.List; @@ -14,6 +17,15 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { return instance; } + @Override + public ListLinkHandler fromUrl(String url) throws ParsingException { + if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){ + return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL); + } else { + return super.fromUrl(url); + } + } + @Override public String getUrl(String id) { return "https://m.youtube.com/watch?v=" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 65ec7e3f..96532555 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -1,17 +1,32 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; +import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URL; +import java.net.URLDecoder; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; +import java.util.*; + +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; /* * Created by Christian Schabesberger on 02.03.16. @@ -38,6 +53,15 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + /** + * The official youtube app supports intents in this format, where after the ':' is the videoId. + * Accordingly there are other apps sharing streams in this format. + */ + public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; + + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static String clientVersion; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -143,4 +167,245 @@ public class YoutubeParsingHelper { uploadDate.setTime(date); return uploadDate; } + + public static JsonObject getInitialData(String html) throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } + } + + public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException { + final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers).responseBody(); + + return response.length() > 50; // ensure to have a valid response + } + + /** + * Get the client version from a page + * @return + * @throws ParsingException + */ + public static String getClientVersion() throws IOException, ExtractionException { + if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; + + if (isHardcodedClientVersionValid()) { + clientVersion = HARDCODED_CLIENT_VERSION; + return clientVersion; + } + + final String url = "https://www.youtube.com/results?search_query=test"; + final String html = getDownloader().get(url).responseBody(); + JsonObject initialData = getInitialData(html); + JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + String shortClientVersion = null; + + // try to get version from initial data first + for (Object service : serviceTrackingParams) { + JsonObject s = (JsonObject) service; + if (s.getString("service").equals("CSI")) { + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("cver")) { + clientVersion = p.getString("value"); + return clientVersion; + } + } + } else if (s.getString("service").equals("ECATCHER")) { + // fallback to get a shortened client version which does not contain the last two digits + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("client.version")) { + shortClientVersion = p.getString("value"); + } + } + } + } + + String contextClientVersion; + String[] patterns = { + "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", + "innertube_context_client_version\":\"([0-9\\.]+?)\"", + "client.version=([0-9\\.]+)" + }; + for (String pattern : patterns) { + try { + contextClientVersion = Parser.matchGroup1(pattern, html); + if (contextClientVersion != null && !contextClientVersion.isEmpty()) { + clientVersion = contextClientVersion; + return clientVersion; + } + } catch (Exception ignored) { + } + } + + if (shortClientVersion != null) { + clientVersion = shortClientVersion; + return clientVersion; + } + + throw new ParsingException("Could not get client version"); + } + + public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { + if (navigationEndpoint.getObject("urlEndpoint") != null) { + String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + String[] params = internUrl.split("&"); + for (String param : params) { + if (param.split("=")[0].equals("q")) { + String url; + try { + url = URLDecoder.decode(param.split("=")[1], "UTF-8"); + } catch (UnsupportedEncodingException e) { + return null; + } + return url; + } + } + } else if (internUrl.startsWith("http")) { + return internUrl; + } + } else if (navigationEndpoint.getObject("browseEndpoint") != null) { + final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); + final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl"); + final String browseId = browseEndpoint.getString("browseId"); + + // All channel ids are prefixed with UC + if (browseId != null && browseId.startsWith("UC")) { + return "https://www.youtube.com/channel/" + browseId; + } + + if (canonicalBaseUrl != null && !canonicalBaseUrl.isEmpty()) { + return "https://www.youtube.com" + canonicalBaseUrl; + } + + throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")"); + } else if (navigationEndpoint.getObject("watchEndpoint") != null) { + StringBuilder url = new StringBuilder(); + url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + return url.toString(); + } + return null; + } + + public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException { + if (textObject.has("simpleText")) return textObject.getString("simpleText"); + + StringBuilder textBuilder = new StringBuilder(); + for (Object textPart : textObject.getArray("runs")) { + String text = ((JsonObject) textPart).getString("text"); + if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) { + String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) { + textBuilder.append("").append(text).append(""); + continue; + } + } + textBuilder.append(text); + } + + String text = textBuilder.toString(); + + if (html) { + text = text.replaceAll("\\n", "
    "); + text = text.replaceAll(" ", "  "); + } + + return text; + } + + public static String getTextFromObject(JsonObject textObject) throws ParsingException { + return getTextFromObject(textObject, false); + } + + public static String fixThumbnailUrl(String thumbnailUrl) { + if (thumbnailUrl.startsWith("//")) { + thumbnailUrl = thumbnailUrl.substring(2); + } + + if (thumbnailUrl.startsWith(HTTP)) { + thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl); + } else if (!thumbnailUrl.startsWith(HTTPS)) { + thumbnailUrl = "https://" + thumbnailUrl; + } + + return thumbnailUrl; + } + + public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + final Response response = getDownloader().get(url, headers, localization); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + // Check if the request was redirected to the error page. + final URL latestUrl = new URL(response.latestUrl()); + if (latestUrl.getHost().equalsIgnoreCase("www.youtube.com")) { + final String path = latestUrl.getPath(); + if (path.equalsIgnoreCase("/oops") || path.equalsIgnoreCase("/error")) { + throw new ContentNotAvailableException("Content unavailable"); + } + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + try { + return JsonParser.array().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } + + /** + * Shared alert detection function, multiple endpoints return the error similarly structured. + *

    + * Will check if the object has an alert of the type "ERROR". + * + * @param initialData the object which will be checked if an alert is present + * @throws ContentNotAvailableException if an alert is detected + */ + public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException { + final JsonArray alerts = initialData.getArray("alerts"); + if (alerts != null && !alerts.isEmpty()) { + final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); + final String alertText = alertRenderer.getObject("text").getString("simpleText"); + final String alertType = alertRenderer.getString("type"); + if (alertType.equalsIgnoreCase("ERROR")) { + throw new ContentNotAvailableException("Got error: \"" + alertText + "\""); + } + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 62a0e737..29bbdfdf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -31,7 +31,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { } String path = urlObj.getPath(); - if (!path.equals("/watch" ) && !path.equals("/playlist")) { + if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index ba5ee152..13481b34 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -24,13 +24,13 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { try { final String url = "https://www.youtube.com/results" - + "?q=" + URLEncoder.encode(searchString, CHARSET_UTF_8); + + "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8); - if(contentFilters.size() > 0) { + if (contentFilters.size() > 0) { switch (contentFilters.get(0)) { - case VIDEOS: return url + "&sp=EgIQAVAU"; - case CHANNELS: return url + "&sp=EgIQAlAU"; - case PLAYLISTS: return url + "&sp=EgIQA1AU"; + case VIDEOS: return url + "&sp=EgIQAQ%253D%253D"; + case CHANNELS: return url + "&sp=EgIQAg%253D%253D"; + case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D"; case ALL: default: } @@ -44,7 +44,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, VIDEOS, CHANNELS, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index e3ac4d52..fd5b5b46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -1,7 +1,10 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; + import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -49,6 +52,15 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return id; } + @Override + public LinkHandler fromUrl(String url) throws ParsingException { + if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){ + return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL); + } else { + return super.fromUrl(url); + } + } + @Override public String getUrl(String id) { return "https://www.youtube.com/watch?v=" + id; @@ -190,7 +202,12 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return assertIsID(id); } - break; + String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsID(viewQueryValue); + } + + return assertIsID(path); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java new file mode 100644 index 00000000..b19b8de3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -0,0 +1,31 @@ +package org.schabi.newpipe.extractor.stream; + +import java.io.Serializable; + +public class Description implements Serializable { + + public static final int HTML = 1; + public static final int MARKDOWN = 2; + public static final int PLAIN_TEXT = 3; + public static final Description emptyDescription = new Description("", PLAIN_TEXT); + + private String content; + private int type; + + public Description(String content, int type) { + this.type = type; + if (content == null) { + this.content = ""; + } else { + this.content = content; + } + } + + public String getContent() { + return content; + } + + public int getType() { + return type; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java index 2d4010dd..cc92cd7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java @@ -1,66 +1,64 @@ package org.schabi.newpipe.extractor.stream; -import javax.annotation.Nullable; -import java.util.Collection; import java.util.List; public final class Frameset { - private List urls; - private int frameWidth; - private int frameHeight; - private int totalCount; - private int framesPerPageX; - private int framesPerPageY; + private List urls; + private int frameWidth; + private int frameHeight; + private int totalCount; + private int framesPerPageX; + private int framesPerPageY; - public Frameset(List urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) { - this.urls = urls; - this.totalCount = totalCount; - this.frameWidth = frameWidth; - this.frameHeight = frameHeight; - this.framesPerPageX = framesPerPageX; - this.framesPerPageY = framesPerPageY; - } + public Frameset(List urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) { + this.urls = urls; + this.totalCount = totalCount; + this.frameWidth = frameWidth; + this.frameHeight = frameHeight; + this.framesPerPageX = framesPerPageX; + this.framesPerPageY = framesPerPageY; + } - /** - * @return list of urls to images with frames - */ - public List getUrls() { - return urls; - } + /** + * @return list of urls to images with frames + */ + public List getUrls() { + return urls; + } - /** - * @return total count of frames - */ - public int getTotalCount() { - return totalCount; - } + /** + * @return total count of frames + */ + public int getTotalCount() { + return totalCount; + } - /** - * @return maximum frames count by x - */ - public int getFramesPerPageX() { - return framesPerPageX; - } + /** + * @return maximum frames count by x + */ + public int getFramesPerPageX() { + return framesPerPageX; + } - /** - * @return maximum frames count by y - */ - public int getFramesPerPageY() { - return framesPerPageY; - } + /** + * @return maximum frames count by y + */ + public int getFramesPerPageY() { + return framesPerPageY; + } - /** - * @return width of a one frame, in pixels - */ - public int getFrameWidth() { - return frameWidth; - } + /** + * @return width of a one frame, in pixels + */ + public int getFrameWidth() { + return frameWidth; + } - /** - * @return height of a one frame, in pixels - */ - public int getFrameHeight() { - return frameHeight; - } + /** + * @return height of a one frame, in pixels + */ + public int getFrameHeight() { + return frameHeight; + } } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 1d49e253..b48eeab2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.MediaFormat; + import java.io.Serializable; import java.util.List; -import org.schabi.newpipe.extractor.MediaFormat; - /** * Creates a stream object from url, format and optional torrent url */ @@ -22,19 +22,19 @@ public abstract class Stream implements Serializable { /** * Instantiates a new stream object. * - * @param url the url + * @param url the url * @param format the format */ public Stream(String url, MediaFormat format) { this(url, null, format); } - + /** * Instantiates a new stream object. * - * @param url the url + * @param url the url * @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent - * @param format the format + * @param format the format */ public Stream(String url, String torrentUrl, MediaFormat format) { this.url = url; @@ -76,7 +76,7 @@ public abstract class Stream implements Serializable { public String getUrl() { return url; } - + /** * Gets the torrent url. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index d697dff2..78a5fbf3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -34,6 +34,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Locale; /** * Scrapes information from a video/audio streaming service (eg, YouTube). @@ -75,6 +76,7 @@ public abstract class StreamExtractor extends Extractor { /** * This will return the url to the thumbnail of the stream. Try to return the medium resolution here. + * * @return The url of the thumbnail. * @throws ParsingException */ @@ -82,15 +84,17 @@ public abstract class StreamExtractor extends Extractor { public abstract String getThumbnailUrl() throws ParsingException; /** - * This is the stream description. On YouTube this is the video description. You can return simple HTML here. - * @return The description of the stream/video. + * This is the stream description. + * + * @return The description of the stream/video or Description.emptyDescription if the description is empty. * @throws ParsingException */ @Nonnull - public abstract String getDescription() throws ParsingException; + public abstract Description getDescription() throws ParsingException; /** * Get the age limit. + * * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit * @throws ParsingException if an error occurs while parsing */ @@ -98,6 +102,7 @@ public abstract class StreamExtractor extends Extractor { /** * This should return the length of a video in seconds. + * * @return The length of the stream in seconds. * @throws ParsingException */ @@ -107,6 +112,7 @@ public abstract class StreamExtractor extends Extractor { * If the url you are currently handling contains a time stamp/seek, you can return the * position it represents here. * If the url has no time stamp simply return zero. + * * @return the timestamp in seconds * @throws ParsingException */ @@ -115,22 +121,25 @@ public abstract class StreamExtractor extends Extractor { /** * The count of how many people have watched the video/listened to the audio stream. * If the current stream has no view count or its not available simply return -1 + * * @return amount of views. * @throws ParsingException */ public abstract long getViewCount() throws ParsingException; /** - * The Amount of likes a video/audio stream got. + * The amount of likes a video/audio stream got. * If the current stream has no likes or its not available simply return -1 + * * @return the amount of likes the stream got * @throws ParsingException */ public abstract long getLikeCount() throws ParsingException; /** - * The Amount of dislikes a video/audio stream got. + * The amount of dislikes a video/audio stream got. * If the current stream has no dislikes or its not available simply return -1 + * * @return the amount of likes the stream got * @throws ParsingException */ @@ -142,6 +151,7 @@ public abstract class StreamExtractor extends Extractor { * ChannelExtractor, * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects * this url. + * * @return the url to the page of the creator/uploader of the stream or an empty String * @throws ParsingException */ @@ -151,6 +161,7 @@ public abstract class StreamExtractor extends Extractor { /** * The name of the creator/uploader of the stream. * If the name is not available you can simply return an empty string. + * * @return the name of the creator/uploader of the stream or an empty String * @throws ParsingException */ @@ -160,6 +171,7 @@ public abstract class StreamExtractor extends Extractor { /** * The url to the image file/profile picture/avatar of the creator/uploader of the stream. * If the url is not available you can return an empty String. + * * @return The url of the image file of the uploader or an empty String * @throws ParsingException */ @@ -169,20 +181,24 @@ public abstract class StreamExtractor extends Extractor { /** * Get the dash mpd url. If you don't know what a dash MPD is you can read about it * here. + * * @return the url as a string or an empty string * @throws ParsingException if an error occurs while reading */ - @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + @Nonnull + public abstract String getDashMpdUrl() throws ParsingException; /** * I am not sure if this is in use, and how this is used. However the frontend is missing support * for HLS streams. Prove me if I am wrong. Please open an * issue, * or fix this description if you know whats up with this. + * * @return The Url to the hls stream. * @throws ParsingException */ - @Nonnull public abstract String getHlsUrl() throws ParsingException; + @Nonnull + public abstract String getHlsUrl() throws ParsingException; /** * This should return a list of available @@ -190,6 +206,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of audio only streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -203,6 +220,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of combined video and streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -216,6 +234,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of video and streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -226,6 +245,7 @@ public abstract class StreamExtractor extends Extractor { * This will return a list of available * Subtitless. * If no subtitles are available an empty list can returned. + * * @return a list of available subtitles or an empty list * @throws IOException * @throws ExtractionException @@ -238,6 +258,7 @@ public abstract class StreamExtractor extends Extractor { * Subtitless. * given by a specific type. * If no subtitles in that specific format are available an empty list can returned. + * * @param format the media format by which the subtitles should be filtered * @return a list of available subtitles or an empty list * @throws IOException @@ -248,15 +269,17 @@ public abstract class StreamExtractor extends Extractor { /** * Get the StreamType. + * * @return the type of the stream * @throws ParsingException */ public abstract StreamType getStreamType() throws ParsingException; /** - * should return the url of the next stream. NewPipe will automatically play + * Should return the url of the next stream. NewPipe will automatically play * the next stream if the user wants that. * If the next stream is is not available simply return null + * * @return the InfoItem of the next stream * @throws IOException * @throws ExtractionException @@ -268,7 +291,8 @@ public abstract class StreamExtractor extends Extractor { * streams. If you don't like suggested streams you should implement them anyway since they can * be disabled by the user later in the frontend. * This list MUST NOT contain the next available video as this should be return through getNextStream() - * If is is not available simply return null + * If it is not available simply return null + * * @return a list of InfoItems showing the related videos/streams * @throws IOException * @throws ExtractionException @@ -277,6 +301,7 @@ public abstract class StreamExtractor extends Extractor { /** * Should return a list of Frameset object that contains preview of stream frames + * * @return list of preview frames or empty list if frames preview is not supported or not found for specified stream * @throws IOException * @throws ExtractionException @@ -299,9 +324,10 @@ public abstract class StreamExtractor extends Extractor { /** * Override this function if the format of time stamp in the url is not the same format as that form youtube. - * Honestly I don't even know the time stamp fromat of youtube. + * Honestly I don't even know the time stamp format of YouTube. + * * @param regexPattern - * @return the sime stamp/seek for the video in seconds + * @return the time stamp/seek for the video in seconds * @throws ParsingException */ protected long getTimestampSeconds(String regexPattern) throws ParsingException { @@ -309,10 +335,10 @@ public abstract class StreamExtractor extends Extractor { try { timeStamp = Parser.matchGroup1(regexPattern, getOriginalUrl()); } catch (Parser.RegexException e) { - // catch this instantly since an url does not necessarily have to have a time stamp + // catch this instantly since a url does not necessarily have a timestamp - // -2 because well the testing system will then know its the regex that failed :/ - // not good i know + // -2 because the testing system will consequently know that the regex failed + // not good, I know return -2; } @@ -349,4 +375,81 @@ public abstract class StreamExtractor extends Extractor { return 0; } } + + /** + * The host of the stream (Eg. peertube.cpy.re). + * If the host is not available, or if the service doesn't use + * a federated system, but a centralised system, + * you can simply return an empty string. + * + * @return the host of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getHost() throws ParsingException; + + /** + * The privacy of the stream (Eg. Public, Private, Unlisted…). + * If the privacy is not available you can simply return an empty string. + * + * @return the privacy of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getPrivacy() throws ParsingException; + + /** + * The name of the category of the stream. + * If the category is not available you can simply return an empty string. + * + * @return the category of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getCategory() throws ParsingException; + + /** + * The name of the licence of the stream. + * If the licence is not available you can simply return an empty string. + * + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getLicence() throws ParsingException; + + /** + * The locale language of the stream. + * If the language is not available you can simply return null. + * If the language is provided by a language code, you can return + * new Locale(language_code); + * + * @return the locale language of the stream or null. + * @throws ParsingException + */ + @Nullable + public abstract Locale getLanguageInfo() throws ParsingException; + + /** + * The list of tags of the stream. + * If the tag list is not available you can simply return an empty list. + * + * @return the list of tags of the stream or an empty list. + * @throws ParsingException + */ + @Nonnull + public abstract List getTags() throws ParsingException; + + /** + * The support information of the stream. + * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37 + * (support button). + * If the support information are not available, + * you can simply return an empty String. + * + * @return the support information of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getSupportInfo() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 34ea703c..580f9c21 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; /* * Created by Christian Schabesberger on 26.08.15. @@ -270,6 +271,43 @@ public class StreamInfo extends Info { streamInfo.addError(e); } + //additional info + try { + streamInfo.setHost(extractor.getHost()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setPrivacy(extractor.getPrivacy()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setCategory(extractor.getCategory()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLicence(extractor.getLicence()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLanguageInfo(extractor.getLanguageInfo()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setTags(extractor.getTags()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setSupportInfo(extractor.getSupportInfo()); + } catch (Exception e) { + streamInfo.addError(e); + } + streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -281,7 +319,7 @@ public class StreamInfo extends Info { private DateWrapper uploadDate; private long duration = -1; private int ageLimit = -1; - private String description; + private Description description; private long viewCount = -1; private long likeCount = -1; @@ -308,6 +346,14 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles = new ArrayList<>(); + private String host = ""; + private String privacy = ""; + private String category = ""; + private String licence = ""; + private String support = ""; + private Locale language = null; + private List tags = new ArrayList<>(); + /** * Get the stream type * @@ -371,11 +417,11 @@ public class StreamInfo extends Info { this.ageLimit = ageLimit; } - public String getDescription() { + public Description getDescription() { return description; } - public void setDescription(String description) { + public void setDescription(Description description) { this.description = description; } @@ -533,4 +579,59 @@ public class StreamInfo extends Info { this.subtitles = subtitles; } + public String getHost() { + return this.host; + } + + public void setHost(String str) { + this.host = str; + } + + public String getPrivacy() { + return this.privacy; + } + + public void setPrivacy(String str) { + this.privacy = str; + } + + public String getCategory() { + return this.category; + } + + public void setCategory(String cat) { + this.category = cat; + } + + public String getLicence() { + return this.licence; + } + + public void setLicence(String str) { + this.licence = str; + } + + public Locale getLanguageInfo() { + return this.language; + } + + public void setLanguageInfo(Locale lang) { + this.language = lang; + } + + public List getTags() { + return this.tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public void setSupportInfo(String support) { + this.support = support; + } + + public String getSupportInfo() { + return this.support; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 113f3a87..828d4520 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -31,6 +31,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream type + * * @return the stream type * @throws ParsingException thrown if there is an error in the extraction */ @@ -38,6 +39,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Check if the stream is an ad. + * * @return {@code true} if the stream is an ad. * @throws ParsingException thrown if there is an error in the extraction */ @@ -45,6 +47,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream duration in seconds + * * @return the stream duration in seconds * @throws ParsingException thrown if there is an error in the extraction */ @@ -52,6 +55,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Parses the number of views + * * @return the number of views or -1 for live streams * @throws ParsingException thrown if there is an error in the extraction */ @@ -59,6 +63,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the uploader name + * * @return the uploader name * @throws ParsingException if parsing fails */ @@ -80,9 +85,9 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Extracts the upload date and time of this item and parses it. *

    - * If the service doesn't provide an exact time, an approximation can be returned. - *
    - * If the service doesn't provide any date at all, then {@code null} should be returned. + * If the service doesn't provide an exact time, an approximation can be returned. + *
    + * If the service doesn't provide any date at all, then {@code null} should be returned. *

    * * @return The date and time (can be approximated) this item was uploaded or {@code null}. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 85dce8f0..b81052b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -101,8 +101,8 @@ public class StreamInfoItemsCollector extends InfoItemsCollector getStreamInfoItemList() { List siiList = new Vector<>(); - for(InfoItem ii : super.getItems()) { - if(ii instanceof StreamInfoItem) { + for (InfoItem ii : super.getItems()) { + if (ii instanceof StreamInfoItem) { siiList.add((StreamInfoItem) ii); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java index d0e09ac1..8f16942e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -52,7 +52,7 @@ public class SubtitlesStream extends Stream implements Serializable { @Override public boolean equalStats(Stream cmp) { - return super.equalStats(cmp)&& + return super.equalStats(cmp) && cmp instanceof SubtitlesStream && code.equals(((SubtitlesStream) cmp).code) && autoGenerated == ((SubtitlesStream) cmp).autoGenerated; @@ -67,7 +67,7 @@ public class SubtitlesStream extends Stream implements Serializable { } public Locale getLocale() { - return locale; + return locale; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java index f4531d36..67219240 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java @@ -36,7 +36,7 @@ public class VideoStream extends Stream { this.resolution = resolution; this.isVideoOnly = isVideoOnly; } - + public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) { this(url, torrentUrl, format, resolution, false); } @@ -56,6 +56,7 @@ public class VideoStream extends Stream { /** * Get the video resolution + * * @return the video resolution */ public String getResolution() { @@ -64,8 +65,9 @@ public class VideoStream extends Stream { /** * Check if the video is video only. - * + *

    * Video only streams have no audio + * * @return {@code true} if this stream is vid */ public boolean isVideoOnly() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 7b954a4c..b8b191ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor.utils; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.services.youtube.ItagItem; @@ -109,11 +109,11 @@ public class DashMpdParser { *

    * It has video, video only and audio streams and will only add to the list if it don't * find a similar stream in the respective lists (calling {@link Stream#equalStats}). - * + *

    * Info about dash MPD can be found here - * @see www.brendanlog.com * * @param streamInfo where the parsed streams will be added + * @see www.brendanlog.com */ public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { @@ -160,7 +160,7 @@ public class DashMpdParser { final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - if(segmentationList == null) { + if (segmentationList == null) { final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { audioStreams.add(audioStream); @@ -172,7 +172,7 @@ public class DashMpdParser { } else { boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - if(segmentationList == null) { + if (segmentationList == null) { final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, @@ -191,7 +191,7 @@ public class DashMpdParser { itag.resolutionString, isVideoOnly); - if(isVideoOnly) { + if (isVideoOnly) { segmentedVideoOnlyStreams.add(videoStream); } else { segmentedVideoStreams.add(videoStream); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index 28300649..c6ef73ca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; -import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; @@ -31,7 +30,7 @@ public class ExtractorHelper { public static List getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { try { InfoItemsCollector collector = extractor.getRelatedStreams(); - if(collector == null) return Collections.emptyList(); + if (collector == null) return Collections.emptyList(); info.addAllErrors(collector.getErrors()); //noinspection unchecked @@ -41,5 +40,5 @@ public class ExtractorHelper { return Collections.emptyList(); } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index e5d7bb62..25bb3f6c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -1,76 +1,85 @@ package org.schabi.newpipe.extractor.utils; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; - public class JsonUtils { + public static final JsonObject DEFAULT_EMPTY = new JsonObject(); private JsonUtils() { } - + @Nonnull - public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { List keys = Arrays.asList(path.split("\\.")); object = getObject(object, keys.subList(0, keys.size() - 1)); if (null == object) throw new ParsingException("Unable to get " + path); Object result = object.get(keys.get(keys.size() - 1)); - if(null == result) throw new ParsingException("Unable to get " + path); + if (null == result) throw new ParsingException("Unable to get " + path); return result; } - + @Nonnull - public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof String) { + if (value instanceof String) { return (String) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Number) { + if (value instanceof Boolean) { + return (Boolean) value; + } else { + throw new ParsingException("Unable to get " + path); + } + } + + @Nonnull + public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { + Object value = getValue(object, path); + if (value instanceof Number) { return (Number) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonObject) { + if (value instanceof JsonObject) { return (JsonObject) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonArray) { + if (value instanceof JsonArray) { return (JsonArray) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { - + List result = new ArrayList<>(); for (int i = 0; i < array.size(); i++) { JsonObject obj = array.getObject(i); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index b03e304c..30786420 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -1,5 +1,10 @@ package org.schabi.newpipe.extractor.utils; +import org.nibor.autolink.LinkExtractor; +import org.nibor.autolink.LinkSpan; +import org.nibor.autolink.LinkType; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; @@ -9,11 +14,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.nibor.autolink.LinkExtractor; -import org.nibor.autolink.LinkSpan; -import org.nibor.autolink.LinkType; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - /* * Created by Christian Schabesberger on 02.02.16. * @@ -51,7 +51,7 @@ public class Parser { public static String matchGroup1(String pattern, String input) throws RegexException { return matchGroup(pattern, input, 1); } - + public static String matchGroup1(Pattern pattern, String input) throws RegexException { return matchGroup(pattern, input, 1); } @@ -60,7 +60,7 @@ public class Parser { Pattern pat = Pattern.compile(pattern); return matchGroup(pat, input, group); } - + public static String matchGroup(Pattern pat, String input, int group) throws RegexException { Matcher mat = pat.matcher(input); boolean foundMatch = mat.find(); @@ -102,7 +102,7 @@ public class Parser { .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW)) .build(); Iterable linkss = linkExtractor.extractLinks(txt); - for(LinkSpan ls : linkss) { + for (LinkSpan ls : linkss) { links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex())); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index fd06d40f..76aa2944 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -1,15 +1,18 @@ package org.schabi.newpipe.extractor.utils; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.List; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - public class Utils { + public static final String HTTP = "http://"; + public static final String HTTPS = "https://"; + private Utils() { //no instance } @@ -35,6 +38,7 @@ public class Utils { *
  • 1.23K -> 1230
    • *
  • 1.23M -> 1230000
    • * + * * @param numberWord string to be converted to a long * @return a long * @throws NumberFormatException @@ -82,9 +86,6 @@ public class Utils { } } - private static final String HTTP = "http://"; - private static final String HTTPS = "https://"; - public static String replaceHttpWithHttps(final String url) { if (url == null) return null; @@ -165,17 +166,17 @@ public class Utils { return setsNoPort || usesDefaultPort; } - + public static String removeUTF8BOM(String s) { if (s.startsWith("\uFEFF")) { s = s.substring(1); } if (s.endsWith("\uFEFF")) { - s = s.substring(0, s.length()-1); + s = s.substring(0, s.length() - 1); } return s; } - + public static String getBaseUrl(String url) throws ParsingException { URL uri; try { @@ -185,5 +186,4 @@ public class Utils { } return uri.getProtocol() + "://" + uri.getAuthority(); } - } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java index 948975a0..c93f31b7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java +++ b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java @@ -20,7 +20,7 @@ import java.util.Map; public class DownloaderTestImpl extends Downloader { - private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; + private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0"; private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en"; private static DownloaderTestImpl instance = null; @@ -99,19 +99,25 @@ public class DownloaderTestImpl extends Downloader { final int responseCode = connection.getResponseCode(); final String responseMessage = connection.getResponseMessage(); final Map> responseHeaders = connection.getHeaderFields(); + final String latestUrl = connection.getURL().toString(); - return new Response(responseCode, responseMessage, responseHeaders, response.toString()); + return new Response(responseCode, responseMessage, responseHeaders, response.toString(), latestUrl); } catch (Exception e) { + final int responseCode = connection.getResponseCode(); + /* * HTTP 429 == Too Many Request * Receive from Youtube.com = ReCaptcha challenge request * See : https://github.com/rg3/youtube-dl/issues/5138 */ - if (connection.getResponseCode() == 429) { + if (responseCode == 429) { throw new ReCaptchaException("reCaptcha Challenge requested", url); + } else if (responseCode != -1) { + final String latestUrl = connection.getURL().toString(); + return new Response(responseCode, connection.getResponseMessage(), connection.getHeaderFields(), null, latestUrl); } - throw new IOException(connection.getResponseCode() + " " + connection.getResponseMessage(), e); + throw new IOException("Error occurred while fetching the content", e); } finally { if (outputStream != null) outputStream.close(); if (input != null) input.close(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index 8cf27094..6c2dd9ef 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -2,20 +2,29 @@ package org.schabi.newpipe.extractor.services; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.Calendar; import java.util.List; +import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.*; +import static org.schabi.newpipe.extractor.StreamingService.*; public final class DefaultTests { - public static void defaultTestListOfItems(int expectedServiceId, List itemsList, List errors) { - assertTrue("List of items is empty", !itemsList.isEmpty()); + public static void defaultTestListOfItems(StreamingService expectedService, List itemsList, List errors) throws ParsingException { + assertFalse("List of items is empty", itemsList.isEmpty()); assertFalse("List of items contains a null element", itemsList.contains(null)); - assertEmptyErrors("Errors during stream list extraction", errors); + assertEmptyErrors("Errors during extraction", errors); for (InfoItem item : itemsList) { assertIsSecureUrl(item.getUrl()); @@ -23,12 +32,17 @@ public final class DefaultTests { assertIsSecureUrl(item.getThumbnailUrl()); } assertNotNull("InfoItem type not set: " + item, item.getInfoType()); - assertEquals("Service id doesn't match: " + item, expectedServiceId, item.getServiceId()); + assertEquals("Unexpected item service id", expectedService.getServiceId(), item.getServiceId()); + assertNotEmpty("Item name not set: " + item, item.getName()); if (item instanceof StreamInfoItem) { StreamInfoItem streamInfoItem = (StreamInfoItem) item; assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName()); assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); + assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + + assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); + assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); final String textualUploadDate = streamInfoItem.getTextualUploadDate(); if (textualUploadDate != null && !textualUploadDate.isEmpty()) { @@ -37,34 +51,56 @@ public final class DefaultTests { assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance())); } + } else if (item instanceof ChannelInfoItem) { + final ChannelInfoItem channelInfoItem = (ChannelInfoItem) item; + assertExpectedLinkType(expectedService, channelInfoItem.getUrl(), LinkType.CHANNEL); + + } else if (item instanceof PlaylistInfoItem) { + final PlaylistInfoItem playlistInfoItem = (PlaylistInfoItem) item; + assertExpectedLinkType(expectedService, playlistInfoItem.getUrl(), LinkType.PLAYLIST); } } } - public static ListExtractor.InfoItemsPage defaultTestRelatedItems(ListExtractor extractor, int expectedServiceId) throws Exception { + private static void assertExpectedLinkType(StreamingService expectedService, String url, LinkType expectedLinkType) throws ParsingException { + final LinkType linkTypeByUrl = expectedService.getLinkTypeByUrl(url); + + assertNotEquals("Url is not recognized by its own service: \"" + url + "\"", + LinkType.NONE, linkTypeByUrl); + assertEquals("Service returned wrong link type for: \"" + url + "\"", + expectedLinkType, linkTypeByUrl); + } + + public static void assertNoMoreItems(ListExtractor extractor) throws Exception { + assertFalse("More items available when it shouldn't", extractor.hasNextPage()); + final String nextPageUrl = extractor.getNextPageUrl(); + assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); + } + + public static ListExtractor.InfoItemsPage defaultTestRelatedItems(ListExtractor extractor) throws Exception { final ListExtractor.InfoItemsPage page = extractor.getInitialPage(); final List itemsList = page.getItems(); List errors = page.getErrors(); - defaultTestListOfItems(expectedServiceId, itemsList, errors); + defaultTestListOfItems(extractor.getService(), itemsList, errors); return page; } - public static ListExtractor.InfoItemsPage defaultTestMoreItems(ListExtractor extractor, int expectedServiceId) throws Exception { + public static ListExtractor.InfoItemsPage defaultTestMoreItems(ListExtractor extractor) throws Exception { assertTrue("Doesn't have more items", extractor.hasNextPage()); ListExtractor.InfoItemsPage nextPage = extractor.getPage(extractor.getNextPageUrl()); final List items = nextPage.getItems(); - assertTrue("Next page is empty", !items.isEmpty()); + assertFalse("Next page is empty", items.isEmpty()); assertEmptyErrors("Next page have errors", nextPage.getErrors()); - defaultTestListOfItems(expectedServiceId, nextPage.getItems(), nextPage.getErrors()); + defaultTestListOfItems(extractor.getService(), nextPage.getItems(), nextPage.getErrors()); return nextPage; } - public static void defaultTestGetPageInNewExtractor(ListExtractor extractor, ListExtractor newExtractor, int expectedServiceId) throws Exception { + public static void defaultTestGetPageInNewExtractor(ListExtractor extractor, ListExtractor newExtractor) throws Exception { final String nextPageUrl = extractor.getNextPageUrl(); final ListExtractor.InfoItemsPage page = newExtractor.getPage(nextPageUrl); - defaultTestListOfItems(expectedServiceId, page.getItems(), page.getErrors()); + defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index b9c67c4e..258a55b0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -4,47 +4,86 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import static junit.framework.TestCase.assertEquals; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test {@link MediaCCCConferenceExtractor} */ public class MediaCCCConferenceExtractorTest { - private static ChannelExtractor extractor; - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017"); - extractor.fetchPage(); + public static class FrOSCon2017 { + private static MediaCCCConferenceExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017"); + extractor.fetchPage(); + } + + @Test + public void testName() throws Exception { + assertEquals("FrOSCon 2017", extractor.getName()); + } + + @Test + public void testGetUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); + } + + @Test + public void testGetOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); + } + + @Test + public void testGetThumbnailUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); + } + + @Test + public void testGetInitalPage() throws Exception { + assertEquals(97, extractor.getInitialPage().getItems().size()); + } } - @Test - public void testName() throws Exception { - assertEquals("FrOSCon 2017", extractor.getName()); - } + public static class Oscal2019 { + private static MediaCCCConferenceExtractor extractor; - @Test - public void testGetUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); - } + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19"); + extractor.fetchPage(); + } - @Test - public void testGetOriginalUrl() throws Exception { - assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); - } + @Test + public void testName() throws Exception { + assertEquals("Open Source Conference Albania 2019", extractor.getName()); + } - @Test - public void testGetThumbnailUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); - } + @Test + public void testGetUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/oscal19", extractor.getUrl()); + } - @Test - public void testGetInitalPage() throws Exception { - assertEquals(97,extractor.getInitialPage().getItems().size()); + @Test + public void testGetOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/c/oscal19", extractor.getOriginalUrl()); + } + + @Test + public void testGetThumbnailUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", extractor.getAvatarUrl()); + } + + @Test + public void testGetInitalPage() throws Exception { + assertTrue(extractor.getInitialPage().getItems().size() >= 21); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java index 252e4dbb..90ebf028 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java @@ -24,7 +24,7 @@ public class MediaCCCConferenceListExtractorTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); + extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); extractor.fetchPage(); } @@ -49,8 +49,8 @@ public class MediaCCCConferenceListExtractorTest { } private boolean contains(List itemList, String name) { - for(InfoItem item : itemList) { - if(item.getName().equals(name)) + for (InfoItem item : itemList) { + if (item.getName().equals(name)) return true; } return false; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java index 781f9138..25116af3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java @@ -22,7 +22,7 @@ public class MediaCCCOggTest { public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); + extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); extractor.fetchPage(); } @@ -33,7 +33,7 @@ public class MediaCCCOggTest { @Test public void getAudioStreamsContainOgg() throws Exception { - for(AudioStream stream : extractor.getAudioStreams()) { + for (AudioStream stream : extractor.getAudioStreams()) { assertEquals("OGG", stream.getFormat().toString()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java index dc38338e..cd33c562 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java @@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorAllTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("c3", Arrays.asList(new String[0]), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -37,8 +37,8 @@ public class MediaCCCSearchExtractorAllTest { @Test public void testIfChannelInfoItemsAvailable() { boolean isAvialable = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof ChannelInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof ChannelInfoItem) { isAvialable = true; } } @@ -48,8 +48,8 @@ public class MediaCCCSearchExtractorAllTest { @Test public void testIfStreamInfoitemsAvailable() { boolean isAvialable = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { isAvialable = true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java index 18fbc234..f9ce2334 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java @@ -27,7 +27,7 @@ public class MediaCCCSearchExtractorConferencesTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("c3", Arrays.asList(new String[]{"conferences"}), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -35,7 +35,7 @@ public class MediaCCCSearchExtractorConferencesTest { @Test public void testReturnTypeChannel() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { assertTrue("Item is not of type channel", item instanceof ChannelInfoItem); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java index 1ed65a8d..29f90caf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java @@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorEventsTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("linux", Arrays.asList(new String[]{"events"}), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -65,7 +65,7 @@ public class MediaCCCSearchExtractorEventsTest { @Test public void testReturnTypeStream() throws Exception { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 71b1e779..8120e07a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -6,96 +6,201 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.BaseExtractorTest; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.UtilsTest; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.List; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test {@link MediaCCCStreamExtractor} */ -public class MediaCCCStreamExtractorTest implements BaseExtractorTest { - private static StreamExtractor extractor; +public class MediaCCCStreamExtractorTest { - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); + public static class Gpn18Tmux { + private static MediaCCCStreamExtractor extractor; - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); - extractor.fetchPage(); + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + + extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht"); + extractor.fetchPage(); + } + + @Test + public void testServiceId() throws Exception { + assertEquals(2, extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId()); + } + + @Test + public void testUrl() throws Exception { + assertIsSecureUrl(extractor.getUrl()); + assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws Exception { + assertIsSecureUrl(extractor.getOriginalUrl()); + assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl()); + } + + @Test + public void testThumbnail() throws Exception { + assertIsSecureUrl(extractor.getThumbnailUrl()); + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); + } + + @Test + public void testUploaderName() throws Exception { + assertEquals("gpn18", extractor.getUploaderName()); + } + + @Test + public void testUploaderUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); + } + + @Test + public void testUploaderAvatarUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); + } + + @Test + public void testVideoStreams() throws Exception { + List videoStreamList = extractor.getVideoStreams(); + assertEquals(4, videoStreamList.size()); + for (VideoStream stream : videoStreamList) { + assertIsSecureUrl(stream.getUrl()); + } + } + + @Test + public void testAudioStreams() throws Exception { + List audioStreamList = extractor.getAudioStreams(); + assertEquals(2, audioStreamList.size()); + for (AudioStream stream : audioStreamList) { + assertIsSecureUrl(stream.getUrl()); + } + } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } } - @Override - public void testServiceId() throws Exception { - assertEquals(2, extractor.getServiceId()); - } + public static class _36c3PrivacyMessaging { + private static MediaCCCStreamExtractor extractor; - @Override - public void testName() throws Exception { - assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); - } + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging"); + extractor.fetchPage(); + } - @Override - public void testId() throws Exception { - assertEquals("", extractor.getId()); - } + @Test + public void testName() throws Exception { + assertEquals("What's left for private messaging?", extractor.getName()); + } - @Override - public void testUrl() throws Exception { - assertEquals("", extractor.getUrl()); - } + @Test + public void testId() throws Exception { + assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId()); + } - @Override - public void testOriginalUrl() throws Exception { - assertEquals("", extractor.getOriginalUrl()); - } + @Test + public void testUrl() throws Exception { + assertIsSecureUrl(extractor.getUrl()); + assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); + } - @Test - public void testThumbnail() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); - } + @Test + public void testOriginalUrl() throws Exception { + assertIsSecureUrl(extractor.getOriginalUrl()); + assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl()); + } - @Test - public void testUploaderName() throws Exception { - assertEquals("gpn18", extractor.getUploaderName()); - } + @Test + public void testThumbnail() throws Exception { + assertIsSecureUrl(extractor.getThumbnailUrl()); + assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); + } - @Test - public void testUploaderUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); - } + @Test + public void testUploaderName() throws Exception { + assertEquals("36c3", extractor.getUploaderName()); + } - @Test - public void testUploaderAvatarUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); - } + @Test + public void testUploaderUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); + } - @Test - public void testVideoStreams() throws Exception { - assertEquals(4, extractor.getVideoStreams().size()); - } + @Test + public void testUploaderAvatarUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); + } - @Test - public void testAudioStreams() throws Exception { - assertEquals(2, extractor.getAudioStreams().size()); - } + @Test + public void testVideoStreams() throws Exception { + List videoStreamList = extractor.getVideoStreams(); + assertEquals(8, videoStreamList.size()); + for (VideoStream stream : videoStreamList) { + assertIsSecureUrl(stream.getUrl()); + } + } - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); - } + @Test + public void testAudioStreams() throws Exception { + List audioStreamList = extractor.getAudioStreams(); + assertEquals(2, audioStreamList.size()); + for (AudioStream stream : audioStreamList) { + assertIsSecureUrl(stream.getUrl()); + } + } - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } } -} +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 3d8bdff3..7c7a9b80 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -1,15 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; - import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -20,6 +10,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + /** * Test for {@link PeertubeChannelExtractor} */ @@ -72,12 +68,12 @@ public class PeertubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, PeerTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -131,7 +127,7 @@ public class PeertubeChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -169,12 +165,12 @@ public class PeertubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, PeerTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -204,7 +200,7 @@ public class PeertubeChannelExtractorTest { @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index 633307a2..f9b0c69d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeChannelLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index fef1c3ee..1241b0a8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -1,12 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.io.IOException; -import java.util.List; - import org.jsoup.helper.StringUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -18,6 +11,13 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + public class PeertubeCommentsExtractorTest { private static PeertubeCommentsExtractor extractor; @@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (PeertubeCommentsExtractor) PeerTube - .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); } @Test @@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); assertTrue("Comments".equals(commentsInfo.getName())); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); @@ -59,11 +59,11 @@ public class PeertubeCommentsExtractorTest { assertTrue(result); } - + @Test public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); - for(CommentsInfoItem c: comments.getItems()) { + for (CommentsInfoItem c : comments.getItems()) { assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); assertFalse(StringUtil.isBlank(c.getAuthorName())); assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); @@ -82,8 +82,8 @@ public class PeertubeCommentsExtractorTest { } private boolean findInComments(List comments, String comment) { - for(CommentsInfoItem c: comments) { - if(c.getCommentText().contains(comment)) { + for (CommentsInfoItem c : comments) { + if (c.getCommentText().contains(comment)) { return true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java index fcc27c54..f6fb91f7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeCommentsLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java index 9ad29102..ac04a01e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubePlaylistLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 75a692d4..e103e1a8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -1,17 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; - import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -24,21 +12,45 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; + +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link StreamExtractor} */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; + private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); + PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d"); extractor.fetchPage(); } + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + + } + @Test public void testGetInvalidTimeStamp() throws ParsingException { assertTrue(extractor.getTimeStamp() + "", @@ -47,22 +59,37 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "Power Corrupts the Best"); + assertEquals("What is PeerTube?", extractor.getName()); } @Test - public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); + public void testGetLargeDescription() throws ParsingException { + assertEquals(expectedLargeDescription, extractor.getDescription().getContent()); + } + + @Test + public void testGetEmptyDescription() throws Exception { + PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); + extractorEmpty.fetchPage(); + assertEquals("", extractorEmpty.getDescription().getContent()); + } + + @Test + public void testGetSmallDescription() throws Exception { + PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); + PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); + extractorSmall.fetchPage(); + assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent()); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "Rowsedower"); + assertEquals("Framasoft", extractor.getUploaderName()); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 269); + assertEquals(113, extractor.getLength()); } @Test @@ -71,18 +98,10 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - - } - @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); + assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); } @Test @@ -115,11 +134,31 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } @Test public void testGetSubtitlesList() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetAgeLimit() throws ExtractionException, IOException { + assertEquals(0, extractor.getAgeLimit()); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/3c0da7fb-e4d9-442e-84e3-a8c47004ee28"); + ageLimit.fetchPage(); + assertEquals(18, ageLimit.getAgeLimit()); + } + + @Test + public void testGetSupportInformation() throws ExtractionException, IOException { + PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37"); + supportInfoExtractor.fetchPage(); + assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); + } + + @Test + public void testGetLanguageInformation() throws ParsingException { + assertEquals(new Locale("en"), extractor.getLanguageInfo()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java index 893b6eb8..af44f644 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeStreamLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index bb4e1529..e7334fca 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -1,97 +1,79 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.util.List; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudChartsExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -/** - * Test for {@link PeertubeTrendingExtractor} - */ +import java.util.List; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.*; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + public class PeertubeTrendingExtractorTest { + public static class Trending implements BaseListExtractorTest { + private static PeertubeTrendingExtractor extractor; - static KioskExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = PeerTube - .getKioskList() - .getExtractorById("Trending", null); - extractor.fetchPage(); - } - - @Test - public void testGetDownloader() throws Exception { - assertNotNull(NewPipe.getDownloader()); - } - - @Test - public void testGetName() throws Exception { - assertEquals(extractor.getName(), "Trending"); - } - - @Test - public void testId() { - assertEquals(extractor.getId(), "Trending"); - } - - @Test - public void testGetStreams() throws Exception { - ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if(!page.getErrors().isEmpty()) { - System.err.println("----------"); - List errors = page.getErrors(); - for(Throwable e: errors) { - e.printStackTrace(); - System.err.println("----------"); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeTrendingExtractor) PeerTube.getKioskList() + .getExtractorById("Trending", null); + extractor.fetchPage(); } - assertTrue("no streams are received", - !page.getItems().isEmpty() - && page.getErrors().isEmpty()); - } - @Test - public void testGetStreamsErrors() throws Exception { - assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); - } + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ - @Test - public void testHasMoreStreams() throws Exception { - // Setup the streams - extractor.getInitialPage(); - assertTrue("has more streams", extractor.hasNextPage()); - } + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } - @Test - public void testGetNextPageUrl() throws Exception { - assertTrue(extractor.hasNextPage()); - } + @Test + public void testName() throws Exception { + assertEquals("Trending", extractor.getName()); + } - @Test - public void testGetNextPage() throws Exception { - extractor.getInitialPage().getItems(); - assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); - } + @Test + public void testId() throws Exception { + assertEquals("Trending", extractor.getId()); + } - @Test - public void testGetCleanUrl() throws Exception { - assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending"); + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java index d192e8be..033be650 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -1,9 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeTrendingLinkHandlerFactory} */ @@ -48,13 +48,13 @@ public class PeertubeTrendingLinkHandlerFactoryTest { public void acceptUrl() throws ParsingException { assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe")); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java index 08dd7a0a..030bd6c5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertTrue; - import org.junit.Test; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeSearchExtractor} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java index dd9c6deb..eeebf526 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -15,6 +10,9 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeSearchExtractor} */ @@ -38,15 +36,15 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB @Test public void testResultList_FirstElement() { InfoItem firstInfoItem = itemsPage.getItems().get(0); - + assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde")); } @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -67,7 +65,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } @@ -75,7 +73,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB assertFalse("First and second page are equal", equals); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", - secondPage.getNextPageUrl()); + secondPage.getNextPageUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java index 23e3100c..d7bd92aa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertEquals; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + public class PeertubeSearchQHTest { - + @BeforeClass public static void setUpClass() throws Exception { // setting instance might break test when running in parallel @@ -18,7 +18,7 @@ public class PeertubeSearchQHTest { @Test public void testRegularValues() throws Exception { assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans",PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans", PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 0de8e847..485462fb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -64,12 +64,12 @@ public class SoundcloudChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -120,7 +120,7 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -158,12 +158,12 @@ public class SoundcloudChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index d96b4c6e..189be417 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -1,92 +1,131 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.List; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; -/** - * Test for {@link SoundcloudChartsLinkHandlerFactory} - */ public class SoundcloudChartsExtractorTest { + public static class NewAndHot implements BaseListExtractorTest { + private static SoundcloudChartsExtractor extractor; - static KioskExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = SoundCloud - .getKioskList() - .getExtractorById("Top 50", null); - extractor.fetchPage(); - } - - @Test - public void testGetDownloader() throws Exception { - assertNotNull(NewPipe.getDownloader()); - } - - @Test - public void testGetName() throws Exception { - assertEquals(extractor.getName(), "Top 50"); - } - - @Test - public void testId() { - assertEquals(extractor.getId(), "Top 50"); - } - - @Test - public void testGetStreams() throws Exception { - ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if(!page.getErrors().isEmpty()) { - System.err.println("----------"); - List errors = page.getErrors(); - for(Throwable e: errors) { - e.printStackTrace(); - System.err.println("----------"); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudChartsExtractor) SoundCloud.getKioskList() + .getExtractorById("New & hot", null); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(SoundCloud.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() { + assertEquals("New & hot", extractor.getName()); + } + + @Test + public void testId() { + assertEquals("New & hot", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/new", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/new", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); } - assertTrue("no streams are received", - !page.getItems().isEmpty() - && page.getErrors().isEmpty()); } - @Test - public void testGetStreamsErrors() throws Exception { - assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); - } + public static class Top50Charts implements BaseListExtractorTest { + private static SoundcloudChartsExtractor extractor; - @Test - public void testHasMoreStreams() throws Exception { - // Setup the streams - extractor.getInitialPage(); - assertTrue("has more streams", extractor.hasNextPage()); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudChartsExtractor) SoundCloud.getKioskList() + .getExtractorById("Top 50", null); + extractor.fetchPage(); + } - @Test - public void testGetNextPageUrl() throws Exception { - assertTrue(extractor.hasNextPage()); - } + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ - @Test - public void testGetNextPage() throws Exception { - extractor.getInitialPage().getItems(); - assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); - } + @Test + public void testServiceId() { + assertEquals(SoundCloud.getServiceId(), extractor.getServiceId()); + } - @Test - public void testGetCleanUrl() throws Exception { - assertEquals(extractor.getUrl(), "https://soundcloud.com/charts/top"); + @Test + public void testName() { + assertEquals("Top 50", extractor.getName()); + } + + @Test + public void testId() { + assertEquals("Top 50", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/top", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/top", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index 1c4f4159..16089aab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -1,10 +1,12 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.junit.*; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; public class SoundcloudParsingHelperTest { @BeforeClass diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index c27dfdf8..b93c2cfb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -7,7 +7,6 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -67,13 +66,13 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() { try { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } catch (Throwable ignored) { return; } @@ -165,12 +164,12 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -226,11 +225,10 @@ public class SoundcloudPlaylistExtractorTest { // Additional Testing //////////////////////////////////////////////////////////////////////////*/ - @Ignore @Test public void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -269,18 +267,18 @@ public class SoundcloudPlaylistExtractorTest { @Ignore @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } //TODO: FUCK THIS: This triggers a 500 at sever @Ignore @Test public void testMoreRelatedItems() throws Exception { - ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId()); + ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); // Test for 2 more levels for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); - defaultTestListOfItems(SoundCloud.getServiceId(), currentPage.getItems(), currentPage.getErrors()); + defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 8a3883fd..0ffe0622 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { - assertEquals("The Perfect LUV Tape®️", extractor.getDescription()); + assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java index abd26951..04e7e0ab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java @@ -38,7 +38,7 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } @@ -57,8 +57,8 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx @Test public void testOnlyContainChannels() { - for(InfoItem item : itemsPage.getItems()) { - if(!(item instanceof ChannelInfoItem)) { + for (InfoItem item : itemsPage.getItems()) { + if (!(item instanceof ChannelInfoItem)) { fail("The following item is no channel item: " + item.toString()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java index 685c52d7..ca0d7d59 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java @@ -60,8 +60,8 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -80,7 +80,7 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 251ae246..dfcaee40 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,29 +1,46 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + /** * Test for {@link ChannelExtractor} */ public class YoutubeChannelExtractorTest { + + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void deletedFetch() throws Exception { + final ChannelExtractor extractor = + YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw"); + extractor.fetchPage(); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final ChannelExtractor extractor = + YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST"); + extractor.fetchPage(); + } + } + public static class Gronkh implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; @@ -70,12 +87,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -160,12 +177,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -175,7 +192,7 @@ public class YoutubeChannelExtractorTest { @Test public void testDescription() throws Exception { assertTrue("What it actually was: " + extractor.getDescription(), - extractor.getDescription().contains("Our World is Amazing. Questions? Ideas? Tweet me:")); + extractor.getDescription().contains("Our World is Amazing. \n\nQuestions? Ideas? Tweet me:")); } @Test @@ -223,7 +240,7 @@ public class YoutubeChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -262,12 +279,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -353,12 +370,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -443,12 +460,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -489,7 +506,6 @@ public class YoutubeChannelExtractorTest { } - public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; @@ -536,13 +552,13 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() { try { - defaultTestMoreItems(extractor, YouTube.getServiceId()); + defaultTestMoreItems(extractor); } catch (Throwable ignored) { return; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index 25f097a0..fc409ffa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -38,10 +38,10 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); - + assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh")); assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos")); - + assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); } @@ -60,10 +60,10 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); - + assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId()); assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId()); - + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java index 29f3d032..d5a629b0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java @@ -49,7 +49,7 @@ public class YoutubeChannelLocalizationTest { final ChannelExtractor extractor = YouTube.getChannelExtractor(channelUrl); extractor.forceLocalization(currentLocalization); extractor.fetchPage(); - itemsPage = defaultTestRelatedItems(extractor, YouTube.getServiceId()); + itemsPage = defaultTestRelatedItems(extractor); } catch (Throwable e) { System.out.println("[!] " + currentLocalization + " → failed"); throw e; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 3345ae3a..4349727b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -84,7 +84,7 @@ public class YoutubeCommentsExtractorTest { public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractorYT.getInitialPage(); - DefaultTests.defaultTestListOfItems(YouTube.getServiceId(), comments.getItems(), comments.getErrors()); + DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); for (CommentsInfoItem c : comments.getItems()) { assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); assertFalse(StringUtil.isBlank(c.getAuthorName())); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java index c7f1b1c1..a6a10ac5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtra import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; public class YoutubeFeedExtractorTest { @@ -60,13 +61,12 @@ public class YoutubeFeedExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test - public void testMoreRelatedItems() { - assertFalse(extractor.hasNextPage()); - assertNull(extractor.getNextPageUrl()); + public void testMoreRelatedItems() throws Exception { + assertNoMoreItems(extractor); } } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java new file mode 100644 index 00000000..bbe8a76c --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java @@ -0,0 +1,70 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + +public class YoutubeKioskExtractorTest { + public static class Trending implements BaseListExtractorTest { + private static YoutubeTrendingExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeTrendingExtractor) YouTube.getKioskList().getDefaultKioskExtractor(); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("Trending", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("Trending", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/feed/trending", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/feed/trending", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + assertNoMoreItems(extractor); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java new file mode 100644 index 00000000..87dbbd75 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -0,0 +1,25 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; + +public class YoutubeParsingHelperTest { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testIsHardcodedClientVersionValid() throws IOException, ExtractionException { + assertTrue("Hardcoded client version is not valid anymore", + YoutubeParsingHelper.isHardcodedClientVersionValid()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index d4de9175..65bb7f99 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -6,7 +6,8 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.ServiceList; + +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; @@ -23,6 +24,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link YoutubePlaylistExtractor} */ public class YoutubePlaylistExtractorTest { + + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final PlaylistExtractor extractor = + YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL11111111111111111111111111111111"); + extractor.fetchPage(); + } + + @Test(expected = ContentNotAvailableException.class) + public void invalidId() throws Exception { + final PlaylistExtractor extractor = + YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID"); + extractor.fetchPage(); + } + } + public static class TimelessPopHits implements BasePlaylistExtractorTest { private static YoutubePlaylistExtractor extractor; @@ -70,12 +93,12 @@ public class YoutubePlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -138,7 +161,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -177,18 +200,17 @@ public class YoutubePlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - ListExtractor.InfoItemsPage currentPage - = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); // test for 2 more levels for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); - defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors()); + defaultTestListOfItems(YouTube, currentPage.getItems(), currentPage.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 796238f9..908642fd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -77,8 +77,8 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", "youtube.com").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI", "youtube.com").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); } @@ -123,7 +123,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId()); } - + @Test public void testAcceptInvidioUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU")); @@ -133,7 +133,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE")); } - + @Test public void testGetInvidioIdfromUrl() throws ParsingException { assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java deleted file mode 100644 index 4931fc50..00000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -/* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2017 - * YoutubeTrendingExtractorTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.localization.ContentCountry; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.utils.Utils; - -import static junit.framework.TestCase.assertFalse; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - - -/** - * Test for {@link YoutubeTrendingLinkHandlerFactory} - */ -public class YoutubeTrendingExtractorTest { - - static YoutubeTrendingExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeTrendingExtractor) YouTube - .getKioskList() - .getExtractorById("Trending", null); - extractor.forceContentCountry(new ContentCountry("de")); - extractor.fetchPage(); - } - - @Test - public void testGetDownloader() throws Exception { - assertNotNull(NewPipe.getDownloader()); - } - - @Test - public void testGetName() throws Exception { - assertFalse(extractor.getName().isEmpty()); - } - - @Test - public void testId() throws Exception { - assertEquals(extractor.getId(), "Trending"); - } - - @Test - public void testGetStreamsQuantity() throws Exception { - ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - Utils.printErrors(page.getErrors()); - assertTrue("no streams are received", page.getItems().size() >= 20); - } - - @Test - public void testGetStreamsErrors() throws Exception { - assertEmptyErrors("errors during stream list extraction", extractor.getInitialPage().getErrors()); - } - - @Test - public void testHasMoreStreams() throws Exception { - // Setup the streams - extractor.getInitialPage(); - assertFalse("has more streams", extractor.hasNextPage()); - } - - @Test - public void testGetNextPage() { - assertTrue("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); - } - - @Test - public void testGetUrl() throws Exception { - assertEquals(extractor.getUrl(), extractor.getUrl(), "https://www.youtube.com/feed/trending"); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java index c8e23c06..c85d6b52 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java @@ -1,24 +1,24 @@ package org.schabi.newpipe.extractor.services.youtube; - /* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2017 - * YoutubeTrendingKioskInfoTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingKioskInfoTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ import org.junit.BeforeClass; import org.junit.Test; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java index 8b19b11e..729254ad 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java @@ -3,10 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.Test; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 91cbebd9..cb72622b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -12,8 +12,17 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + import static java.util.Arrays.asList; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { @@ -45,19 +54,32 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Ignore @Test public void testOnlyContainChannels() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { if (!(item instanceof ChannelInfoItem)) { fail("The following item is no channel item: " + item.toString()); } @@ -66,17 +88,22 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto @Test public void testChannelUrl() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { if (item instanceof ChannelInfoItem) { ChannelInfoItem channel = (ChannelInfoItem) item; - if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie + if (channel.getSubscriberCount() > 1e8) { // the real PewDiePie assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); - } else { - assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); + break; } } } + + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof ChannelInfoItem) { + assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); + } + } } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 49e3520d..1f905bde 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -6,11 +6,15 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -49,13 +53,28 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testGetUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl()); + assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Test @@ -76,8 +95,8 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -96,20 +115,18 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test public void testSuggestionNotNull() throws Exception { //todo write a real test - assertTrue(extractor.getSearchSuggestion() != null); + assertNotNull(extractor.getSearchSuggestion()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java index 7df7b0cb..656e197a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java @@ -1,15 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.search; -import static java.util.Collections.singletonList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -19,6 +9,14 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Collections.singletonList; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + public class YoutubeSearchPagingTest { private static ListExtractor.InfoItemsPage page1; private static ListExtractor.InfoItemsPage page2; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index c3d3f93c..fc6af4c4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -1,24 +1,21 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.Test; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; public class YoutubeSearchQHTest { @Test public void testRegularValues() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans",YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); - assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); - assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); - assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); } @Test @@ -31,13 +28,13 @@ public class YoutubeSearchQHTest { @Test public void testWithContentfilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAVAU", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAlAU", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQA1AU", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index b9da6d17..3710b4ee 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -64,7 +64,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test @@ -117,7 +117,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { streams.addAll(extractor.getVideoStreams()); streams.addAll(extractor.getVideoOnlyStreams()); - assertTrue(Integer.toString(streams.size()),streams.size() > 0); + assertTrue(Integer.toString(streams.size()), streams.size() > 0); for (VideoStream s : streams) { assertTrue(s.getUrl(), s.getUrl().contains(HTTPS)); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index f0bd7715..915f42ae 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 5f21df55..0d36fd9a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -7,20 +7,28 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; -import static java.util.Objects.*; -import static org.junit.Assert.*; +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -49,6 +57,27 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; */ public class YoutubeStreamExtractorDefaultTest { + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final StreamExtractor extractor = + YouTube.getStreamExtractor("https://www.youtube.com/watch?v=don-t-exist"); + extractor.fetchPage(); + } + + @Test(expected = ParsingException.class) + public void invalidId() throws Exception { + final StreamExtractor extractor = + YouTube.getStreamExtractor("https://www.youtube.com/watch?v=INVALID_ID_INVALID_ID"); + extractor.fetchPage(); + } + } + /** * Test for {@link StreamExtractor} */ @@ -83,13 +112,12 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("http://adele.com")); - assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); + assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); } @Test @@ -126,7 +154,7 @@ public class YoutubeStreamExtractorDefaultTest { public void testGetUploaderUrl() throws ParsingException { String url = extractor.getUploaderUrl(); if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") && - !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { + !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { fail("Uploader url is neither the music channel one nor the Vevo one"); } } @@ -142,12 +170,12 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetAudioStreams() throws IOException, ExtractionException { + public void testGetAudioStreams() throws ExtractionException { assertFalse(extractor.getAudioStreams().isEmpty()); } @Test - public void testGetVideoStreams() throws IOException, ExtractionException { + public void testGetVideoStreams() throws ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { assertIsSecureUrl(s.url); assertTrue(s.resolution.length() > 0); @@ -169,7 +197,7 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -177,13 +205,13 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } @@ -215,18 +243,14 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); - assertTrue(extractor.getDescription().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); - - assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub...")); - assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8...")); - assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/...")); + assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); + assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); } } @@ -244,20 +268,16 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - - assertFalse(extractor.getDescription().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); + final String description = extractor.getDescription().getContent(); + assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index eb2541ed..f5c7fe10 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -13,9 +13,12 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -26,7 +29,7 @@ public class YoutubeStreamExtractorLivestreamTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY"); + .getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A"); extractor.fetchPage(); } @@ -44,13 +47,12 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/")); - assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba...")); + assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists")); } @Test @@ -119,7 +121,7 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -127,12 +129,12 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java new file mode 100644 index 00000000..5dcd73ec --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java @@ -0,0 +1,166 @@ +package org.schabi.newpipe.extractor.services.youtube.stream; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.List; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeStreamExtractorUnlistedTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=udsB8KnIJTg"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetTitle() throws ParsingException { + assertFalse(extractor.getName().isEmpty()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().getContent().isEmpty()); + } + + @Test + public void testGetFullLinksInDescription() throws ParsingException { + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/user/Roccowschiptune")); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertNotNull(extractor.getUploaderName()); + assertFalse(extractor.getUploaderName().isEmpty()); + } + + + @Test + public void testGetLength() throws ParsingException { + assertEquals(2488, extractor.getLength()); + } + + @Test + public void testGetViewCount() throws ParsingException { + long count = extractor.getViewCount(); + assertTrue(Long.toString(count), count >= /* specific to that video */ 1225); + } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2017-09-22", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-09-22")); + assertNotNull(extractor.getUploadDate()); + assertEquals(instance, extractor.getUploadDate().date()); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetAudioStreams() throws ExtractionException { + List audioStreams = extractor.getAudioStreams(); + assertFalse(audioStreams.isEmpty()); + for (AudioStream s : audioStreams) { + assertIsSecureUrl(s.url); + assertTrue(Integer.toString(s.getFormatId()), + 0x100 <= s.getFormatId() && s.getFormatId() < 0x1000); + } + } + + @Test + public void testGetVideoStreams() throws ExtractionException { + for (VideoStream s : extractor.getVideoStreams()) { + assertIsSecureUrl(s.url); + assertTrue(s.resolution.length() > 0); + assertTrue(Integer.toString(s.getFormatId()), + 0 <= s.getFormatId() && s.getFormatId() < 0x100); + } + } + + @Test + public void testStreamType() throws ParsingException { + assertSame(StreamType.VIDEO_STREAM, extractor.getStreamType()); + } + + @Test + public void testGetNextVideo() throws ExtractionException { + assertNull(extractor.getNextStream()); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); + Utils.printErrors(relatedVideos.getErrors()); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() { + assertFalse(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() { + assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + } + + @Test + public void testGetLikeCount() throws ParsingException { + long likeCount = extractor.getLikeCount(); + assertTrue("" + likeCount, likeCount >= 96); + } + + @Test + public void testGetDislikeCount() throws ParsingException { + long dislikeCount = extractor.getDislikeCount(); + assertTrue("" + dislikeCount, dislikeCount >= 0); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java index dc8f2b04..925a647c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java @@ -1,47 +1,46 @@ package org.schabi.newpipe.extractor.utils; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.junit.Test; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.junit.Test; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.util.List; + +import static org.junit.Assert.assertTrue; public class JsonUtilsTest { - + @Test public void testGetValueFlat() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); - assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); + assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); } - + @Test public void testGetValueNested() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2"))); } - + @Test public void testGetArray() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "batters.batter"); assertTrue(!arr.isEmpty()); } - + @Test public void testGetValues() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping"); List types = JsonUtils.getValues(arr, "type"); assertTrue(types.contains("Chocolate with Sprinkles")); - + } }