diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4bf58ad9..76ca39a5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,3 @@ - [ ] I carefully read the [contribution guidelines](https://github.com/TeamNewPipe/NewPipe/blob/HEAD/.github/CONTRIBUTING.md) and agree to them. -- [ ] I did test the API against [NewPipe](https://github.com/TeamNewPipe/NewPipe). -- [ ] I agree to ASAP create a PULL request for [NewPipe](https://github.com/TeamNewPipe/NewPipe) for making in compatible when I changed the api. +- [ ] I have tested the API against [NewPipe](https://github.com/TeamNewPipe/NewPipe). +- [ ] I agree to create a pull request for [NewPipe](https://github.com/TeamNewPipe/NewPipe) as soon as possible to make it compatible with the changed API. diff --git a/README.md b/README.md index 66515055..835b640a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # NewPipe Extractor -[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [Documentation](https://teamnewpipe.github.io/documentation/) +[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [JDoc](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) • [Documentation](https://teamnewpipe.github.io/documentation/) NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently. @@ -11,11 +11,21 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `compile 'com.github.TeamNewPipe:NewPipeExtractor:v0.11.0'`the `dependencies` in your `build.gradle`. Replace `v0.11.0` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.0'`the `dependencies` in your `build.gradle`. Replace `v0.19.0` with the latest release. ### Testing changes -To test changes quickly you can build the library locally. Using the local Maven repository is a good approach, here's a gist of how to use it: +To test changes quickly you can build the library locally. A good approach would be to add something like the following to your `settings.gradle`: + +```groovy +includeBuild('../NewPipeExtractor') { + dependencySubstitution { + substitute module('com.github.TeamNewPipe:NewPipeExtractor') with project(':extractor') + } +} +``` + +Another approach would be to use the local Maven repository, here's a gist of how to use it: 1. Add `mavenLocal()` in your project `repositories` list (usually as the first entry to give priority above the others). 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). diff --git a/build.gradle b/build.gradle index a6259806..611e1033 100644 --- a/build.gradle +++ b/build.gradle @@ -5,11 +5,12 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.19.0' + version 'v0.19.4' group 'com.github.TeamNewPipe' repositories { jcenter() + maven { url "https://jitpack.io" } } } diff --git a/extractor/build.gradle b/extractor/build.gradle index 1b7fbf00..2138df88 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -1,11 +1,11 @@ dependencies { implementation project(':timeago-parser') - implementation 'com.grack:nanojson:1.1' + implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation 'org.jsoup:jsoup:1.9.2' implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' testImplementation 'junit:junit:4.12' -} \ No newline at end of file +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index b254adbd..0e21f04b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -3,16 +3,33 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + /** * Base class to extractors that have a list (e.g. playlists, users). */ public abstract class ListExtractor extends Extractor { + /** + * Constant that should be returned whenever + * a list has an unknown number of items. + */ + public static final long ITEM_COUNT_UNKNOWN = -1; + /** + * Constant that should be returned whenever a list has an + * infinite number of items. For example a YouTube mix. + */ + public static final long ITEM_COUNT_INFINITE = -2; + /** + * Constant that should be returned whenever a list + * has an unknown number of items bigger than 100. + */ + public static final long ITEM_COUNT_MORE_THAN_100 = -3; + public ListExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotSupportedException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotSupportedException.java new file mode 100644 index 00000000..c32575b4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotSupportedException.java @@ -0,0 +1,11 @@ +package org.schabi.newpipe.extractor.exceptions; + +public class ContentNotSupportedException extends ParsingException { + public ContentNotSupportedException(String message) { + super(message); + } + + public ContentNotSupportedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index f2d9b673..523584e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -6,7 +6,12 @@ import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; @@ -21,19 +26,17 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import java.io.IOException; - import static java.util.Arrays.asList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; public class MediaCCCService extends StreamingService { - public MediaCCCService(int id) { + public MediaCCCService(final int id) { super(id, "MediaCCC", asList(AUDIO, VIDEO)); } @Override - public SearchExtractor getSearchExtractor(SearchQueryHandler query) { + public SearchExtractor getSearchExtractor(final SearchQueryHandler query) { return new MediaCCCSearchExtractor(this, query); } @@ -58,17 +61,17 @@ public class MediaCCCService extends StreamingService { } @Override - public StreamExtractor getStreamExtractor(LinkHandler linkHandler) { + public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) { return new MediaCCCStreamExtractor(this, linkHandler); } @Override - public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) { + public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) { return new MediaCCCConferenceExtractor(this, linkHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { return null; } @@ -85,9 +88,9 @@ public class MediaCCCService extends StreamingService { try { list.addKioskEntry(new KioskList.KioskExtractorFactory() { @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, - String url, - String kioskId) throws ExtractionException, IOException { + public KioskExtractor createNewKiosk(final StreamingService streamingService, + final String url, final String kioskId) + throws ExtractionException { return new MediaCCCConferenceKiosk(MediaCCCService.this, new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId); } @@ -111,8 +114,7 @@ public class MediaCCCService extends StreamingService { } @Override - public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) - throws ExtractionException { + public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 5c4079ae..4cd21c06 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -14,45 +15,46 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.Medi import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; -public class MediaCCCConferenceExtractor extends ChannelExtractor { +import javax.annotation.Nonnull; +public class MediaCCCConferenceExtractor extends ChannelExtractor { private JsonObject conferenceData; - public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler) { + public MediaCCCConferenceExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @Override - public String getAvatarUrl() throws ParsingException { + public String getAvatarUrl() { return conferenceData.getString("logo_url"); } @Override - public String getBannerUrl() throws ParsingException { + public String getBannerUrl() { return conferenceData.getString("logo_url"); } @Override - public String getFeedUrl() throws ParsingException { + public String getFeedUrl() { return null; } @Override - public long getSubscriberCount() throws ParsingException { + public long getSubscriberCount() { return -1; } @Override - public String getDescription() throws ParsingException { + public String getDescription() { return null; } @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray events = conferenceData.getArray("events"); for (int i = 0; i < events.size(); i++) { @@ -62,17 +64,18 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { } @Override - public String getNextPageUrl() throws IOException, ExtractionException { + public String getNextPageUrl() { return null; } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) { return null; } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { try { conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody()); } catch (JsonParserException jpe) { @@ -88,7 +91,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { @Nonnull @Override - public String getOriginalUrl() throws ParsingException { + public String getOriginalUrl() { return "https://media.ccc.de/c/" + conferenceData.getString("acronym"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java index f9800b46..010d3881 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; @@ -14,22 +15,22 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCConferenceInfoItemExtractor; -import javax.annotation.Nonnull; import java.io.IOException; -public class MediaCCCConferenceKiosk extends KioskExtractor { +import javax.annotation.Nonnull; +public class MediaCCCConferenceKiosk extends KioskExtractor { private JsonObject doc; - public MediaCCCConferenceKiosk(StreamingService streamingService, - ListLinkHandler linkHandler, - String kioskId) { + public MediaCCCConferenceKiosk(final StreamingService streamingService, + final ListLinkHandler linkHandler, + final String kioskId) { super(streamingService, linkHandler, kioskId); } @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() { JsonArray conferences = doc.getArray("conferences"); ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId()); for (int i = 0; i < conferences.size(); i++) { @@ -40,18 +41,20 @@ public class MediaCCCConferenceKiosk extends KioskExtractor { } @Override - public String getNextPageUrl() throws IOException, ExtractionException { + public String getNextPageUrl() { return ""; } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) { return InfoItemsPage.emptyPage(); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization()).responseBody(); + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + final String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization()) + .responseBody(); try { doc = JsonParser.object().from(site); } catch (JsonParserException jpe) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index 0a51af2a..9e5baaf9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -7,11 +7,10 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -public class MediaCCCParsingHelper { - private MediaCCCParsingHelper() { - } +public final class MediaCCCParsingHelper { + private MediaCCCParsingHelper() { } - public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { + public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { Date date; try { date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 96618fd9..d5ced534 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -4,31 +4,34 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.List; -import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.*; +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.ALL; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS; public class MediaCCCSearchExtractor extends SearchExtractor { - private JsonObject doc; private MediaCCCConferenceKiosk conferenceKiosk; - public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + public MediaCCCSearchExtractor(final StreamingService service, + final SearchQueryHandler linkHandler) { super(service, linkHandler); try { conferenceKiosk = new MediaCCCConferenceKiosk(service, @@ -40,13 +43,13 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } @Override - public String getSearchSuggestion() throws ParsingException { + public String getSearchSuggestion() { return null; } @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() { final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId()); if (getLinkHandler().getContentFilters().contains(CONFERENCES) @@ -70,17 +73,18 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } @Override - public String getNextPageUrl() throws IOException, ExtractionException { + public String getNextPageUrl() { return ""; } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) { return InfoItemsPage.emptyPage(); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { if (getLinkHandler().getContentFilters().contains(EVENTS) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { @@ -95,44 +99,45 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) - || getLinkHandler().getContentFilters().isEmpty()) + || getLinkHandler().getContentFilters().isEmpty()) { conferenceKiosk.fetchPage(); + } } - private void searchConferences(String searchString, - List channelItems, - InfoItemsSearchCollector collector) { + private void searchConferences(final String searchString, + final List channelItems, + final InfoItemsSearchCollector collector) { for (final ChannelInfoItem item : channelItems) { if (item.getName().toUpperCase().contains( searchString.toUpperCase())) { collector.commit(new ChannelInfoItemExtractor() { @Override - public String getDescription() throws ParsingException { + public String getDescription() { return item.getDescription(); } @Override - public long getSubscriberCount() throws ParsingException { + public long getSubscriberCount() { return item.getSubscriberCount(); } @Override - public long getStreamCount() throws ParsingException { + public long getStreamCount() { return item.getStreamCount(); } @Override - public String getName() throws ParsingException { + public String getName() { return item.getName(); } @Override - public String getUrl() throws ParsingException { + public String getUrl() { return item.getUrl(); } @Override - public String getThumbnailUrl() throws ParsingException { + public String getThumbnailUrl() { return item.getThumbnailUrl(); } }); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 59bfe1f4..894a0f0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -11,27 +12,34 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; -public class MediaCCCStreamExtractor extends StreamExtractor { +import javax.annotation.Nonnull; +public class MediaCCCStreamExtractor extends StreamExtractor { private JsonObject data; private JsonObject conferenceData; - public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler) { + public MediaCCCStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); } @Nonnull @Override - public String getTextualUploadDate() throws ParsingException { + public String getTextualUploadDate() { return data.getString("release_date"); } @@ -43,79 +51,79 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getThumbnailUrl() throws ParsingException { + public String getThumbnailUrl() { return data.getString("thumb_url"); } @Nonnull @Override - public Description getDescription() throws ParsingException { + public Description getDescription() { return new Description(data.getString("description"), Description.PLAIN_TEXT); } @Override - public int getAgeLimit() throws ParsingException { + public int getAgeLimit() { return 0; } @Override - public long getLength() throws ParsingException { + public long getLength() { return data.getInt("length"); } @Override - public long getTimeStamp() throws ParsingException { + public long getTimeStamp() { return 0; } @Override - public long getViewCount() throws ParsingException { + public long getViewCount() { return data.getInt("view_count"); } @Override - public long getLikeCount() throws ParsingException { + public long getLikeCount() { return -1; } @Override - public long getDislikeCount() throws ParsingException { + public long getDislikeCount() { return -1; } @Nonnull @Override - public String getUploaderUrl() throws ParsingException { + public String getUploaderUrl() { return data.getString("conference_url"); } @Nonnull @Override - public String getUploaderName() throws ParsingException { + public String getUploaderName() { return data.getString("conference_url") - .replace("https://api.media.ccc.de/public/conferences/", ""); + .replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", ""); } @Nonnull @Override - public String getUploaderAvatarUrl() throws ParsingException { + public String getUploaderAvatarUrl() { return conferenceData.getString("logo_url"); } @Nonnull @Override - public String getDashMpdUrl() throws ParsingException { + public String getDashMpdUrl() { return ""; } @Nonnull @Override - public String getHlsUrl() throws ParsingException { + public String getHlsUrl() { return ""; } @Override - public List getAudioStreams() throws IOException, ExtractionException { + public List getAudioStreams() throws ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List audioStreams = new ArrayList<>(); for (int i = 0; i < recordings.size(); i++) { @@ -134,14 +142,15 @@ public class MediaCCCStreamExtractor extends StreamExtractor { throw new ExtractionException("Unknown media format: " + mimeType); } - audioStreams.add(new AudioStream(recording.getString("recording_url"), mediaFormat, -1)); + audioStreams.add(new AudioStream(recording.getString("recording_url"), + mediaFormat, -1)); } } return audioStreams; } @Override - public List getVideoStreams() throws IOException, ExtractionException { + public List getVideoStreams() throws ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List videoStreams = new ArrayList<>(); for (int i = 0; i < recordings.size(); i++) { @@ -167,34 +176,34 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public List getVideoOnlyStreams() throws IOException, ExtractionException { + public List getVideoOnlyStreams() { return null; } @Nonnull @Override - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitlesDefault() { return Collections.emptyList(); } @Nonnull @Override - public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { + public List getSubtitles(final MediaFormat format) { return Collections.emptyList(); } @Override - public StreamType getStreamType() throws ParsingException { + public StreamType getStreamType() { return StreamType.VIDEO_STREAM; } @Override - public StreamInfoItem getNextStream() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() { return null; } @Override - public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() { return new StreamInfoItemsCollector(getServiceId()); } @@ -204,14 +213,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { try { data = JsonParser.object().from( downloader.get(getLinkHandler().getUrl()).responseBody()); conferenceData = JsonParser.object() .from(downloader.get(getUploaderUrl()).responseBody()); } catch (JsonParserException jpe) { - throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe); + throw new ExtractionException("Could not parse json returned by url: " + + getLinkHandler().getUrl(), jpe); } } @@ -223,44 +234,44 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getOriginalUrl() throws ParsingException { + public String getOriginalUrl() { return data.getString("frontend_link"); } @Override - public String getHost() throws ParsingException { + public String getHost() { return ""; } @Override - public String getPrivacy() throws ParsingException { + public String getPrivacy() { return ""; } @Override - public String getCategory() throws ParsingException { + public String getCategory() { return ""; } @Override - public String getLicence() throws ParsingException { + public String getLicence() { return ""; } @Override - public Locale getLanguageInfo() throws ParsingException { + public Locale getLanguageInfo() { return null; } @Nonnull @Override - public List getTags() throws ParsingException { + public List getTags() { return new ArrayList<>(); } @Nonnull @Override - public String getSupportInfo() throws ParsingException { + public String getSupportInfo() { return ""; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java deleted file mode 100644 index b8423075..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.schabi.newpipe.extractor.services.media_ccc.extractors; - -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class MediaCCCSuggestionExtractor extends SuggestionExtractor { - - public MediaCCCSuggestionExtractor(StreamingService service) { - super(service); - } - - @Override - public List suggestionList(String query) throws IOException, ExtractionException { - return new ArrayList<>(0); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index 05914dd7..9099cb1a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -5,25 +5,24 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtractor { + private JsonObject conference; - JsonObject conference; - - public MediaCCCConferenceInfoItemExtractor(JsonObject conference) { + public MediaCCCConferenceInfoItemExtractor(final JsonObject conference) { this.conference = conference; } @Override - public String getDescription() throws ParsingException { + public String getDescription() { return ""; } @Override - public long getSubscriberCount() throws ParsingException { + public long getSubscriberCount() { return -1; } @Override - public long getStreamCount() throws ParsingException { + public long getStreamCount() { return -1; } @@ -38,7 +37,7 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra } @Override - public String getThumbnailUrl() throws ParsingException { + public String getThumbnailUrl() { return conference.getString("logo_url"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 76347b39..23a6aa93 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -10,47 +10,46 @@ import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor { + private JsonObject event; - JsonObject event; - - public MediaCCCStreamInfoItemExtractor(JsonObject event) { + public MediaCCCStreamInfoItemExtractor(final JsonObject event) { this.event = event; } @Override - public StreamType getStreamType() throws ParsingException { + public StreamType getStreamType() { return StreamType.VIDEO_STREAM; } @Override - public boolean isAd() throws ParsingException { + public boolean isAd() { return false; } @Override - public long getDuration() throws ParsingException { + public long getDuration() { return event.getInt("length"); } @Override - public long getViewCount() throws ParsingException { + public long getViewCount() { return event.getInt("view_count"); } @Override - public String getUploaderName() throws ParsingException { + public String getUploaderName() { return event.getString("conference_url") - .replace("https://api.media.ccc.de/public/conferences/", ""); + .replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", ""); } @Override - public String getUploaderUrl() throws ParsingException { + public String getUploaderUrl() { return event.getString("conference_url"); } @Nullable @Override - public String getTextualUploadDate() throws ParsingException { + public String getTextualUploadDate() { return event.getString("release_date"); } @@ -67,12 +66,12 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor @Override public String getUrl() throws ParsingException { - return "https://api.media.ccc.de/public/events/" + - event.getString("guid"); + return "https://media.ccc.de/public/events/" + + event.getString("guid"); } @Override - public String getThumbnailUrl() throws ParsingException { + public String getThumbnailUrl() { return event.getString("thumb_url"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index f4074c0d..5dc903d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -7,16 +7,17 @@ import org.schabi.newpipe.extractor.utils.Parser; import java.util.List; public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { - @Override - public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { - return "https://api.media.ccc.de/public/conferences/" + id; + public String getUrl(final String id, final List contentFilter, final String sortFilter) + throws ParsingException { + return "https://media.ccc.de/public/conferences/" + id; } @Override - public String getId(String url) throws ParsingException { - if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { - return url.replace("https://api.media.ccc.de/public/conferences/", ""); + public String getId(final String url) throws ParsingException { + if (url.startsWith("https://media.ccc.de/public/conferences/") + || url.startsWith("https://api.media.ccc.de/public/conferences/")) { + return url.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", ""); } else if (url.startsWith("https://media.ccc.de/c/")) { return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); } else if (url.startsWith("https://media.ccc.de/b/")) { @@ -26,7 +27,7 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory } @Override - public boolean onAcceptUrl(String url) throws ParsingException { + public boolean onAcceptUrl(final String url) { try { getId(url); return true; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java index d44dc538..f5dc8c6c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java @@ -7,18 +7,20 @@ import java.util.List; public class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory { @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { return "conferences"; } @Override - public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { - return "https://api.media.ccc.de/public/conferences"; + public String getUrl(final String id, final List contentFilter, + final String sortFilter) throws ParsingException { + return "https://media.ccc.de/public/conferences"; } @Override - public boolean onAcceptUrl(String url) throws ParsingException { + public boolean onAcceptUrl(final String url) { return url.equals("https://media.ccc.de/b/conferences") + || url.equals("https://media.ccc.de/public/conferences") || url.equals("https://api.media.ccc.de/public/conferences"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index dcc0b29b..0fd62806 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -8,7 +8,6 @@ import java.net.URLEncoder; import java.util.List; public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - public static final String ALL = "all"; public static final String CONFERENCES = "conferences"; public static final String EVENTS = "events"; @@ -28,11 +27,13 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory } @Override - public String getUrl(String querry, List contentFilter, String sortFilter) throws ParsingException { + public String getUrl(final String query, final List contentFilter, + final String sortFilter) throws ParsingException { try { - return "https://api.media.ccc.de/public/events/search?q=" + URLEncoder.encode(querry, "UTF-8"); + return "https://media.ccc.de/public/events/search?q=" + + URLEncoder.encode(query, "UTF-8"); } catch (UnsupportedEncodingException e) { - throw new ParsingException("Could not create search string with querry: " + querry, e); + throw new ParsingException("Could not create search string with querry: " + query, e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index 2c91d056..a335abf8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -9,11 +8,15 @@ import java.net.MalformedURLException; import java.net.URL; public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { - @Override - public String getId(String urlString) throws ParsingException { - if (urlString.startsWith("https://api.media.ccc.de/public/events/") && - !urlString.contains("?q=")) { + public String getId(final String urlString) throws ParsingException { + if (urlString.startsWith("https://media.ccc.de/public/events/") + && !urlString.contains("?q=")) { + return urlString.substring(35); //remove …/public/events part + } + + if (urlString.startsWith("https://api.media.ccc.de/public/events/") + && !urlString.contains("?q=")) { return urlString.substring(39); //remove api…/public/events part } @@ -38,12 +41,12 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { } @Override - public String getUrl(String id) throws ParsingException { - return "https://api.media.ccc.de/public/events/" + id; + public String getUrl(final String id) throws ParsingException { + return "https://media.ccc.de/public/events/" + id; } @Override - public boolean onAcceptUrl(String url) throws ParsingException { + public boolean onAcceptUrl(final String url) { try { getId(url); return true; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 29cc2b1d..e025c2be 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -77,7 +77,12 @@ public class PeertubeService extends StreamingService { @Override public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return new PeertubeChannelExtractor(this, linkHandler); + + if (linkHandler.getUrl().contains("/video-channels/")) { + return new PeertubeChannelExtractor(this, linkHandler); + } else { + return new PeertubeAccountExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java new file mode 100644 index 00000000..81cb0afa --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -0,0 +1,187 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import java.io.IOException; + +public class PeertubeAccountExtractor extends ChannelExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + private JsonObject json; + private final String baseUrl; + + public PeertubeAccountExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + super(service, linkHandler); + this.baseUrl = getBaseUrl(); + } + + @Override + public String getAvatarUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(json, "avatar.path"); + } catch (Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + + @Override + public String getBannerUrl() throws ParsingException { + return null; + } + + @Override + public String getFeedUrl() throws ParsingException { + return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + Number number = JsonUtils.getNumber(json, "followersCount"); + return number.longValue(); + } + + @Override + public String getDescription() throws ParsingException { + try { + return JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return "No description"; + } + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + } catch (Exception e) { + throw new ParsingException("unable to extract channel streams", e); + } + + for (Object c : contents) { + if (c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Response response = getDownloader().get(pageUrl); + JsonObject json = null; + if (response != null && !StringUtil.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if (json != null) { + PeertubeParsingHelper.validate(json); + Number number = JsonUtils.getNumber(json, "total"); + if (number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get PeerTube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if (StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if (nextStart >= total) { + return ""; + } else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + if (null != response && null != response.responseBody()) { + setInitialData(response.responseBody()); + } else { + throw new ExtractionException("Unable to extract PeerTube channel data"); + } + + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract PeerTube channel data", e); + } + if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "displayName"); + } + + @Override + public String getOriginalUrl() throws ParsingException { + return baseUrl + "/" + getId(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index e5acf1ee..dc27be80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -57,7 +57,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { - return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id"); + return getBaseUrl() + "/feeds/videos.xml?videoChannelId=" + json.get("id"); } @Override @@ -181,7 +181,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public String getOriginalUrl() throws ParsingException { - return baseUrl + "/accounts/" + getId(); + return baseUrl + "/" + getId(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index b7cbcec7..d3b43fae 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -97,7 +97,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getAuthorEndpoint() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 76ead5d6..3cf65154 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -128,7 +128,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(json, "account.name"); String host = JsonUtils.getString(json, "account.host"); - return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } @Override @@ -167,7 +167,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - JsonArray streams = json.getArray("files", new JsonArray()); + JsonArray streams = json.getArray("files"); for (Object s : streams) { if (!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index c4ed7730..df8b8a60 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -51,7 +51,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); - return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + + return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index c1e3f570..ef81ca4b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -10,8 +10,8 @@ import java.util.List; public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); - private static final String ID_PATTERN = "/accounts/([^/?&#]*)"; - private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/"; + private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)"; + private static final String API_ENDPOINT = "/api/v1/"; public static PeertubeChannelLinkHandlerFactory getInstance() { return instance; @@ -19,7 +19,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { - return Parser.matchGroup1(ID_PATTERN, url); + return Parser.matchGroup(ID_PATTERN, url, 0); } @Override @@ -31,11 +31,17 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return baseUrl + ACCOUNTS_ENDPOINT + id; + + if (id.matches(ID_PATTERN)) { + return baseUrl + API_ENDPOINT + id; + } else { + // This is needed for compatibility with older versions were we didn't support video channels yet + return baseUrl + API_ENDPOINT + "accounts/" + id; + } } @Override public boolean onAcceptUrl(String url) { - return url.contains("/accounts/"); + return url.contains("/accounts/") || url.contains("/video-channels/"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 853b6fc1..3c51271c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -16,6 +16,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; + @SuppressWarnings("WeakerAccess") public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; @@ -62,10 +64,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() { - return user.getObject("visuals", new JsonObject()) - .getArray("visuals", new JsonArray()) - .getObject(0, new JsonObject()) - .getString("visual_url"); + return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url"); } @Override @@ -80,7 +79,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getDescription() { - return user.getString("description", ""); + return user.getString("description", EMPTY_STRING); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index 118b59cc..641438e7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { @@ -24,7 +25,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getThumbnailUrl() { - String avatarUrl = itemObject.getString("avatar_url", ""); + String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING); String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg"); return avatarUrlBetterResolution; } @@ -41,6 +42,6 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getDescription() { - return itemObject.getString("description", ""); + return itemObject.getString("description", EMPTY_STRING); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index a08502d7..9a877fab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -11,6 +11,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + public class SoundcloudChartsExtractor extends KioskExtractor { private StreamInfoItemsCollector collector = null; private String nextPageUrl = null; @@ -57,11 +59,9 @@ public class SoundcloudChartsExtractor extends KioskExtractor { apiUrl += "&kind=trending"; } - /*List supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US"); - String contentCountry = getContentCountry(); - if (supportedCountries.contains(contentCountry)) { - apiUrl += "®ion=soundcloud:regions:" + contentCountry; - }*/ + + String contentCountry = SoundCloud.getContentCountry().getCountryCode(); + apiUrl += "®ion=soundcloud:regions:" + contentCountry; nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index a957ed81..4ea61516 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -28,6 +28,7 @@ import java.util.*; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { @@ -256,17 +257,17 @@ public class SoundcloudParsingHelper { @Nonnull static String getUploaderUrl(JsonObject object) { - String url = object.getObject("user").getString("permalink_url", ""); + String url = object.getObject("user").getString("permalink_url", EMPTY_STRING); return replaceHttpWithHttps(url); } @Nonnull static String getAvatarUrl(JsonObject object) { - String url = object.getObject("user", new JsonObject()).getString("avatar_url", ""); + String url = object.getObject("user").getString("avatar_url", EMPTY_STRING); return replaceHttpWithHttps(url); } public static String getUploaderName(JsonObject object) { - return object.getObject("user").getString("username", ""); + return object.getObject("user").getString("username", EMPTY_STRING); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java index 53123859..ae6875e7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { @@ -31,7 +32,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr public String getThumbnailUrl() throws ParsingException { // Over-engineering at its finest if (itemObject.isString(ARTWORK_URL_KEY)) { - final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, ""); + final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); if (!artworkUrl.isEmpty()) { String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); return artworkUrlBetterResolution; @@ -45,7 +46,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr // First look for track artwork url if (trackObject.isString(ARTWORK_URL_KEY)) { - String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, ""); + String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); if (!artworkUrl.isEmpty()) { String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); return artworkUrlBetterResolution; @@ -53,8 +54,8 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr } // Then look for track creator avatar url - final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject()); - final String creatorAvatar = creator.getString(AVATAR_URL_KEY, ""); + final JsonObject creator = trackObject.getObject(USER_KEY); + final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING); if (!creatorAvatar.isEmpty()) return creatorAvatar; } } catch (Exception ignored) { @@ -63,7 +64,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr try { // Last resort, use user avatar url. If still not found, then throw exception. - return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, ""); + return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING); } catch (Exception e) { throw new ParsingException("Failed to extract playlist thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 780e7e2b..1e20818d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -23,6 +23,7 @@ import java.net.MalformedURLException; import java.net.URL; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; public class SoundcloudSearchExtractor extends SearchExtractor { @@ -84,7 +85,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor { if (!(result instanceof JsonObject)) continue; //noinspection ConstantConditions JsonObject searchResult = (JsonObject) result; - String kind = searchResult.getString("kind", ""); + String kind = searchResult.getString("kind", EMPTY_STRING); switch (kind) { case "user": collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 30f2fc87..1c299879 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -7,11 +7,14 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import java.util.List; + import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; @@ -46,6 +49,13 @@ public class SoundcloudService extends StreamingService { return SoundcloudPlaylistLinkHandlerFactory.getInstance(); } + @Override + public List getSupportedCountries() { + //Country selector here https://soundcloud.com/charts/top?genre=all-music + return ContentCountry.listFrom( + "AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US" + ); + } @Override public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 16eaad01..1ee7ca33 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -33,6 +34,8 @@ import java.util.Locale; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; + public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -44,7 +47,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl()); - String policy = track.getString("policy", ""); + String policy = track.getString("policy", EMPTY_STRING); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { throw new ContentNotAvailableException("Content not available: policy " + policy); } @@ -77,9 +80,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public String getThumbnailUrl() { - String artworkUrl = track.getString("artwork_url", ""); + String artworkUrl = track.getString("artwork_url", EMPTY_STRING); if (artworkUrl.isEmpty()) { - artworkUrl = track.getObject("user").getString("avatar_url", ""); + artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING); } String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); return artworkUrlBetterResolution; @@ -196,6 +199,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { throw new ExtractionException("Could not get SoundCloud's track audio url", e); } + if (audioStreams.isEmpty()) { + throw new ContentNotSupportedException("HLS audio streams are not yet supported"); + } + return audioStreams; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java index da0ca1b5..31a719ea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -62,7 +63,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto @Override public String getThumbnailUrl() { - String artworkUrl = itemObject.getString("artwork_url", ""); + String artworkUrl = itemObject.getString("artwork_url", EMPTY_STRING); if (artworkUrl.isEmpty()) { artworkUrl = itemObject.getObject("user").getString("avatar_url"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index eae3bcb9..51967214 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -7,22 +7,45 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import javax.annotation.Nonnull; import java.util.List; +import javax.annotation.Nonnull; + import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; /* * Created by Christian Schabesberger on 23.08.15. @@ -92,7 +115,13 @@ public class YoutubeService extends StreamingService { @Override public SearchExtractor getSearchExtractor(SearchQueryHandler query) { - return new YoutubeSearchExtractor(this, query); + final List contentFilters = query.getContentFilters(); + + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) { + return new YoutubeMusicSearchExtractor(this, query); + } else { + return new YoutubeSearchExtractor(this, query); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 38598074..8c1b8961 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,9 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -15,11 +17,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; -import static org.schabi.newpipe.extractor.utils.JsonUtils.*; +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; /* * Created by Christian Schabesberger on 25.07.16. @@ -71,22 +76,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor { while (level < 3) { final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization()); - final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT) - .getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY) - .getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT) - .getObject("endpoint", EMPTY_OBJECT); + final JsonObject endpoint = jsonResponse.getObject(1).getObject("response") + .getArray("onResponseReceivedActions").getObject(0).getObject("navigateAction") + .getObject("endpoint"); - final String webPageType = endpoint - .getObject("commandMetadata", EMPTY_OBJECT) - .getObject("webCommandMetadata", EMPTY_OBJECT) + final String webPageType = endpoint.getObject("commandMetadata").getObject("webCommandMetadata") .getString("webPageType", EMPTY_STRING); - final String browseId = endpoint - .getObject("browseEndpoint", EMPTY_OBJECT) - .getString("browseId", EMPTY_STRING); + final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING); if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) { - if (!browseId.startsWith("UC")) { throw new ExtractionException("Redirected id is not pointing to a channel"); } @@ -130,10 +129,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public String getId() throws ParsingException { - final String channelId = initialData - .getObject("header", EMPTY_OBJECT) - .getObject("c4TabbedHeaderRenderer", EMPTY_OBJECT) - .getString("channelId", EMPTY_STRING); + final String channelId = initialData.getObject("header").getObject("c4TabbedHeaderRenderer") + .getString("channelId", EMPTY_STRING); if (!channelId.isEmpty()) { return channelId; @@ -169,11 +166,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - String url = null; - try { - url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner") + String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner") .getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} + if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; } @@ -195,19 +190,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); - if (subscriberInfo != null) { + final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header").getObject("c4TabbedHeaderRenderer"); + if (c4TabbedHeaderRenderer.has("subscriberCountText")) { try { - return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo)); + return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer.getObject("subscriberCountText"))); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } } else { // If there's no subscribe button, the channel has the subscriber count disabled - if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) { - return -1; - } else { + if (c4TabbedHeaderRenderer.has("subscribeButton")) { return 0; + } else { + return -1; } } } @@ -259,7 +254,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null) return ""; + if (continuations == null || continuations.isEmpty()) { + return ""; + } JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); @@ -276,7 +273,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object video : videos) { - if (((JsonObject) video).getObject("gridVideoRenderer") != null) { + if (((JsonObject) video).has("gridVideoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor( ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) { @Override @@ -301,8 +298,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { JsonObject videoTab = null; for (Object tab : tabs) { - if (((JsonObject) tab).getObject("tabRenderer") != null) { - if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) { + if (((JsonObject) tab).has("tabRenderer")) { + if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) { videoTab = ((JsonObject) tab).getObject("tabRenderer"); break; } @@ -310,16 +307,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } if (videoTab == null) { - throw new ParsingException("Could not find Videos tab"); + throw new ContentNotSupportedException("This channel has no Videos tab"); } - try { - if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("messageRenderer") - .getObject("text")).equals("This channel has no videos.")) - return null; - } catch (Exception ignored) {} + final String messageRendererText = getTextFromObject(videoTab.getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("messageRenderer").getObject("text")); + if (messageRendererText != null + && messageRendererText.equals("This channel has no videos.")) { + return null; + } this.videoTab = videoTab; return videoTab; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 27c08235..68d1c48b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -70,14 +70,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getSubscriberCount() throws ParsingException { try { - final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText"); - - if (subscriberCountObject == null) { + if (!channelInfoItem.has("subscriberCountText")) { // Subscription count is not available for this channel item. return -1; } - return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject)); + return Utils.mixedNumberWordToLong(getTextFromObject(channelInfoItem.getObject("subscriberCountText"))); } catch (Exception e) { throw new ParsingException("Could not get subscriber count", e); } @@ -86,14 +84,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getStreamCount() throws ParsingException { try { - final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText"); - - if (videoCountObject == null) { + if (!channelInfoItem.has("videoCountText")) { // Video count is not available, channel probably has no public uploads. return -1; } - return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject))); + return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( + channelInfoItem.getObject("videoCountText")))); } catch (Exception e) { throw new ParsingException("Could not get stream count", e); } @@ -102,14 +99,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getDescription() throws ParsingException { try { - final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet"); - - if (descriptionObject == null) { + if (!channelInfoItem.has("descriptionSnippet")) { // Channel have no description. return null; } - return getTextFromObject(descriptionObject); + return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); } catch (Exception e) { throw new ParsingException("Could not get description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java new file mode 100644 index 00000000..4f7847b2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -0,0 +1,486 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; + +public class YoutubeMusicSearchExtractor extends SearchExtractor { + private JsonObject initialData; + + public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + + final String params; + + switch (getLinkHandler().getContentFilters().get(0)) { + case MUSIC_SONGS: + params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_VIDEOS: + params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ALBUMS: + params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_PLAYLISTS: + params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ARTISTS: + params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; + break; + default: + params = null; + break; + } + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", getSearchString()) + .value("params", params) + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json)); + + try { + initialData = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return super.getUrl(); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("didYouMeanRenderer"); + if (!didYouMeanRenderer.has("correctedQuery")) { + return ""; + } + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).has("musicShelfRenderer")) { + collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); + } + } + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws ExtractionException, IOException { + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).has("musicShelfRenderer")) { + return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); + } + } + + return ""; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final String responseBody = getValidJsonResponseBody(getDownloader().post(pageUrl, headers, json)); + + final JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + + final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); + + collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); + final JsonArray continuations = musicShelfContinuation.getArray("continuations"); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + } + + private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) { + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object item : videos) { + final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null); + if (info != null) { + final String searchType = getLinkHandler().getContentFilters().get(0); + if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { + collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public long getDuration() throws ParsingException { + final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (duration != null && !duration.isEmpty()) { + return YoutubeParsingHelper.parseDurationString(duration); + } + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getUploaderUrl() throws ParsingException { + if (searchType.equals(MUSIC_VIDEOS)) { + JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); + for (Object item : items) { + final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer.getObject("icon").getString("iconType", EMPTY_STRING).equals("ARTIST")) { + return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); + } + } + + return null; + } else { + final JsonObject navigationEndpointHolder = info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0); + + if (!navigationEndpointHolder.has("navigationEndpoint")) return null; + + final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint")); + + if (url != null && !url.isEmpty()) { + return url; + } + + throw new ParsingException("Could not get uploader URL"); + } + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) { + return -1; + } + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get view count"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } else if (searchType.equals(MUSIC_ARTISTS)) { + collector.commit(new YoutubeChannelInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public String getDescription() { + return null; + } + }); + } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { + collector.commit(new YoutubePlaylistInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } else { + name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) { + return ITEM_COUNT_UNKNOWN; + } + final String count = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (count != null && !count.isEmpty()) { + if (count.contains("100+")) { + return ITEM_COUNT_MORE_THAN_100; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } + throw new ParsingException("Could not get count"); + } + }); + } + } + } + } + + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { + if (continuations == null || continuations.isEmpty()) { + return ""; + } + + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + + return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index eef85aa9..68b53855 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -47,23 +47,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private JsonObject getUploaderInfo() throws ParsingException { JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items"); - try { - JsonObject uploaderInfo = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer") - .getObject("videoOwner").getObject("videoOwnerRenderer"); - if (uploaderInfo != null) { - return uploaderInfo; - } - } catch (Exception ignored) {} + + JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); + if (videoOwner.has("videoOwnerRenderer")) { + return videoOwner.getObject("videoOwnerRenderer"); + } // we might want to create a loop here instead of using duplicated code - try { - JsonObject uploaderInfo = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer") - .getObject("videoOwner").getObject("videoOwnerRenderer"); - if (uploaderInfo != null) { - return uploaderInfo; - } - } catch (Exception e) { - throw new ParsingException("Could not get uploader info", e); + videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); + if (videoOwner.has("videoOwnerRenderer")) { + return videoOwner.getObject("videoOwnerRenderer"); } throw new ParsingException("Could not get uploader info"); } @@ -89,33 +82,22 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public String getName() throws ParsingException { - try { - String name = getTextFromObject(playlistInfo.getObject("title")); - if (name != null) return name; - } catch (Exception ignored) {} - try { - return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); - } catch (Exception e) { - throw new ParsingException("Could not get playlist name", e); - } + String name = getTextFromObject(playlistInfo.getObject("title")); + if (name != null && !name.isEmpty()) return name; + + return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); } @Override public String getThumbnailUrl() throws ParsingException { - String url = null; + String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - try { - url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} + if (url == null || url.isEmpty()) { + url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") + .getArray("thumbnails").getObject(0).getString("url"); - if (url == null) { - try { - url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") - .getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} - - if (url == null) throw new ParsingException("Could not get playlist thumbnail"); + if (url == null || url.isEmpty()) throw new ParsingException("Could not get playlist thumbnail"); } return fixThumbnailUrl(url); @@ -123,8 +105,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getBannerUrl() { - return ""; // Banner can't be handled by frontend right now. + // Banner can't be handled by frontend right now. // Whoever is willing to implement this should also implement it in the frontend. + return ""; } @Override @@ -199,7 +182,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null) { + if (continuations == null || continuations.isEmpty()) { return ""; } @@ -216,7 +199,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object video : videos) { - if (((JsonObject) video).getObject("playlistVideoRenderer") != null) { + if (((JsonObject) video).has("playlistVideoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { @Override public long getViewCount() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 27a24709..633cfda6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -43,12 +43,12 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP public class YoutubeSearchExtractor extends SearchExtractor { private JsonObject initialData; - public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); @@ -64,23 +64,23 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { - JsonObject showingResultsForRenderer = initialData.getObject("contents") + final JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (showingResultsForRenderer == null) { + if (!showingResultsForRenderer.has("correctedQuery")) { return ""; - } else { - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + + final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); for (Object section : sections) { @@ -98,7 +98,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -106,41 +106,41 @@ public class YoutubeSearchExtractor extends SearchExtractor { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("itemSectionContinuation"); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + final JsonArray continuations = itemSectionRenderer.getArray("continuations"); - return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations"))); + return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); } - private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { - collector.reset(); - + private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { - if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { + if (((JsonObject) item).has("backgroundPromoRenderer")) { throw new NothingFoundException(getTextFromObject(((JsonObject) item) .getObject("backgroundPromoRenderer").getObject("bodyText"))); - } else if (((JsonObject) item).getObject("videoRenderer") != null) { + } else if (((JsonObject) item).has("videoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); - } else if (((JsonObject) item).getObject("channelRenderer") != null) { + } else if (((JsonObject) item).has("channelRenderer")) { collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer"))); - } else if (((JsonObject) item).getObject("playlistRenderer") != null) { + } else if (((JsonObject) item).has("playlistRenderer")) { collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); } } } - private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException { - if (continuations == null) { + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { + if (continuations == null || continuations.isEmpty()) { return ""; } - JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - String continuation = nextContinuationData.getString("continuation"); - String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + "&itct=" + clickTrackingParams; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f6b0750c..100e6dca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -33,7 +33,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -57,6 +56,7 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; /* * Created by Christian Schabesberger on 06.08.15. @@ -117,18 +117,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getName() throws ParsingException { assertPageFetched(); - String title = null; + String title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - try { - title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - } catch (Exception ignored) {} + if (title == null || title.isEmpty()) { + title = playerResponse.getObject("videoDetails").getString("title"); - if (title == null) { - try { - title = playerResponse.getObject("videoDetails").getString("title"); - } catch (Exception ignored) {} - - if (title == null) throw new ParsingException("Could not get name"); + if (title == null || title.isEmpty()) throw new ParsingException("Could not get name"); } return title; @@ -140,35 +134,31 @@ public class YoutubeStreamExtractor extends StreamExtractor { return null; } - try { - JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); - if (micro.getString("uploadDate") != null && !micro.getString("uploadDate").isEmpty()) { - return micro.getString("uploadDate"); - } - if (micro.getString("publishDate") != null && !micro.getString("publishDate").isEmpty()) { - return micro.getString("publishDate"); - } - } catch (Exception ignored) {} + JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); + if (micro.isString("uploadDate") && !micro.getString("uploadDate").isEmpty()) { + return micro.getString("uploadDate"); + } + if (micro.isString("publishDate") && !micro.getString("publishDate").isEmpty()) { + return micro.getString("publishDate"); + } + + if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { + String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); + + try { // Premiered 20 hours ago + TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); + Calendar parsedTime = timeAgoParser.parse(time).date(); + return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); + } catch (Exception ignored) {} + + try { // Premiered Feb 21, 2020 + Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); + return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); + } catch (Exception ignored) {} + } try { - if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { - String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); - - try { // Premiered 20 hours ago - TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); - Calendar parsedTime = timeAgoParser.parse(time).date(); - return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); - } catch (Exception ignored) {} - - try { // Premiered Feb 21, 2020 - Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); - return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); - } catch (Exception ignored) {} - } - } catch (Exception ignored) {} - - try { - // TODO this parses English formatted dates only, we need a better approach to parse the textual date + // TODO: this parses English formatted dates only, we need a better approach to parse the textual date Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); return new SimpleDateFormat("yyyy-MM-dd").format(d); @@ -180,7 +170,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public DateWrapper getUploadDate() throws ParsingException { final String textualUploadDate = getTextualUploadDate(); - if (textualUploadDate == null) { + if (textualUploadDate == null || textualUploadDate.isEmpty()) { return null; } @@ -208,17 +198,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { public Description getDescription() throws ParsingException { assertPageFetched(); // description with more info on links - try { - String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); - return new Description(description, Description.HTML); - } catch (Exception ignored) { } + String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); + if (description != null && !description.isEmpty()) return new Description(description, Description.HTML); // raw non-html description - try { - return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); - } catch (Exception ignored) { - throw new ParsingException("Could not get description"); - } + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); } @Override @@ -264,19 +248,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getViewCount() throws ParsingException { assertPageFetched(); - String views = null; - - try { - views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") + String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") .getObject("videoViewCountRenderer").getObject("viewCount")); - } catch (Exception ignored) {} - if (views == null) { - try { - views = playerResponse.getObject("videoDetails").getString("viewCount"); - } catch (Exception ignored) {} + if (views == null || views.isEmpty()) { + views = playerResponse.getObject("videoDetails").getString("viewCount"); - if (views == null) throw new ParsingException("Could not get view count"); + if (views == null || views.isEmpty()) throw new ParsingException("Could not get view count"); } if (views.toLowerCase().contains("no views")) return 0; @@ -334,16 +312,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderUrl() throws ParsingException { assertPageFetched(); - try { + String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer() .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint")); - if (uploaderUrl != null) return uploaderUrl; - } catch (Exception ignored) {} - try { - String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); - if (uploaderId != null) - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); - } catch (Exception ignored) {} + if (uploaderUrl != null && !uploaderUrl.isEmpty()) return uploaderUrl; + + + String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); + if (uploaderId != null && !uploaderId.isEmpty()) + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); + throw new ParsingException("Could not get uploader url"); } @@ -351,19 +329,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderName() throws ParsingException { assertPageFetched(); - String uploaderName = null; - - try { - uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") + String uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") .getObject("videoOwnerRenderer").getObject("title")); - } catch (Exception ignored) {} - if (uploaderName == null) { - try { - uploaderName = playerResponse.getObject("videoDetails").getString("author"); - } catch (Exception ignored) {} + if (uploaderName == null || uploaderName.isEmpty()) { + uploaderName = playerResponse.getObject("videoDetails").getString("author"); - if (uploaderName == null) throw new ParsingException("Could not get uploader name"); + if (uploaderName == null || uploaderName.isEmpty()) throw new ParsingException("Could not get uploader name"); } return uploaderName; @@ -392,7 +364,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (videoInfoPage.containsKey("dashmpd")) { dashManifestUrl = videoInfoPage.get("dashmpd"); } else if (playerArgs != null && playerArgs.isString("dashmpd")) { - dashManifestUrl = playerArgs.getString("dashmpd", ""); + dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING); } else { return ""; } @@ -561,9 +533,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object ul : results) { - final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer"); - - if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); + if (((JsonObject) ul).has("compactVideoRenderer")) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul).getObject("compactVideoRenderer"), timeAgoParser)); + } } return collector; } catch (Exception e) { @@ -612,7 +584,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String playerUrl; - if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos + if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos initialData = initialAjaxJson.getObject(2).getObject("response"); ageLimit = 18; @@ -631,7 +603,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerResponse = getPlayerResponse(); - final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.EMPTY_OBJECT); + final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); final String status = playabilityStatus.getString("status"); // If status exist, and is not "OK", throw a ContentNotAvailableException with the reason. if (status != null && !status.toLowerCase().equals("ok")) { @@ -808,10 +780,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } captions = playerResponse.getObject("captions"); - final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject()); - final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray()); + final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer"); + final JsonArray captionsArray = renderer.getArray("captionTracks"); // todo: use this to apply auto translation to different language from a source language -// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray()); +// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages"); // This check is necessary since there may be cases where subtitles metadata do not contain caption track info // e.g. https://www.youtube.com/watch?v=-Vpwatutnko @@ -876,7 +848,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { JsonObject videoPrimaryInfoRenderer = null; for (Object content : contents) { - if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) { + if (((JsonObject) content).has("videoPrimaryInfoRenderer")) { videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); break; } @@ -898,7 +870,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { JsonObject videoSecondaryInfoRenderer = null; for (Object content : contents) { - if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) { + if (((JsonObject) content).has("videoSecondaryInfoRenderer")) { videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer"); break; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 2330130e..77850dd8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -11,12 +12,16 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nullable; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; /* * Copyright (C) Christian Schabesberger 2016 @@ -37,7 +42,6 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP */ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - private JsonObject videoInfo; private final TimeAgoParser timeAgoParser; private StreamType cachedStreamType; @@ -59,23 +63,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return cachedStreamType; } - try { - JsonArray badges = videoInfo.getArray("badges"); - for (Object badge : badges) { - if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { - return cachedStreamType = StreamType.LIVE_STREAM; - } - } - - } catch (Exception ignored) {} - - try { - final String style = videoInfo.getArray("thumbnailOverlays").getObject(0) - .getObject("thumbnailOverlayTimeStatusRenderer").getString("style"); - if (style.equalsIgnoreCase("LIVE")) { + final JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("LIVE NOW")) { return cachedStreamType = StreamType.LIVE_STREAM; } - } catch (Exception ignored) {} + } + + final String style = videoInfo.getArray("thumbnailOverlays").getObject(0) + .getObject("thumbnailOverlayTimeStatusRenderer").getString("style", EMPTY_STRING); + if (style.equalsIgnoreCase("LIVE")) { + return cachedStreamType = StreamType.LIVE_STREAM; + } return cachedStreamType = StreamType.VIDEO_STREAM; } @@ -108,23 +107,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return -1; } - String duration = null; + String duration = getTextFromObject(videoInfo.getObject("lengthText")); - try { - duration = getTextFromObject(videoInfo.getObject("lengthText")); - } catch (Exception ignored) {} - - if (duration == null) { - try { - for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { - if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) { - duration = getTextFromObject(((JsonObject) thumbnailOverlay) - .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); - } + if (duration == null || duration.isEmpty()) { + for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { + if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) { + duration = getTextFromObject(((JsonObject) thumbnailOverlay) + .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); } - } catch (Exception ignored) {} + } - if (duration == null) throw new ParsingException("Could not get duration"); + if (duration == null || duration.isEmpty()) throw new ParsingException("Could not get duration"); } return YoutubeParsingHelper.parseDurationString(duration); @@ -132,23 +125,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderName() throws ParsingException { - String name = null; + String name = getTextFromObject(videoInfo.getObject("longBylineText")); - try { - name = getTextFromObject(videoInfo.getObject("longBylineText")); - } catch (Exception ignored) {} + if (name == null || name.isEmpty()) { + name = getTextFromObject(videoInfo.getObject("ownerText")); - if (name == null) { - try { - name = getTextFromObject(videoInfo.getObject("ownerText")); - } catch (Exception ignored) {} + if (name == null || name.isEmpty()) { + name = getTextFromObject(videoInfo.getObject("shortBylineText")); - if (name == null) { - try { - name = getTextFromObject(videoInfo.getObject("shortBylineText")); - } catch (Exception ignored) {} - - if (name == null) throw new ParsingException("Could not get uploader name"); + if (name == null || name.isEmpty()) throw new ParsingException("Could not get uploader name"); } } @@ -157,26 +142,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderUrl() throws ParsingException { - String url = null; + String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); - try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + if (url == null || url.isEmpty()) { + url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); - } catch (Exception ignored) {} - if (url == null) { - try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") + if (url == null || url.isEmpty()) { + url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); - } catch (Exception ignored) {} - if (url == null) { - try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - } catch (Exception ignored) {} - - if (url == null) throw new ParsingException("Could not get uploader url"); + if (url == null || url.isEmpty()) throw new ParsingException("Could not get uploader url"); } } @@ -195,12 +172,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date); } - try { - return getTextFromObject(videoInfo.getObject("publishedTimeText")); - } catch (Exception e) { - // upload date is not always available, e.g. in playlists - return null; - } + final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); + if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText; + + return null; } @Nullable @@ -228,17 +203,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getViewCount() throws ParsingException { try { - if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { + if (videoInfo.has("topStandaloneBadge") || isPremium()) { return -1; } - final JsonObject viewCountObject = videoInfo.getObject("viewCountText"); - if (viewCountObject == null) { + if (!videoInfo.has("viewCountText")) { // This object is null when a video has its views hidden. return -1; } - final String viewCount = getTextFromObject(viewCountObject); + final String viewCount = getTextFromObject(videoInfo.getObject("viewCountText")); if (viewCount.toLowerCase().contains("no views")) { return 0; @@ -266,14 +240,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } private boolean isPremium() { - try { - JsonArray badges = videoInfo.getArray("badges"); - for (Object badge : badges) { - if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("Premium")) { - return true; - } + JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("Premium")) { + return true; } - } catch (Exception ignored) { } return false; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index 8ed35c19..2c4c74be 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -59,7 +59,7 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { // trim JSONP part "JP(...)" response = response.substring(3, response.length() - 1); try { - JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray()); + JsonArray collection = JsonParser.array().from(response).getArray(1); for (Object suggestion : collection) { if (!(suggestion instanceof JsonArray)) continue; String suggestionStr = ((JsonArray) suggestion).getString(0); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index e15463b7..958af8b3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -72,12 +72,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Nonnull @Override public String getName() throws ParsingException { - String name; - try { - name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")); - } catch (Exception e) { - throw new ParsingException("Could not get Trending name", e); - } + String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")); if (name != null && !name.isEmpty()) { return name; } @@ -97,14 +92,11 @@ public class YoutubeTrendingExtractor extends KioskExtractor { JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer") .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") .getObject("expandedShelfContentsRenderer"); - if (expandedShelfContentsRenderer != null) { - for (Object ul : expandedShelfContentsRenderer.getArray("items")) { - final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); - collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); - } + for (Object ul : expandedShelfContentsRenderer.getArray("items")) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); } } return new InfoItemsPage<>(collector, getNextPageUrl()); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 96532555..da0decfa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -5,6 +5,8 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; @@ -18,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.text.ParseException; @@ -25,6 +28,7 @@ import java.text.SimpleDateFormat; import java.util.*; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @@ -62,6 +66,9 @@ public class YoutubeParsingHelper { private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; + private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"}; + private static String[] youtubeMusicKeys; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -196,11 +203,7 @@ public class YoutubeParsingHelper { */ public static String getClientVersion() throws IOException, ExtractionException { if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - - if (isHardcodedClientVersionValid()) { - clientVersion = HARDCODED_CLIENT_VERSION; - return clientVersion; - } + if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION; final String url = "https://www.youtube.com/results?search_query=test"; final String html = getDownloader().get(url).responseBody(); @@ -217,8 +220,7 @@ public class YoutubeParsingHelper { JsonObject p = (JsonObject) param; String key = p.getString("key"); if (key != null && key.equals("cver")) { - clientVersion = p.getString("value"); - return clientVersion; + return clientVersion = p.getString("value"); } } } else if (s.getString("service").equals("ECATCHER")) { @@ -244,23 +246,96 @@ public class YoutubeParsingHelper { try { contextClientVersion = Parser.matchGroup1(pattern, html); if (contextClientVersion != null && !contextClientVersion.isEmpty()) { - clientVersion = contextClientVersion; - return clientVersion; + return clientVersion = contextClientVersion; } } catch (Exception ignored) { } } if (shortClientVersion != null) { - clientVersion = shortClientVersion; - return clientVersion; + return clientVersion = shortClientVersion; } throw new ParsingException("Could not get client version"); } + public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException { + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0]; + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEYS[2]) + .value("hl", "en") + .value("gl", "GB") + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", "test") + .value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D") + .end().done().getBytes("UTF-8"); + // @formatter:on + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + String response = getDownloader().post(url, headers, json).responseBody(); + + return response.length() > 50; // ensure to have a valid response + } + + public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException { + if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys; + if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS; + + final String url = "https://music.youtube.com/"; + final String html = getDownloader().get(url).responseBody(); + + String key; + try { + key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); + } catch (Parser.RegexException e) { + key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html); + } + + final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html); + + String clientVersion; + try { + clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + } catch (Parser.RegexException e) { + try { + clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); + } catch (Parser.RegexException ee) { + clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html); + } + } + + return youtubeMusicKeys = new String[]{key, clientName, clientVersion}; + } + public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { - if (navigationEndpoint.getObject("urlEndpoint") != null) { + if (navigationEndpoint.has("urlEndpoint")) { String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); if (internUrl.startsWith("/redirect?")) { // q parameter can be the first parameter @@ -280,7 +355,7 @@ public class YoutubeParsingHelper { } else if (internUrl.startsWith("http")) { return internUrl; } - } else if (navigationEndpoint.getObject("browseEndpoint") != null) { + } else if (navigationEndpoint.has("browseEndpoint")) { final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl"); final String browseId = browseEndpoint.getString("browseId"); @@ -295,7 +370,7 @@ public class YoutubeParsingHelper { } throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")"); - } else if (navigationEndpoint.getObject("watchEndpoint") != null) { + } else if (navigationEndpoint.has("watchEndpoint")) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) @@ -303,17 +378,30 @@ public class YoutubeParsingHelper { if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); return url.toString(); + } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { + return "https://www.youtube.com/playlist?list=" + + navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId"); } return null; } + /** + * Get the text from a JSON object that has either a simpleText or a runs array. + * @param textObject JSON object to get the text from + * @param html whether to return HTML, by parsing the navigationEndpoint + * @return text in the JSON object or {@code null} + */ public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException { + if (textObject == null || textObject.isEmpty()) return null; + if (textObject.has("simpleText")) return textObject.getString("simpleText"); + if (textObject.getArray("runs").isEmpty()) return null; + StringBuilder textBuilder = new StringBuilder(); for (Object textPart : textObject.getArray("runs")) { String text = ((JsonObject) textPart).getString("text"); - if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) { + if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (url != null && !url.isEmpty()) { textBuilder.append("").append(text).append(""); @@ -351,12 +439,8 @@ public class YoutubeParsingHelper { return thumbnailUrl; } - public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); - final Response response = getDownloader().get(url, headers, localization); - + public static String getValidJsonResponseBody(final Response response) + throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { throw new ContentNotAvailableException("Not found" + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); @@ -377,11 +461,24 @@ public class YoutubeParsingHelper { } final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + if (responseContentType != null + && responseContentType.toLowerCase().contains("text/html")) { throw new ParsingException("Got HTML document, expected JSON response" + " (latest url was: \"" + response.latestUrl() + "\")"); } + return responseBody; + } + + public static JsonArray getJsonResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + final Response response = getDownloader().get(url, headers, localization); + + final String responseBody = getValidJsonResponseBody(response); + try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { @@ -393,16 +490,17 @@ public class YoutubeParsingHelper { * Shared alert detection function, multiple endpoints return the error similarly structured. *

* Will check if the object has an alert of the type "ERROR". + *

* * @param initialData the object which will be checked if an alert is present * @throws ContentNotAvailableException if an alert is detected */ - public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException { + public static void defaultAlertsCheck(final JsonObject initialData) throws ParsingException { final JsonArray alerts = initialData.getArray("alerts"); - if (alerts != null && !alerts.isEmpty()) { + if (!alerts.isEmpty()) { final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); - final String alertText = alertRenderer.getObject("text").getString("simpleText"); - final String alertType = alertRenderer.getString("type"); + final String alertText = getTextFromObject(alertRenderer.getObject("text")); + final String alertType = alertRenderer.getString("type", EMPTY_STRING); if (alertType.equalsIgnoreCase("ERROR")) { throw new ContentNotAvailableException("Got error: \"" + alertText + "\""); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index cf51281b..1950f697 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -47,7 +48,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { - throw new ParsingException("YouTube Mix playlists are not yet supported"); + throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported"); } return listID; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 13481b34..866c1856 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -8,13 +8,21 @@ import java.net.URLEncoder; import java.util.List; public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - public static final String CHARSET_UTF_8 = "UTF-8"; + public static final String ALL = "all"; public static final String VIDEOS = "videos"; public static final String CHANNELS = "channels"; public static final String PLAYLISTS = "playlists"; - public static final String ALL = "all"; + + public static final String MUSIC_SONGS = "music_songs"; + public static final String MUSIC_VIDEOS = "music_videos"; + public static final String MUSIC_ALBUMS = "music_albums"; + public static final String MUSIC_PLAYLISTS = "music_playlists"; + public static final String MUSIC_ARTISTS = "music_artists"; + + private static final String SEARCH_URL = "https://www.youtube.com/results?search_query="; + private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q="; public static YoutubeSearchQueryHandlerFactory getInstance() { return new YoutubeSearchQueryHandlerFactory(); @@ -23,20 +31,27 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { try { - final String url = "https://www.youtube.com/results" - + "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8); - if (contentFilters.size() > 0) { switch (contentFilters.get(0)) { - case VIDEOS: return url + "&sp=EgIQAQ%253D%253D"; - case CHANNELS: return url + "&sp=EgIQAg%253D%253D"; - case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D"; case ALL: default: + break; + case VIDEOS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D"; + case CHANNELS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D"; + case PLAYLISTS: + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D"; + case MUSIC_SONGS: + case MUSIC_VIDEOS: + case MUSIC_ALBUMS: + case MUSIC_PLAYLISTS: + case MUSIC_ARTISTS: + return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); } } - return url; + return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8); } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } @@ -48,6 +63,12 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory ALL, VIDEOS, CHANNELS, - PLAYLISTS}; + PLAYLISTS, + MUSIC_SONGS, + MUSIC_VIDEOS, + MUSIC_ALBUMS, + MUSIC_PLAYLISTS +// MUSIC_ARTISTS + }; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 580f9c21..0e5ff080 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.utils.DashMpdParser; @@ -47,7 +48,7 @@ public class StreamInfo extends Info { } public StreamInfo(int serviceId, String url, String originalUrl, StreamType streamType, String id, String name, - int ageLimit) { + int ageLimit) { super(serviceId, id, url, originalUrl, name); this.streamType = streamType; this.ageLimit = ageLimit; @@ -131,6 +132,8 @@ public class StreamInfo extends Info { /* Load and extract audio */ try { streamInfo.setAudioStreams(extractor.getAudioStreams()); + } catch (ContentNotSupportedException e) { + throw e; } catch (Exception e) { streamInfo.addError(new ExtractionException("Couldn't get audio streams", e)); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index b34bbbff..a5269841 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -37,11 +37,14 @@ public final class DefaultTests { if (item instanceof StreamInfoItem) { StreamInfoItem streamInfoItem = (StreamInfoItem) item; assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName()); - assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); - assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + +// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); + if (streamInfoItem.getUploaderUrl() != null && !streamInfoItem.getUploaderUrl().isEmpty()) { + assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); + } assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); - assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); final String textualUploadDate = streamInfoItem.getTextualUploadDate(); if (textualUploadDate != null && !textualUploadDate.isEmpty()) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index 258a55b0..98b6203c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -14,7 +14,6 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; * Test {@link MediaCCCConferenceExtractor} */ public class MediaCCCConferenceExtractorTest { - public static class FrOSCon2017 { private static MediaCCCConferenceExtractor extractor; @@ -32,7 +31,7 @@ public class MediaCCCConferenceExtractorTest { @Test public void testGetUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); + assertEquals("https://media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); } @Test @@ -68,7 +67,7 @@ public class MediaCCCConferenceExtractorTest { @Test public void testGetUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/oscal19", extractor.getUrl()); + assertEquals("https://media.ccc.de/public/conferences/oscal19", extractor.getUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java index 25116af3..21a49da3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java @@ -15,14 +15,14 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; * Test {@link MediaCCCStreamExtractor} */ public class MediaCCCOggTest { - // test against https://api.media.ccc.de/public/events/1317 + // test against https://media.ccc.de/public/events/1317 private static StreamExtractor extractor; @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); + extractor = MediaCCC.getStreamExtractor("https://media.ccc.de/public/events/1317"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 8120e07a..95c88229 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.VideoStream; -import org.schabi.newpipe.extractor.utils.UtilsTest; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -25,7 +24,6 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; * Test {@link MediaCCCStreamExtractor} */ public class MediaCCCStreamExtractorTest { - public static class Gpn18Tmux { private static MediaCCCStreamExtractor extractor; @@ -55,7 +53,7 @@ public class MediaCCCStreamExtractorTest { @Test public void testUrl() throws Exception { assertIsSecureUrl(extractor.getUrl()); - assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); + assertEquals("https://media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); } @Test @@ -78,7 +76,7 @@ public class MediaCCCStreamExtractorTest { @Test public void testUploaderUrl() throws Exception { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); + assertEquals("https://media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); } @Test @@ -141,7 +139,7 @@ public class MediaCCCStreamExtractorTest { @Test public void testUrl() throws Exception { assertIsSecureUrl(extractor.getUrl()); - assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); + assertEquals("https://media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); } @Test @@ -164,7 +162,7 @@ public class MediaCCCStreamExtractorTest { @Test public void testUploaderUrl() throws Exception { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); + assertEquals("https://media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); } @Test @@ -203,4 +201,4 @@ public class MediaCCCStreamExtractorTest { assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } } -} \ No newline at end of file +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java index 812d6367..50e0021c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/search/MediaCCCSearchExtractorTest.java @@ -16,7 +16,6 @@ import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaC import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS; public class MediaCCCSearchExtractorTest { - public static class All extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "kde"; @@ -32,8 +31,8 @@ public class MediaCCCSearchExtractorTest { @Override public StreamingService expectedService() { return MediaCCC; } @Override public String expectedName() { return QUERY; } @Override public String expectedId() { return QUERY; } - @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } - @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } @@ -55,8 +54,8 @@ public class MediaCCCSearchExtractorTest { @Override public StreamingService expectedService() { return MediaCCC; } @Override public String expectedName() { return QUERY; } @Override public String expectedId() { return QUERY; } - @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } - @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } @@ -79,8 +78,8 @@ public class MediaCCCSearchExtractorTest { @Override public StreamingService expectedService() { return MediaCCC; } @Override public String expectedName() { return QUERY; } @Override public String expectedId() { return QUERY; } - @Override public String expectedUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } - @Override public String expectedOriginalUrlContains() { return "api.media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "media.ccc.de/public/events/search?q=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java new file mode 100644 index 00000000..fd944f49 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -0,0 +1,205 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + +/** + * Test for {@link PeertubeAccountExtractor} + */ +public class PeertubeAccountExtractorTest { + public static class KDE implements BaseChannelExtractorTest { + private static PeertubeAccountExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeAccountExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("The KDE Community", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("accounts/kde", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + } + } + + public static class Booteille implements BaseChannelExtractorTest { + private static PeertubeAccountExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeAccountExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Additional Testing + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testGetPageInNewExtractor() throws Exception { + final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("booteille", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("accounts/booteille", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 46109559..9dc4b013 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; @@ -20,7 +19,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link PeertubeChannelExtractor} */ public class PeertubeChannelExtractorTest { - public static class KDE implements BaseChannelExtractorTest { + public static class DanDAugeTutoriels implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -29,7 +28,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + .getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"); extractor.fetchPage(); } @@ -44,22 +43,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("The KDE Community", extractor.getName()); + assertEquals("Dan d'Auge tutoriels", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("kde", extractor.getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -98,16 +97,16 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1361", extractor.getFeedUrl()); } @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 4); } } - public static class Booteille implements BaseChannelExtractorTest { + public static class Divers implements BaseChannelExtractorTest { private static PeertubeChannelExtractor extractor; @BeforeClass @@ -116,7 +115,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + .getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457"); extractor.fetchPage(); } @@ -141,22 +140,22 @@ public class PeertubeChannelExtractorTest { @Test public void testName() throws ParsingException { - assertEquals("booteille", extractor.getName()); + assertEquals("Divers", extractor.getName()); } @Test public void testId() throws ParsingException { - assertEquals("booteille", extractor.getId()); + assertEquals("video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -195,12 +194,12 @@ public class PeertubeChannelExtractorTest { @Test public void testFeedUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl()); + assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1227", extractor.getFeedUrl()); } @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index f9b0c69d..d47dc6f6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChanne import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; /** * Test for {@link PeertubeChannelLinkHandlerFactory} @@ -19,6 +20,7 @@ public class PeertubeChannelLinkHandlerFactoryTest { @BeforeClass public static void setUp() { + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); linkHandler = PeertubeChannelLinkHandlerFactory.getInstance(); NewPipe.init(DownloaderTestImpl.getInstance()); } @@ -26,11 +28,20 @@ public class PeertubeChannelLinkHandlerFactoryTest { @Test public void acceptUrlTest() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa")); } @Test public void getIdFromUrl() throws ParsingException { - assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); - assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); + assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId()); + } + + @Test + public void getUrlFromId() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("kranti@videos.squat.net").getUrl()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 86203f4c..75178c4c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -5,6 +5,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -25,107 +26,134 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; * Test for {@link StreamExtractor} */ public class SoundcloudStreamExtractorDefaultTest { - private static SoundcloudStreamExtractor extractor; - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); - extractor.fetchPage(); + public static class LilUziVertDoWhatIWant { + private static SoundcloudStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetValidTimeStamp() throws IOException, ExtractionException { + StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69"); + assertEquals("69", extractor.getTimeStamp() + ""); + } + + @Test + public void testGetTitle() throws ParsingException { + assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertEquals("LIL UZI VERT", extractor.getUploaderName()); + } + + @Test + public void testGetLength() throws ParsingException { + assertEquals(175, extractor.getLength()); + } + + @Test + public void testGetViewCount() throws ParsingException { + assertTrue(Long.toString(extractor.getViewCount()), + extractor.getViewCount() > 44227978); + } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse("2016/07/31 18:18:07 +0000")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetAudioStreams() throws IOException, ExtractionException { + assertFalse(extractor.getAudioStreams().isEmpty()); + } + + @Test + public void testStreamType() throws ParsingException { + assertTrue(extractor.getStreamType() == StreamType.AUDIO_STREAM); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } } - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } + public static class ContentNotSupported { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } - @Test - public void testGetValidTimeStamp() throws IOException, ExtractionException { - StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69"); - assertEquals("69", extractor.getTimeStamp() + ""); - } + @Test(expected = ContentNotSupportedException.class) + public void hlsAudioStream() throws Exception { + final StreamExtractor extractor = + SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool"); + extractor.fetchPage(); + extractor.getAudioStreams(); + } - @Test - public void testGetTitle() throws ParsingException { - assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertEquals("LIL UZI VERT", extractor.getUploaderName()); - } - - @Test - public void testGetLength() throws ParsingException { - assertEquals(175, extractor.getLength()); - } - - @Test - public void testGetViewCount() throws ParsingException { - assertTrue(Long.toString(extractor.getViewCount()), - extractor.getViewCount() > 44227978); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse("2016/07/31 18:18:07 +0000")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - } - - @Test - public void testGetUploaderUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl()); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws IOException, ExtractionException { - assertFalse(extractor.getAudioStreams().isEmpty()); - } - - @Test - public void testStreamType() throws ParsingException { - assertTrue(extractor.getStreamType() == StreamType.AUDIO_STREAM); - } - - @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } - - @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + @Test(expected = ContentNotSupportedException.class) + public void bothHlsAndOpusAudioStreams() throws Exception { + final StreamExtractor extractor = + SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker"); + extractor.fetchPage(); + extractor.getAudioStreams(); + } } } + diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 3615bc63..a916c560 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -5,13 +5,14 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; -import org.schabi.newpipe.extractor.channel.ChannelInfo; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; -import java.util.List; +import java.io.IOException; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; @@ -45,6 +46,20 @@ public class YoutubeChannelExtractorTest { } } + public static class NotSupported { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotSupportedException.class) + public void noVideoTab() throws Exception { + final ChannelExtractor extractor = YouTube.getChannelExtractor("https://invidio.us/channel/UC-9-kyTW8ZkZNDHQJ6FgpwQ"); + extractor.fetchPage(); + extractor.getInitialPage(); + } + } + public static class Gronkh implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 87dbbd75..669bc329 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -22,4 +22,10 @@ public class YoutubeParsingHelperTest { assertTrue("Hardcoded client version is not valid anymore", YoutubeParsingHelper.isHardcodedClientVersionValid()); } + + @Test + public void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException { + assertTrue("Hardcoded YouTube Music keys are not valid anymore", + YoutubeParsingHelper.areHardcodedYoutubeMusicKeysValid()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java new file mode 100644 index 00000000..420db0ad --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -0,0 +1,153 @@ +package org.schabi.newpipe.extractor.services.youtube.search; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; + +import java.net.URLEncoder; + +import javax.annotation.Nullable; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeMusicSearchExtractorTest { + public static class MusicSongs extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "mocromaniac"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class MusicVideos extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "fresku"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class MusicAlbums extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "johnny sellah"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + public static class MusicPlaylists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "louivos"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; } + } + + @Ignore + public static class MusicArtists extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "kevin"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; } + } + + public static class Suggestion extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "megaman x3"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index fc6af4c4..b558ad88 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -16,6 +16,12 @@ public class YoutubeSearchQHTest { assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + + assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl()); + assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl()); } @Test @@ -24,6 +30,9 @@ public class YoutubeSearchQHTest { .fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0)); assertEquals(CHANNELS, YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0)); + + assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0)); } @Test @@ -36,16 +45,23 @@ public class YoutubeSearchQHTest { .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); + + assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl()); } @Test public void testGetAvailableContentFilter() { final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter(); - assertEquals(4, contentFilter.length); + assertEquals(8, contentFilter.length); assertEquals("all", contentFilter[0]); assertEquals("videos", contentFilter[1]); assertEquals("channels", contentFilter[2]); assertEquals("playlists", contentFilter[3]); + assertEquals("music_songs", contentFilter[4]); + assertEquals("music_videos", contentFilter[5]); + assertEquals("music_albums", contentFilter[6]); + assertEquals("music_playlists", contentFilter[7]); } @Test diff --git a/timeago-parser/build.gradle b/timeago-parser/build.gradle index bd99b48b..ff23db9e 100644 --- a/timeago-parser/build.gradle +++ b/timeago-parser/build.gradle @@ -1,6 +1,6 @@ dependencies { testImplementation 'junit:junit:4.12' - implementation 'com.grack:nanojson:1.1' + implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' -} \ No newline at end of file +}