From fc6b45ee363fc871108054df67638f0392e395c9 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sat, 12 Mar 2022 13:06:00 +0100 Subject: [PATCH 1/5] Implement some methods in PlaylistExtractor This will prevent their override in each child class where the values corresponding to the methods could not be extracted. --- .../extractor/playlist/PlaylistExtractor.java | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 683fcb62..f7866454 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -8,15 +8,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; + public abstract class PlaylistExtractor extends ListExtractor { - public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public PlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) { super(service, linkHandler); } - public abstract String getThumbnailUrl() throws ParsingException; - public abstract String getBannerUrl() throws ParsingException; - public abstract String getUploaderUrl() throws ParsingException; public abstract String getUploaderName() throws ParsingException; public abstract String getUploaderAvatarUrl() throws ParsingException; @@ -24,8 +23,30 @@ public abstract class PlaylistExtractor extends ListExtractor { public abstract long getStreamCount() throws ParsingException; - @Nonnull public abstract String getSubChannelName() throws ParsingException; - @Nonnull public abstract String getSubChannelUrl() throws ParsingException; - @Nonnull public abstract String getSubChannelAvatarUrl() throws ParsingException; + @Nonnull + public String getThumbnailUrl() throws ParsingException { + return EMPTY_STRING; + } + @Nonnull + public String getBannerUrl() throws ParsingException { + // Banner can't be handled by frontend right now. + // Whoever is willing to implement this should also implement it in the frontend. + return EMPTY_STRING; + } + + @Nonnull + public String getSubChannelName() throws ParsingException { + return EMPTY_STRING; + } + + @Nonnull + public String getSubChannelUrl() throws ParsingException { + return EMPTY_STRING; + } + + @Nonnull + public String getSubChannelAvatarUrl() throws ParsingException { + return EMPTY_STRING; + } } From 58a247907e66252483ddd33a55f13301d909867f Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sat, 12 Mar 2022 13:16:00 +0100 Subject: [PATCH 2/5] Apply changes in all playlist extractors except YoutubePlaylistExtractor Also fix some issues in the extractors, remove uneeded overrides, use the Java 8 Stream API where possible and replace usages of Utils.UTF_8 with StandardCharsets.UTF_8 in these classes. --- .../extractors/BandcampPlaylistExtractor.java | 45 ++++--------- .../extractors/PeertubePlaylistExtractor.java | 6 +- .../SoundcloudPlaylistExtractor.java | 57 +++++----------- .../YoutubeMixPlaylistExtractor.java | 67 +++++++------------ 4 files changed, 54 insertions(+), 121 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 8cc21a6e..7bd98b86 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -19,10 +19,13 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import java.util.Objects; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; public class BandcampPlaylistExtractor extends PlaylistExtractor { @@ -57,33 +60,27 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { throw new ParsingException("JSON does not exist", e); } - - - if (trackInfo.size() <= 0) { + if (trackInfo.isEmpty()) { // Albums without trackInfo need to be purchased before they can be played throw new ContentNotAvailableException("Album needs to be purchased"); } } + @Nonnull @Override public String getThumbnailUrl() throws ParsingException { if (albumJson.isNull("art_id")) { - return ""; + return EMPTY_STRING; } else { return getImageUrl(albumJson.getLong("art_id"), true); } } - @Override - public String getBannerUrl() { - return ""; - } - @Override public String getUploaderUrl() throws ParsingException { final String[] parts = getUrl().split("/"); // https: (/) (/) * .bandcamp.com (/) and leave out the rest - return "https://" + parts[2] + "/"; + return HTTPS + parts[2] + "/"; } @Override @@ -94,9 +91,10 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() { try { - return document.getElementsByClass("band-photo").first().attr("src"); - } catch (NullPointerException e) { - return ""; + return Objects.requireNonNull(document.getElementsByClass("band-photo").first()) + .attr("src"); + } catch (final NullPointerException e) { + return EMPTY_STRING; } } @@ -110,24 +108,6 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { return trackInfo.size(); } - @Nonnull - @Override - public String getSubChannelName() { - return ""; - } - - @Nonnull - @Override - public String getSubChannelUrl() { - return ""; - } - - @Nonnull - @Override - public String getSubChannelAvatarUrl() { - return ""; - } - @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { @@ -146,14 +126,13 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { collector.commit(new BandcampPlaylistStreamInfoItemExtractor( track, getUploaderUrl(), getThumbnailUrl())); } - } return new InfoItemsPage<>(collector, null); } @Override - public InfoItemsPage getPage(Page page) { + public InfoItemsPage getPage(final Page page) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 89340df3..f8fd38b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -29,16 +29,12 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { super(service, linkHandler); } + @Nonnull @Override public String getThumbnailUrl() throws ParsingException { return getBaseUrl() + playlistInfo.getString("thumbnailPath"); } - @Override - public String getBannerUrl() { - return null; - } - @Override public String getUploaderUrl() { return playlistInfo.getObject("ownerAccount").getString("url"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index 2c4ac60f..ffde3fb7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -23,9 +23,9 @@ import java.util.ArrayList; import java.util.List; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @@ -67,7 +67,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { return playlist.getString("title"); } - @Nullable + @Nonnull @Override public String getThumbnailUrl() { String artworkUrl = playlist.getString("artwork_url"); @@ -80,24 +80,21 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { for (final StreamInfoItem item : infoItems.getItems()) { artworkUrl = item.getThumbnailUrl(); - if (!isNullOrEmpty(artworkUrl)) break; + if (!isNullOrEmpty(artworkUrl)) { + break; + } } } catch (final Exception ignored) { } if (artworkUrl == null) { - return null; + return EMPTY_STRING; } } return artworkUrl.replace("large.jpg", "crop.jpg"); } - @Override - public String getBannerUrl() { - return null; - } - @Override public String getUploaderUrl() { return SoundcloudParsingHelper.getUploaderUrl(playlist); @@ -123,24 +120,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { return playlist.getLong("track_count"); } - @Nonnull - @Override - public String getSubChannelName() { - return ""; - } - - @Nonnull - @Override - public String getSubChannelUrl() { - return ""; - } - - @Nonnull - @Override - public String getSubChannelAvatarUrl() { - return ""; - } - @Nonnull @Override public InfoItemsPage getInitialPage() { @@ -149,18 +128,18 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { final List ids = new ArrayList<>(); final JsonArray tracks = playlist.getArray("tracks"); - for (final Object o : tracks) { - if (o instanceof JsonObject) { - final JsonObject track = (JsonObject) o; - if (track.has("title")) { // i.e. if full info is available - streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); - } else { - // %09d would be enough, but a 0 before the number does not create problems, so - // let's be sure - ids.add(String.format("%010d", track.getInt("id"))); - } - } - } + tracks.stream().filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .forEachOrdered(track -> { + if (track.has("title")) { // i.e. if full info is available + streamInfoItemsCollector.commit( + new SoundcloudStreamInfoItemExtractor(track)); + } else { + // %09d would be enough, but a 0 before the number does not create + // problems, so let's be sure + ids.add(String.format("%010d", track.getInt("id"))); + } + }); return new InfoItemsPage<>(streamInfoItemsCollector, new Page(ids)); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index d37320df..0a043139 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -23,6 +23,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.*; import javax.annotation.Nonnull; @@ -71,7 +72,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString)); } - final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8); + final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8); final Map> headers = new HashMap<>(); addClientInfoHeaders(headers); @@ -97,6 +98,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return name; } + @Nonnull @Override public String getThumbnailUrl() throws ParsingException { try { @@ -108,19 +110,15 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { .getObject("watchEndpoint").getString("videoId")); } catch (final Exception ignored) { } + throw new ParsingException("Could not get playlist thumbnail", e); } } - @Override - public String getBannerUrl() { - return ""; - } - @Override public String getUploaderUrl() { // YouTube mixes are auto-generated by YouTube - return ""; + return EMPTY_STRING; } @Override @@ -132,7 +130,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() { // YouTube mixes are auto-generated by YouTube - return ""; + return EMPTY_STRING; } @Override @@ -148,8 +146,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, - ExtractionException { + public InfoItemsPage getInitialPage() + throws IOException, ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); @@ -159,9 +157,10 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies)); } - private Page getNextPageFrom(final JsonObject playlistJson, - final Map cookies) throws IOException, - ExtractionException { + @Nonnull + private Page getNextPageFrom(@Nonnull final JsonObject playlistJson, + final Map cookies) + throws IOException, ExtractionException { final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") .get(playlistJson.getArray("contents").size() - 1)); if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { @@ -181,7 +180,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { .value("playlistIndex", index) .value("params", params) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body); } @@ -217,26 +216,23 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, @Nullable final List streams) { - if (streams == null) { return; } final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Object stream : streams) { - if (stream instanceof JsonObject) { - final JsonObject streamInfo = ((JsonObject) stream) - .getObject("playlistPanelVideoRenderer"); - if (streamInfo != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, - timeAgoParser)); - } - } - } + streams.stream() + .filter(JsonObject.class::isInstance) + .map(stream -> ((JsonObject) stream) + .getObject("playlistPanelVideoRenderer")) + .filter(Objects::nonNull) + .map(streamInfo -> new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)) + .forEachOrdered(collector::commit); } - private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { + @Nonnull + private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); @@ -251,25 +247,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return getThumbnailUrlFromVideoId(videoId); } + @Nonnull private String getThumbnailUrlFromVideoId(final String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } - - @Nonnull - @Override - public String getSubChannelName() { - return ""; - } - - @Nonnull - @Override - public String getSubChannelUrl() { - return ""; - } - - @Nonnull - @Override - public String getSubChannelAvatarUrl() { - return ""; - } } From 8b3f90eb7e4a56ee7e33128af7cf8dd94663414a Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sat, 12 Mar 2022 17:28:36 +0100 Subject: [PATCH 3/5] [YouTube] Fix extraction of series playlists and don't return the view count as the stream count for learning playlists ITEM_COUNT_UNKNOWN is returned when the JSON array which contains usally the number of videos is less than 3 items. Also apply the same type of optimizations done in other PlaylistExtractors in YoutubePlaylistExtractor. --- .../extractors/YoutubePlaylistExtractor.java | 197 +++++++++++------- 1 file changed, 122 insertions(+), 75 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index d6daa954..0e0c6003 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -21,22 +21,31 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.*; -@SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { - private JsonObject initialData; + // Minimum size of the stats array in the browse response which includes the streams count + private static final int STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE = 2; + + // Names of some objects in JSON response frequently used in this class + private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer"; + private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer"; + private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer"; + + private JsonObject browseResponse; private JsonObject playlistInfo; - public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public YoutubePlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @@ -45,41 +54,46 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { ExtractionException { final Localization localization = getExtractorLocalization(); final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization, - getExtractorContentCountry()) - .value("browseId", "VL" + getId()) - .value("params", "wgYCCAA%3D") // Show unavailable videos - .done()) - .getBytes(UTF_8); + getExtractorContentCountry()) + .value("browseId", "VL" + getId()) + .value("params", "wgYCCAA%3D") // Show unavailable videos + .done()) + .getBytes(StandardCharsets.UTF_8); - initialData = getJsonPostResponse("browse", body, localization); - YoutubeParsingHelper.defaultAlertsCheck(initialData); + browseResponse = getJsonPostResponse("browse", body, localization); + YoutubeParsingHelper.defaultAlertsCheck(browseResponse); playlistInfo = getPlaylistInfo(); } private JsonObject getUploaderInfo() throws ParsingException { - final JsonArray items = initialData.getObject("sidebar") - .getObject("playlistSidebarRenderer").getArray("items"); + final JsonArray items = browseResponse.getObject("sidebar") + .getObject("playlistSidebarRenderer") + .getArray("items"); JsonObject videoOwner = items.getObject(1) - .getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); - if (videoOwner.has("videoOwnerRenderer")) { - return videoOwner.getObject("videoOwnerRenderer"); + .getObject("playlistSidebarSecondaryInfoRenderer") + .getObject("videoOwner"); + if (videoOwner.has(VIDEO_OWNER_RENDERER)) { + return videoOwner.getObject(VIDEO_OWNER_RENDERER); } // we might want to create a loop here instead of using duplicated code videoOwner = items.getObject(items.size()) - .getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); - if (videoOwner.has("videoOwnerRenderer")) { - return videoOwner.getObject("videoOwnerRenderer"); + .getObject("playlistSidebarSecondaryInfoRenderer") + .getObject("videoOwner"); + if (videoOwner.has(VIDEO_OWNER_RENDERER)) { + return videoOwner.getObject(VIDEO_OWNER_RENDERER); } throw new ParsingException("Could not get uploader info"); } private JsonObject getPlaylistInfo() throws ParsingException { try { - return initialData.getObject("sidebar").getObject("playlistSidebarRenderer") - .getArray("items").getObject(0) + return browseResponse.getObject("sidebar") + .getObject("playlistSidebarRenderer") + .getArray("items") + .getObject(0) .getObject("playlistSidebarPrimaryInfoRenderer"); } catch (final Exception e) { throw new ParsingException("Could not get PlaylistInfo", e); @@ -90,33 +104,41 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { final String name = getTextFromObject(playlistInfo.getObject("title")); - if (!isNullOrEmpty(name)) return name; + if (!isNullOrEmpty(name)) { + return name; + } - return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); + return browseResponse.getObject("microformat") + .getObject("microformatDataRenderer") + .getString("title"); } + @Nonnull @Override public String getThumbnailUrl() throws ParsingException { - String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + String url = playlistInfo.getObject("thumbnailRenderer") + .getObject("playlistVideoThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails") + .getObject(0) + .getString("url"); if (isNullOrEmpty(url)) { - url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") - .getArray("thumbnails").getObject(0).getString("url"); + url = browseResponse.getObject("microformat") + .getObject("microformatDataRenderer") + .getObject("thumbnail") + .getArray("thumbnails") + .getObject(0) + .getString("url"); - if (isNullOrEmpty(url)) throw new ParsingException("Could not get playlist thumbnail"); + if (isNullOrEmpty(url)) { + throw new ParsingException("Could not get playlist thumbnail"); + } } return fixThumbnailUrl(url); } - @Override - public String getBannerUrl() { - // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement it in the frontend. - return ""; - } - @Override public String getUploaderUrl() throws ParsingException { try { @@ -138,7 +160,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { try { - final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + final String url = getUploaderInfo() + .getObject("thumbnail") + .getArray("thumbnails") + .getObject(0) + .getString("url"); return fixThumbnailUrl(url); } catch (final Exception e) { @@ -148,14 +174,29 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public boolean isUploaderVerified() throws ParsingException { + // YouTube doesn't provide this information return false; } @Override public long getStreamCount() throws ParsingException { try { - final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0)); - return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); + final JsonArray stats = playlistInfo.getArray("stats"); + // For unknown reasons, YouTube don't provide the stream count for learning playlists + // on the desktop client but only the number of views and the playlist modified date + // On normal playlists, at least 3 items are returned: the number of videos, the number + // of views and the playlist modification date + // We can get it by using another client, however it seems we can't get the avatar + // uploader URL with another client than the WEB client + if (stats.size() > STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE) { + final String videosText = getTextFromObject(playlistInfo.getArray("stats") + .getObject(0)); + if (videosText != null) { + return Long.parseLong(Utils.removeNonDigitCharacters(videosText)); + } + } + + return ITEM_COUNT_UNKNOWN; } catch (final Exception e) { throw new ParsingException("Could not get video count from playlist", e); } @@ -164,19 +205,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public String getSubChannelName() { - return ""; + return EMPTY_STRING; } @Nonnull @Override public String getSubChannelUrl() { - return ""; + return EMPTY_STRING; } @Nonnull @Override public String getSubChannelAvatarUrl() { - return ""; + return EMPTY_STRING; } @Nonnull @@ -185,26 +226,31 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Page nextPage = null; - final JsonArray contents = initialData.getObject("contents") - .getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0) - .getObject("tabRenderer").getObject("content").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") + final JsonArray contents = browseResponse.getObject("contents") + .getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs") + .getObject(0) + .getObject("tabRenderer") + .getObject("content") + .getObject("sectionListRenderer") .getArray("contents"); - if (contents.getObject(0).has("playlistSegmentRenderer")) { - for (final Object segment : contents) { - if (((JsonObject) segment).getObject("playlistSegmentRenderer") - .has("videoList")) { - collectStreamsFrom(collector, ((JsonObject) segment) - .getObject("playlistSegmentRenderer").getObject("videoList") - .getObject("playlistVideoListRenderer").getArray("contents")); - } - } + final JsonObject videoPlaylistObject = contents.stream() + .map(JsonObject.class::cast) + .map(content -> content.getObject("itemSectionRenderer") + .getArray("contents") + .getObject(0)) + .filter(contentItemSectionRendererContents -> + contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER) + || contentItemSectionRendererContents.has( + "playlistSegmentRenderer")) + .findFirst() + .orElse(null); - return new InfoItemsPage<>(collector, null); - } else if (contents.getObject(0).has("playlistVideoListRenderer")) { - final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer"); - final JsonArray videosArray = videos.getArray("contents"); + if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) { + final JsonArray videosArray = videoPlaylistObject + .getObject(PLAYLIST_VIDEO_LIST_RENDERER) + .getArray("contents"); collectStreamsFrom(collector, videosArray); nextPage = getNextPageFrom(videosArray); @@ -229,7 +275,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions") - .getObject(0).getObject("appendContinuationItemsAction") + .getObject(0) + .getObject("appendContinuationItemsAction") .getArray("continuationItems"); collectStreamsFrom(collector, continuation); @@ -237,8 +284,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); } - private Page getNextPageFrom(final JsonArray contents) throws IOException, - ExtractionException { + @Nullable + private Page getNextPageFrom(final JsonArray contents) + throws IOException, ExtractionException { if (isNullOrEmpty(contents)) { return null; } @@ -252,10 +300,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .getString("token"); final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - getExtractorLocalization(), getExtractorContentCountry()) - .value("continuation", continuation) - .done()) - .getBytes(UTF_8); + getExtractorLocalization(), getExtractorContentCountry()) + .value("continuation", continuation) + .done()) + .getBytes(StandardCharsets.UTF_8); return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body); } else { @@ -263,20 +311,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } - private void collectStreamsFrom(final StreamInfoItemsCollector collector, - final JsonArray videos) { + private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, + @Nonnull final JsonArray videos) { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Object video : videos) { - if (((JsonObject) video).has("playlistVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video) - .getObject("playlistVideoRenderer"), timeAgoParser) { + videos.stream() + .filter(video -> ((JsonObject) video).has(PLAYLIST_VIDEO_RENDERER)) + .map(video -> new YoutubeStreamInfoItemExtractor(((JsonObject) video) + .getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser) { @Override public long getViewCount() { return -1; } - }); - } - } + }) + .forEachOrdered(collector::commit); } } From 35e082248e177a8c68fe7d9c0201cb08e3870235 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sat, 12 Mar 2022 19:11:53 +0100 Subject: [PATCH 4/5] Fix YouTube and SoundCloud playlists tests --- .../SoundcloudPlaylistExtractorTest.java | 18 ++++++++++-------- .../youtube/YoutubePlaylistExtractorTest.java | 10 ++++++---- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 3ece9c20..1b8930dc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -7,12 +7,14 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.junit.jupiter.api.Assertions.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.services.DefaultTests.*; @@ -85,9 +87,9 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testBannerUrl() { + public void testBannerUrl() throws ParsingException { // SoundCloud playlists do not have a banner - assertNull(extractor.getBannerUrl()); + assertEmpty(extractor.getBannerUrl()); } @Test @@ -182,9 +184,9 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testBannerUrl() { + public void testBannerUrl() throws ParsingException { // SoundCloud playlists do not have a banner - assertNull(extractor.getBannerUrl()); + assertEmpty(extractor.getBannerUrl()); } @Test @@ -294,9 +296,9 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testBannerUrl() { + public void testBannerUrl() throws ParsingException { // SoundCloud playlists do not have a banner - assertNull(extractor.getBannerUrl()); + assertEmpty(extractor.getBannerUrl()); } @Test @@ -398,9 +400,9 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testBannerUrl() { + public void testBannerUrl() throws ParsingException { // SoundCloud playlists do not have a banner - assertNull(extractor.getBannerUrl()); + assertEmpty(extractor.getBannerUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index fdd91938..3f076312 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; @@ -129,7 +130,7 @@ public class YoutubePlaylistExtractorTest { @Disabled @Test - public void testBannerUrl() { + public void testBannerUrl() throws ParsingException { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt", bannerUrl); @@ -249,7 +250,7 @@ public class YoutubePlaylistExtractorTest { @Disabled @Test - public void testBannerUrl() { + public void testBannerUrl() throws ParsingException { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt", bannerUrl); @@ -352,7 +353,7 @@ public class YoutubePlaylistExtractorTest { @Disabled @Test - public void testBannerUrl() { + public void testBannerUrl() throws ParsingException { final String bannerUrl = extractor.getBannerUrl(); assertIsSecureUrl(bannerUrl); ExtractorAsserts.assertContains("yt", bannerUrl); @@ -377,7 +378,8 @@ public class YoutubePlaylistExtractorTest { @Test public void testStreamCount() throws Exception { - ExtractorAsserts.assertGreater(40, extractor.getStreamCount()); + // We are not able to extract the stream count of YouTube learning playlists + assertEquals(ITEM_COUNT_UNKNOWN, extractor.getStreamCount()); } @Override From c7757c099497a6033af406a8a0873e4d4dd2d65d Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Wed, 16 Mar 2022 20:14:08 +0100 Subject: [PATCH 5/5] Apply requested changes --- .../extractors/SoundcloudPlaylistExtractor.java | 8 +++++--- .../youtube/extractors/YoutubeMixPlaylistExtractor.java | 4 ++-- .../youtube/extractors/YoutubePlaylistExtractor.java | 9 ++++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java index ffde3fb7..3d525e63 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java @@ -127,11 +127,13 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { new StreamInfoItemsCollector(getServiceId()); final List ids = new ArrayList<>(); - final JsonArray tracks = playlist.getArray("tracks"); - tracks.stream().filter(JsonObject.class::isInstance) + playlist.getArray("tracks") + .stream() + .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) .forEachOrdered(track -> { - if (track.has("title")) { // i.e. if full info is available + // i.e. if full info is available + if (track.has("title")) { streamInfoItemsCollector.commit( new SoundcloudStreamInfoItemExtractor(track)); } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 0a043139..a98fa617 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -224,8 +224,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { streams.stream() .filter(JsonObject.class::isInstance) - .map(stream -> ((JsonObject) stream) - .getObject("playlistPanelVideoRenderer")) + .map(JsonObject.class::cast) + .map(stream -> stream.getObject("playlistPanelVideoRenderer")) .filter(Objects::nonNull) .map(streamInfo -> new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)) .forEachOrdered(collector::commit); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 0e0c6003..11ffa1d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -236,6 +236,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .getArray("contents"); final JsonObject videoPlaylistObject = contents.stream() + .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) .map(content -> content.getObject("itemSectionRenderer") .getArray("contents") @@ -316,9 +317,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); videos.stream() - .filter(video -> ((JsonObject) video).has(PLAYLIST_VIDEO_RENDERER)) - .map(video -> new YoutubeStreamInfoItemExtractor(((JsonObject) video) - .getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser) { + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(video -> video.has(PLAYLIST_VIDEO_RENDERER)) + .map(video -> new YoutubeStreamInfoItemExtractor( + video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser) { @Override public long getViewCount() { return -1;