From 8b3f90eb7e4a56ee7e33128af7cf8dd94663414a Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sat, 12 Mar 2022 17:28:36 +0100 Subject: [PATCH] [YouTube] Fix extraction of series playlists and don't return the view count as the stream count for learning playlists ITEM_COUNT_UNKNOWN is returned when the JSON array which contains usally the number of videos is less than 3 items. Also apply the same type of optimizations done in other PlaylistExtractors in YoutubePlaylistExtractor. --- .../extractors/YoutubePlaylistExtractor.java | 197 +++++++++++------- 1 file changed, 122 insertions(+), 75 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index d6daa954..0e0c6003 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -21,22 +21,31 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.*; -@SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { - private JsonObject initialData; + // Minimum size of the stats array in the browse response which includes the streams count + private static final int STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE = 2; + + // Names of some objects in JSON response frequently used in this class + private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer"; + private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer"; + private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer"; + + private JsonObject browseResponse; private JsonObject playlistInfo; - public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + public YoutubePlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { super(service, linkHandler); } @@ -45,41 +54,46 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { ExtractionException { final Localization localization = getExtractorLocalization(); final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization, - getExtractorContentCountry()) - .value("browseId", "VL" + getId()) - .value("params", "wgYCCAA%3D") // Show unavailable videos - .done()) - .getBytes(UTF_8); + getExtractorContentCountry()) + .value("browseId", "VL" + getId()) + .value("params", "wgYCCAA%3D") // Show unavailable videos + .done()) + .getBytes(StandardCharsets.UTF_8); - initialData = getJsonPostResponse("browse", body, localization); - YoutubeParsingHelper.defaultAlertsCheck(initialData); + browseResponse = getJsonPostResponse("browse", body, localization); + YoutubeParsingHelper.defaultAlertsCheck(browseResponse); playlistInfo = getPlaylistInfo(); } private JsonObject getUploaderInfo() throws ParsingException { - final JsonArray items = initialData.getObject("sidebar") - .getObject("playlistSidebarRenderer").getArray("items"); + final JsonArray items = browseResponse.getObject("sidebar") + .getObject("playlistSidebarRenderer") + .getArray("items"); JsonObject videoOwner = items.getObject(1) - .getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); - if (videoOwner.has("videoOwnerRenderer")) { - return videoOwner.getObject("videoOwnerRenderer"); + .getObject("playlistSidebarSecondaryInfoRenderer") + .getObject("videoOwner"); + if (videoOwner.has(VIDEO_OWNER_RENDERER)) { + return videoOwner.getObject(VIDEO_OWNER_RENDERER); } // we might want to create a loop here instead of using duplicated code videoOwner = items.getObject(items.size()) - .getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); - if (videoOwner.has("videoOwnerRenderer")) { - return videoOwner.getObject("videoOwnerRenderer"); + .getObject("playlistSidebarSecondaryInfoRenderer") + .getObject("videoOwner"); + if (videoOwner.has(VIDEO_OWNER_RENDERER)) { + return videoOwner.getObject(VIDEO_OWNER_RENDERER); } throw new ParsingException("Could not get uploader info"); } private JsonObject getPlaylistInfo() throws ParsingException { try { - return initialData.getObject("sidebar").getObject("playlistSidebarRenderer") - .getArray("items").getObject(0) + return browseResponse.getObject("sidebar") + .getObject("playlistSidebarRenderer") + .getArray("items") + .getObject(0) .getObject("playlistSidebarPrimaryInfoRenderer"); } catch (final Exception e) { throw new ParsingException("Could not get PlaylistInfo", e); @@ -90,33 +104,41 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { final String name = getTextFromObject(playlistInfo.getObject("title")); - if (!isNullOrEmpty(name)) return name; + if (!isNullOrEmpty(name)) { + return name; + } - return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); + return browseResponse.getObject("microformat") + .getObject("microformatDataRenderer") + .getString("title"); } + @Nonnull @Override public String getThumbnailUrl() throws ParsingException { - String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + String url = playlistInfo.getObject("thumbnailRenderer") + .getObject("playlistVideoThumbnailRenderer") + .getObject("thumbnail") + .getArray("thumbnails") + .getObject(0) + .getString("url"); if (isNullOrEmpty(url)) { - url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") - .getArray("thumbnails").getObject(0).getString("url"); + url = browseResponse.getObject("microformat") + .getObject("microformatDataRenderer") + .getObject("thumbnail") + .getArray("thumbnails") + .getObject(0) + .getString("url"); - if (isNullOrEmpty(url)) throw new ParsingException("Could not get playlist thumbnail"); + if (isNullOrEmpty(url)) { + throw new ParsingException("Could not get playlist thumbnail"); + } } return fixThumbnailUrl(url); } - @Override - public String getBannerUrl() { - // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement it in the frontend. - return ""; - } - @Override public String getUploaderUrl() throws ParsingException { try { @@ -138,7 +160,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { try { - final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + final String url = getUploaderInfo() + .getObject("thumbnail") + .getArray("thumbnails") + .getObject(0) + .getString("url"); return fixThumbnailUrl(url); } catch (final Exception e) { @@ -148,14 +174,29 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public boolean isUploaderVerified() throws ParsingException { + // YouTube doesn't provide this information return false; } @Override public long getStreamCount() throws ParsingException { try { - final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0)); - return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); + final JsonArray stats = playlistInfo.getArray("stats"); + // For unknown reasons, YouTube don't provide the stream count for learning playlists + // on the desktop client but only the number of views and the playlist modified date + // On normal playlists, at least 3 items are returned: the number of videos, the number + // of views and the playlist modification date + // We can get it by using another client, however it seems we can't get the avatar + // uploader URL with another client than the WEB client + if (stats.size() > STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE) { + final String videosText = getTextFromObject(playlistInfo.getArray("stats") + .getObject(0)); + if (videosText != null) { + return Long.parseLong(Utils.removeNonDigitCharacters(videosText)); + } + } + + return ITEM_COUNT_UNKNOWN; } catch (final Exception e) { throw new ParsingException("Could not get video count from playlist", e); } @@ -164,19 +205,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public String getSubChannelName() { - return ""; + return EMPTY_STRING; } @Nonnull @Override public String getSubChannelUrl() { - return ""; + return EMPTY_STRING; } @Nonnull @Override public String getSubChannelAvatarUrl() { - return ""; + return EMPTY_STRING; } @Nonnull @@ -185,26 +226,31 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Page nextPage = null; - final JsonArray contents = initialData.getObject("contents") - .getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0) - .getObject("tabRenderer").getObject("content").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") + final JsonArray contents = browseResponse.getObject("contents") + .getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs") + .getObject(0) + .getObject("tabRenderer") + .getObject("content") + .getObject("sectionListRenderer") .getArray("contents"); - if (contents.getObject(0).has("playlistSegmentRenderer")) { - for (final Object segment : contents) { - if (((JsonObject) segment).getObject("playlistSegmentRenderer") - .has("videoList")) { - collectStreamsFrom(collector, ((JsonObject) segment) - .getObject("playlistSegmentRenderer").getObject("videoList") - .getObject("playlistVideoListRenderer").getArray("contents")); - } - } + final JsonObject videoPlaylistObject = contents.stream() + .map(JsonObject.class::cast) + .map(content -> content.getObject("itemSectionRenderer") + .getArray("contents") + .getObject(0)) + .filter(contentItemSectionRendererContents -> + contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER) + || contentItemSectionRendererContents.has( + "playlistSegmentRenderer")) + .findFirst() + .orElse(null); - return new InfoItemsPage<>(collector, null); - } else if (contents.getObject(0).has("playlistVideoListRenderer")) { - final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer"); - final JsonArray videosArray = videos.getArray("contents"); + if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) { + final JsonArray videosArray = videoPlaylistObject + .getObject(PLAYLIST_VIDEO_LIST_RENDERER) + .getArray("contents"); collectStreamsFrom(collector, videosArray); nextPage = getNextPageFrom(videosArray); @@ -229,7 +275,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions") - .getObject(0).getObject("appendContinuationItemsAction") + .getObject(0) + .getObject("appendContinuationItemsAction") .getArray("continuationItems"); collectStreamsFrom(collector, continuation); @@ -237,8 +284,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); } - private Page getNextPageFrom(final JsonArray contents) throws IOException, - ExtractionException { + @Nullable + private Page getNextPageFrom(final JsonArray contents) + throws IOException, ExtractionException { if (isNullOrEmpty(contents)) { return null; } @@ -252,10 +300,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .getString("token"); final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - getExtractorLocalization(), getExtractorContentCountry()) - .value("continuation", continuation) - .done()) - .getBytes(UTF_8); + getExtractorLocalization(), getExtractorContentCountry()) + .value("continuation", continuation) + .done()) + .getBytes(StandardCharsets.UTF_8); return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body); } else { @@ -263,20 +311,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } - private void collectStreamsFrom(final StreamInfoItemsCollector collector, - final JsonArray videos) { + private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, + @Nonnull final JsonArray videos) { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Object video : videos) { - if (((JsonObject) video).has("playlistVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video) - .getObject("playlistVideoRenderer"), timeAgoParser) { + videos.stream() + .filter(video -> ((JsonObject) video).has(PLAYLIST_VIDEO_RENDERER)) + .map(video -> new YoutubeStreamInfoItemExtractor(((JsonObject) video) + .getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser) { @Override public long getViewCount() { return -1; } - }); - } - } + }) + .forEachOrdered(collector::commit); } }