From 316fe0109df23dd3e6db25274d85508203308090 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 23 Feb 2020 13:48:54 +0100 Subject: [PATCH] Fix first playlist page --- .../extractors/YoutubePlaylistExtractor.java | 120 ++---------------- .../YoutubeStreamInfoItemExtractor.java | 19 ++- 2 files changed, 28 insertions(+), 111 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 3a4cbffd..f0fb91a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -13,15 +13,12 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -233,112 +230,23 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private void collectStreamsFrom(@Nonnull StreamInfoItemsCollector collector, @Nullable Element element) { collector.reset(); - if (element == null) { - return; - } - - final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Element li : element.children()) { - if (isDeletedItem(li)) { - continue; + JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("playlistVideoListRenderer").getArray("contents"); + + for (Object video : videos) { + if (((JsonObject) video).getObject("playlistVideoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { + @Override + public long getViewCount() { + return -1; + } + }); } - - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - public Element uploaderLink; - - @Override - public boolean isAd() { - return false; - } - - @Override - public String getUrl() throws ParsingException { - try { - return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - return li.attr("data-title"); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } - } - - @Override - public long getDuration() throws ParsingException { - try { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - - Element first = li.select("div[class=\"timestamp\"] span").first(); - if (first == null) { - // Video unavailable (private, deleted, etc.), this is a thing that happens specifically with playlists, - // because in other cases, those videos don't even show up - return -1; - } - - return YoutubeParsingHelper.parseDurationString(first.text()); - } catch (Exception e) { - throw new ParsingException("Could not get duration" + getUrl(), e); - } - } - - - private Element getUploaderLink() { - // should always be present since we filter deleted items - if (uploaderLink == null) { - uploaderLink = li.select("div[class=pl-video-owner] a").first(); - } - return uploaderLink; - } - - @Override - public String getUploaderName() throws ParsingException { - return getUploaderLink().text(); - } - - @Override - public String getUploaderUrl() throws ParsingException { - // this url is not always in the form "/channel/..." - // sometimes Youtube provides urls in the from "/user/..." - return getUploaderLink().attr("abs:href"); - } - - @Override - public String getTextualUploadDate() { - return ""; - } - - @Override - public long getViewCount() throws ParsingException { - return -1; - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() + "/hqdefault.jpg"; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); } } - - /** - * Check if the playlist item is deleted - * - * @param li the list item - * @return true if the item is deleted - */ - private boolean isDeletedItem(Element li) { - return li.select("div[class=pl-video-owner] a").isEmpty(); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 91124fb4..a53194f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; -import org.jsoup.nodes.Element; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -37,10 +37,6 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private JsonObject videoInfo; private final TimeAgoParser timeAgoParser; - public YoutubeStreamInfoItemExtractor(Element a, @Nullable TimeAgoParser timeAgoParser) { - this.timeAgoParser = timeAgoParser; - } - /** * Creates an extractor of StreamInfoItems from a YouTube page. * @@ -115,6 +111,12 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { .getObject(0).getString("text"); } catch (Exception ignored) {} } + if (name == null) { + try { + name = videoInfo.getObject("shortBylineText").getArray("runs") + .getObject(0).getString("text"); + } catch (Exception ignored) {} + } if (name != null && !name.isEmpty()) return name; throw new ParsingException("Could not get uploader name"); } @@ -135,6 +137,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { .getObject("browseEndpoint").getString("browseId"); } catch (Exception ignored) {} } + if (id == null) { + try { + id = videoInfo.getObject("shortBylineText").getArray("runs") + .getObject(0).getObject("navigationEndpoint") + .getObject("browseEndpoint").getString("browseId"); + } catch (Exception ignored) {} + } if (id == null || id.isEmpty()) { throw new IllegalArgumentException("is empty"); }