diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 544a9e46..8e47e779 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -52,7 +52,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; private Document doc; - private JsonObject ytInitialData; + private JsonObject initialData; public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -63,17 +63,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; final Response response = downloader.get(channelUrl, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); - ytInitialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } @Override public String getNextPageUrl() throws ExtractionException { @@ -97,7 +89,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return doc.select("meta[property=\"og:url\"]").first().attr("content").replace(CHANNEL_URL_BASE, ""); } catch (Exception ignored) {} try { - return ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); } catch (Exception ignored) {} // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) @@ -124,7 +116,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { try { - return ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails").getObject(0).getString("url"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -133,7 +125,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - String url = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); + String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); if (url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; } @@ -165,7 +157,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final JsonObject subscriberInfo = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); + final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); if (subscriberInfo != null) { try { @@ -182,7 +174,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getDescription() throws ParsingException { try { - return ytInitialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); + return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); } catch (Exception e) { throw new ParsingException("Could not get channel description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 1498ef68..3a4cbffd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -47,21 +47,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final String url = getUrl(); final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); uploaderInfo = getUploaderInfo(); playlistInfo = getPlaylistInfo(); playlistVideos = getPlaylistVideos(); } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } - private JsonObject getUploaderInfo() throws ParsingException { JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items"); try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index c8ed9efd..57ce3aab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -51,7 +51,7 @@ import javax.annotation.Nonnull; public class YoutubeSearchExtractor extends SearchExtractor { private Document doc; - private JsonObject ytInitialData; + private JsonObject initialData; public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); @@ -62,7 +62,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final String url = getUrl(); final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - ytInitialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); } @Nonnull @@ -119,7 +119,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - JsonArray list = ytInitialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + JsonArray list = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") .getObject(0).getObject("itemSectionRenderer").getArray("contents"); @@ -138,12 +138,4 @@ public class YoutubeSearchExtractor extends SearchExtractor { return collector; } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a6c94732..a0bf1c8f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -104,7 +104,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private final Map videoInfoPage = new HashMap<>(); private JsonObject playerResponse; - private JsonObject ytInitialData; + private JsonObject initialData; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -339,7 +339,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (getStreamType().equals(StreamType.LIVE_STREAM)) { // The array index is variable, therefore we loop throw the complete array. // videoPrimaryInfoRenderer is often stored at index 1 - JsonArray contents = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("results").getObject("results").getArray("contents"); for (Object c : contents) { try { @@ -421,7 +421,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String likesString = ""; try { try { - likesString = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; + likesString = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -444,7 +444,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { Element button = doc.select("button.like-button-renderer-dislike-button").first(); try { - dislikesString = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; + dislikesString = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -507,7 +507,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String uploaderAvatarUrl = null; try { - uploaderAvatarUrl = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") + uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") .getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail") .getArray("thumbnails").getObject(0).getString("url"); @@ -517,7 +517,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} try { - uploaderAvatarUrl = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") + uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") .getObject("results").getArray("contents").getObject(1).getObject("videoSecondaryInfoRenderer") .getObject("owner").getObject("videoOwnerRenderer").getObject("thumbnail").getArray("thumbnails") .getObject(0).getString("url"); @@ -670,7 +670,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public StreamInfoItem getNextStream() throws IOException, ExtractionException { assertPageFetched(); try { - final JsonObject videoInfo = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults").getObject("secondaryResults").getArray("results") .getObject(0).getObject("compactAutoplayRenderer").getArray("contents") .getObject(0).getObject("compactVideoRenderer"); @@ -690,7 +690,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray results = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults").getObject("secondaryResults").getArray("results"); final TimeAgoParser timeAgoParser = getTimeAgoParser(); @@ -778,7 +778,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { isAgeRestricted = false; } playerResponse = getPlayerResponse(); - ytInitialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(pageHtml); if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); @@ -852,14 +852,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } @Nonnull private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 467a1fed..d815e9e1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -61,16 +61,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = getInitialData(); - } - - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 65ec7e3f..78516f6d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -1,11 +1,15 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.Parser; import java.net.URL; import java.text.ParseException; @@ -143,4 +147,14 @@ public class YoutubeParsingHelper { uploadDate.setTime(date); return uploadDate; } + + public static JsonObject getInitialData(String html) throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } + } + }