From ac15df45489178002c95db436a717319020a517a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 1 Apr 2020 10:03:54 +0200 Subject: [PATCH] Move YouTube Music search to its own class --- .../services/youtube/YoutubeService.java | 41 +- .../YoutubeMusicSearchExtractor.java | 526 ++++++++++++++++++ .../extractors/YoutubeSearchExtractor.java | 508 ++--------------- 3 files changed, 598 insertions(+), 477 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index eae3bcb9..51967214 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -7,22 +7,45 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import javax.annotation.Nonnull; import java.util.List; +import javax.annotation.Nonnull; + import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; /* * Created by Christian Schabesberger on 23.08.15. @@ -92,7 +115,13 @@ public class YoutubeService extends StreamingService { @Override public SearchExtractor getSearchExtractor(SearchQueryHandler query) { - return new YoutubeSearchExtractor(this, query); + final List contentFilters = query.getContentFilters(); + + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) { + return new YoutubeMusicSearchExtractor(this, query); + } else { + return new YoutubeSearchExtractor(this, query); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java new file mode 100644 index 00000000..46e9d93c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -0,0 +1,526 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nonnull; + +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; + +public class YoutubeMusicSearchExtractor extends SearchExtractor { + private JsonObject initialData; + + public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + + final String params; + + switch (getLinkHandler().getContentFilters().get(0)) { + case MUSIC_SONGS: + params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_VIDEOS: + params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ALBUMS: + params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_PLAYLISTS: + params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; + break; + case MUSIC_ARTISTS: + params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; + break; + default: + params = null; + break; + } + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .value("query", getSearchString()) + .value("params", params) + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final Response response = getDownloader().post(url, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + try { + initialData = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return super.getUrl(); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return ""; + } + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) { + return ""; + } + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); + } + } + + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() throws ExtractionException, IOException { + final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + + for (Object content : contents) { + if (((JsonObject) content).getObject("musicShelfRenderer") != null) { + return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); + } + } + + return ""; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + + final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + + // @formatter:off + byte[] json = JsonWriter.string() + .object() + .object("context") + .object("client") + .value("clientName", "WEB_REMIX") + .value("clientVersion", youtubeMusicKeys[2]) + .value("hl", "en") + .value("gl", getExtractorContentCountry().getCountryCode()) + .array("experimentIds").end() + .value("experimentsToken", "") + .value("utcOffsetMinutes", 0) + .object("locationInfo").end() + .object("musicAppInfo").end() + .end() + .object("capabilities").end() + .object("request") + .array("internalExperimentFlags").end() + .object("sessionIndex").end() + .end() + .object("activePlayers").end() + .object("user") + .value("enableSafetyMode", false) + .end() + .end() + .end().done().getBytes("UTF-8"); + // @formatter:on + + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + + final Response response = getDownloader().post(pageUrl, headers, json); + + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + + final JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + + if (ajaxJson.getObject("continuationContents") == null) { + return InfoItemsPage.emptyPage(); + } + + final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); + + collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); + final JsonArray continuations = musicShelfContinuation.getArray("continuations"); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); + } + + private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) { + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (Object item : videos) { + final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); + if (info != null) { + final String searchType = getLinkHandler().getContentFilters().get(0); + if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { + collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public long getDuration() throws ParsingException { + final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (duration != null && !duration.isEmpty()) { + return YoutubeParsingHelper.parseDurationString(duration); + } + throw new ParsingException("Could not get duration"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String url = null; + + if (searchType.equals(MUSIC_VIDEOS)) { + JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); + for (Object item : items) { + final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); + if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { + url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); + break; + } + } + } else { + final JsonObject navigationEndpoint = info.getArray("flexColumns") + .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") + .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); + if (navigationEndpoint == null) { + return null; + } + url = getUrlFromNavigationEndpoint(navigationEndpoint); + } + + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get uploader url"); + } + + @Override + public String getTextualUploadDate() { + return null; + } + + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + if (searchType.equals(MUSIC_SONGS)) { + return -1; + } + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get view count"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } else if (searchType.equals(MUSIC_ARTISTS)) { + collector.commit(new YoutubeChannelInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public long getSubscriberCount() throws ParsingException { + final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (viewCount != null && !viewCount.isEmpty()) { + return Utils.mixedNumberWordToLong(viewCount); + } + throw new ParsingException("Could not get subscriber count"); + } + + @Override + public long getStreamCount() { + return -1; + } + + @Override + public String getDescription() { + return null; + } + }); + } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { + collector.commit(new YoutubePlaylistInfoItemExtractor(info) { + @Override + public String getThumbnailUrl() throws ParsingException { + try { + final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get name"); + } + + @Override + public String getUrl() throws ParsingException { + final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); + if (url != null && !url.isEmpty()) { + return url; + } + throw new ParsingException("Could not get url"); + } + + @Override + public String getUploaderName() throws ParsingException { + final String name; + if (searchType.equals(MUSIC_ALBUMS)) { + name = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } else { + name = getTextFromObject(info.getArray("flexColumns").getObject(1) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + } + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get uploader name"); + } + + @Override + public long getStreamCount() throws ParsingException { + if (searchType.equals(MUSIC_ALBUMS)) { + return ITEM_COUNT_UNKNOWN; + } + final String count = getTextFromObject(info.getArray("flexColumns").getObject(2) + .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); + if (count != null && !count.isEmpty()) { + if (count.contains("100+")) { + return ITEM_COUNT_MORE_THAN_100; + } else { + return Long.parseLong(Utils.removeNonDigitCharacters(count)); + } + } + throw new ParsingException("Could not get count"); + } + }); + } + } + } + } + + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { + if (continuations == null) { + return ""; + } + + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + + return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index ca56283d..06d83b24 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,43 +2,23 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; /* * Created by Christian Schabesberger on 22.07.2018 @@ -63,278 +43,79 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS public class YoutubeSearchExtractor extends SearchExtractor { private JsonObject initialData; - public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { super(service, linkHandler); } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - if (isMusicSearch()) { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); + public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String url = getUrl() + "&pbj=1"; - final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - String params = null; - - switch (getLinkHandler().getContentFilters().get(0)) { - case MUSIC_SONGS: - params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_VIDEOS: - params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_ALBUMS: - params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_PLAYLISTS: - params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; - break; - case MUSIC_ARTISTS: - params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; - break; - } - - // @formatter:off - byte[] json = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) - .object("locationInfo").end() - .object("musicAppInfo").end() - .end() - .object("capabilities").end() - .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() - .end() - .object("activePlayers").end() - .object("user") - .value("enableSafetyMode", false) - .end() - .end() - .value("query", getSearchString()) - .value("params", params) - .end().done().getBytes("UTF-8"); - // @formatter:on - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - Response response = getDownloader().post(url, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } - - try { - initialData = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } - } else { - final String url = getUrl() + "&pbj=1"; - - final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - - initialData = ajaxJson.getObject(1).getObject("response"); - } + initialData = ajaxJson.getObject(1).getObject("response"); } @Nonnull @Override public String getUrl() throws ParsingException { - if (isMusicSearch()) return super.getUrl(); return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } @Override public String getSearchSuggestion() throws ParsingException { - if (isMusicSearch()) { - final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) return ""; - final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") - .getObject(0).getObject("didYouMeanRenderer"); - if (didYouMeanRenderer == null) return ""; - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); - } else { - JsonObject showingResultsForRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("showingResultsForRenderer"); - if (showingResultsForRenderer == null) return ""; - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + final JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + if (showingResultsForRenderer == null) { + return ""; } + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException, IOException { + public InfoItemsPage getInitialPage() throws ExtractionException { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - if (isMusicSearch()) { - final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); + final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { - collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents")); - } - } - } else { - JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); - - for (Object section : sections) { - collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); - } + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); } return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override - public String getNextPageUrl() throws ExtractionException, IOException { - if (isMusicSearch()) { - final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents"); - - for (Object content : contents) { - if (((JsonObject) content).getObject("musicShelfRenderer") != null) { - return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations")); - } - } - - return ""; - } else { - return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); - } + public String getNextPageUrl() throws ExtractionException { + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); } @Override - public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemsPage getPage(final String pageUrl) throws IOException, ExtractionException { if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - JsonArray continuations; + final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("itemSectionContinuation"); - if (isMusicSearch()) { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys(); - - // @formatter:off - byte[] json = JsonWriter.string() - .object() - .object("context") - .object("client") - .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") - .value("gl", getExtractorContentCountry().getCountryCode()) - .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) - .object("locationInfo").end() - .object("musicAppInfo").end() - .end() - .object("capabilities").end() - .object("request") - .array("internalExperimentFlags").end() - .object("sessionIndex").end() - .end() - .object("activePlayers").end() - .object("user") - .value("enableSafetyMode", false) - .end() - .end() - .end().done().getBytes("UTF-8"); - // @formatter:on - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - Response response = getDownloader().post(pageUrl, headers, json); - - if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); - } - - final String responseBody = response.responseBody(); - if (responseBody.length() < 50) { // ensure to have a valid response - throw new ParsingException("JSON response is too short"); - } - - final String responseContentType = response.getHeader("Content-Type"); - if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); - } - - final JsonObject ajaxJson; - try { - ajaxJson = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } - - if (ajaxJson.getObject("continuationContents") == null) { - return InfoItemsPage.emptyPage(); - } - - JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); - - collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); - continuations = musicShelfContinuation.getArray("continuations"); - } else { - final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - - JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("itemSectionContinuation"); - - collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - continuations = itemSectionRenderer.getArray("continuations"); - } + collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + final JsonArray continuations = itemSectionRenderer.getArray("continuations"); return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations)); } - private boolean isMusicSearch() { - final List contentFilters = getLinkHandler().getContentFilters(); - return contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_"); - } - - private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { + private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object item : videos) { @@ -351,232 +132,17 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private void collectMusicStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) { - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object item : videos) { - final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer"); - if (info != null) { - final String searchType = getLinkHandler().getContentFilters().get(0); - if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { - collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public long getDuration() throws ParsingException { - String duration = getTextFromObject(info.getArray("flexColumns").getObject(3) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (duration != null && !duration.isEmpty()) - return YoutubeParsingHelper.parseDurationString(duration); - throw new ParsingException("Could not get duration"); - } - - @Override - public String getUploaderName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); - } - - @Override - public String getUploaderUrl() throws ParsingException { - String url = null; - - if (searchType.equals(MUSIC_VIDEOS)) { - JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); - for (Object item : items) { - JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) { - url = getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); - break; - } - } - } else { - JsonObject navigationEndpoint = info.getArray("flexColumns") - .getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint"); - if (navigationEndpoint == null) return null; - url = getUrlFromNavigationEndpoint(navigationEndpoint); - } - - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get uploader url"); - } - - @Override - public String getTextualUploadDate() { - return null; - } - - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - if (searchType.equals(MUSIC_SONGS)) return -1; - String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); - throw new ParsingException("Could not get view count"); - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } else if (searchType.equals(MUSIC_ARTISTS)) { - collector.commit(new YoutubeChannelInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public long getSubscriberCount() throws ParsingException { - String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (viewCount != null && !viewCount.isEmpty()) return Utils.mixedNumberWordToLong(viewCount); - throw new ParsingException("Could not get subscriber count"); - } - - @Override - public long getStreamCount() { - return -1; - } - - @Override - public String getDescription() { - return null; - } - }); - } else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) { - collector.commit(new YoutubePlaylistInfoItemExtractor(info) { - @Override - public String getThumbnailUrl() throws ParsingException { - try { - JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") - .getObject("thumbnail").getArray("thumbnails"); - // the last thumbnail is the one with the highest resolution - String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = getTextFromObject(info.getArray("flexColumns").getObject(0) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get name"); - } - - @Override - public String getUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand")); - if (url != null && !url.isEmpty()) return url; - throw new ParsingException("Could not get url"); - } - - @Override - public String getUploaderName() throws ParsingException { - String name; - if (searchType.equals(MUSIC_ALBUMS)) { - name = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - } else { - name = getTextFromObject(info.getArray("flexColumns").getObject(1) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - } - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); - } - - @Override - public long getStreamCount() throws ParsingException { - if (searchType.equals(MUSIC_ALBUMS)) return ITEM_COUNT_UNKNOWN; - String count = getTextFromObject(info.getArray("flexColumns").getObject(2) - .getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); - if (count != null && !count.isEmpty()) { - if (count.contains("100+")) { - return ITEM_COUNT_MORE_THAN_100; - } else { - return Long.parseLong(Utils.removeNonDigitCharacters(count)); - } - } - throw new ParsingException("Could not get count"); - } - }); - } - } + private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException { + if (continuations == null) { + return ""; } - } - private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException, IOException, ReCaptchaException { - if (continuations == null) return ""; + final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + final String continuation = nextContinuationData.getString("continuation"); + final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - String continuation = nextContinuationData.getString("continuation"); - String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); - - if (isMusicSearch()) { - return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]; - } else { - return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation - + "&itct=" + clickTrackingParams; - } + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } }