diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index b460448d..280babee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -558,6 +558,7 @@ public final class YoutubeParsingHelper { headers.put("X-YouTube-Client-Name", singletonList("1")); headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_CLIENT_VERSION)); + headers.put("Content-Type", Collections.singletonList("application/json")); // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) @@ -805,7 +806,7 @@ public final class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", singletonList( HARDCODED_YOUTUBE_MUSIC_KEY[2])); headers.put("Origin", singletonList("https://music.youtube.com")); - headers.put("Referer", singletonList("music.youtube.com")); + headers.put("Referer", singletonList("https://music.youtube.com")); headers.put("Content-Type", singletonList("application/json")); final Response response = getDownloader().post(url, headers, json); @@ -1148,13 +1149,12 @@ public final class YoutubeParsingHelper { final Localization localization) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + addYouTubeHeaders(headers); headers.put("Content-Type", singletonList("application/json")); - final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" - + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization); - - return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" + getKey() + + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } public static JsonObject getJsonAndroidPostResponse( @@ -1183,17 +1183,18 @@ public final class YoutubeParsingHelper { @Nonnull final String innerTubeApiKey, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); - headers.put("Content-Type", singletonList("application/json")); headers.put("User-Agent", singletonList(userAgent)); headers.put("X-Goog-Api-Format-Version", singletonList("2")); + headers.put("Content-Type", singletonList("application/json")); final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey + DISABLE_PRETTY_PRINT_PARAMETER; - final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) - ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest, - headers, body, localization); - return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) + ? baseEndpointUrl + : baseEndpointUrl + endPartOfUrlRequest, + headers, body, localization))); } @Nonnull @@ -1396,6 +1397,17 @@ public final class YoutubeParsingHelper { + ")"; } + @Nonnull + public static Map> getYoutubeMusicHeaders() { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + return headers; + } + /** * Add required headers and cookies to an existing headers Map. * @see #addClientInfoHeaders(Map) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 045e5dda..46bd3242 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -35,7 +35,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; @@ -707,7 +706,8 @@ public final class YoutubeDashManifestCreatorsUtils { throws CreationException { try { final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + headers.put("Origin", Collections.singletonList("https://www.youtube.com")); + headers.put("Referer", Collections.singletonList("https://www.youtube.com")); String responseMimeType = ""; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 86a16409..00d1bb75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,12 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -19,7 +17,6 @@ import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -30,15 +27,13 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -85,8 +80,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, - ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { final String channelPath = super.getId(); final String[] channelId = channelPath.split("/"); String id = ""; @@ -103,17 +98,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url", body, getExtractorLocalization()); - if (!isNullOrEmpty(jsonResponse.getObject("error"))) { - final JsonObject errorJsonObject = jsonResponse.getObject("error"); - final int errorCode = errorJsonObject.getInt("code"); - if (errorCode == 404) { - throw new ContentNotAvailableException("This channel doesn't exist."); - } else { - throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); - } - } + checkIfChannelResponseIsValid(jsonResponse); final JsonObject endpoint = jsonResponse.getObject("endpoint"); @@ -151,17 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final JsonObject jsonResponse = getJsonPostResponse("browse", body, getExtractorLocalization()); - if (!isNullOrEmpty(jsonResponse.getObject("error"))) { - final JsonObject errorJsonObject = jsonResponse.getObject("error"); - final int errorCode = errorJsonObject.getInt("code"); - if (errorCode == 404) { - throw new ContentNotAvailableException("This channel doesn't exist."); - } else { - throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); - } - } + checkIfChannelResponseIsValid(jsonResponse); final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions") .getObject(0) @@ -199,6 +174,21 @@ public class YoutubeChannelExtractor extends ChannelExtractor { YoutubeParsingHelper.defaultAlertsCheck(initialData); } + private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse) + throws ContentNotAvailableException { + if (!isNullOrEmpty(jsonResponse.getObject("error"))) { + final JsonObject errorJsonObject = jsonResponse.getObject("error"); + final int errorCode = errorJsonObject.getInt("code"); + if (errorCode == 404) { + throw new ContentNotAvailableException("This channel doesn't exist."); + } else { + throw new ContentNotAvailableException("Got error:\"" + + errorJsonObject.getString("status") + "\": " + + errorJsonObject.getString("message")); + } + } + } + @Nonnull @Override public String getUrl() throws ParsingException { @@ -354,8 +344,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, - ExtractionException { + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -363,14 +353,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final List channelIds = page.getIds(); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); - final Response response = getDownloader().post(page.getUrl(), null, page.getBody(), + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), getExtractorLocalization()); - final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); - final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") .getObject(0) .getObject("appendContinuationItemsAction"); @@ -383,8 +369,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nullable private Page getNextPageFrom(final JsonObject continuations, - final List channelIds) throws IOException, - ExtractionException { + final List channelIds) + throws IOException, ExtractionException { if (isNullOrEmpty(continuations)) { return null; } @@ -462,39 +448,43 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nullable private JsonObject getVideoTab() throws ParsingException { - if (this.videoTab != null) { - return this.videoTab; + if (videoTab != null) { + return videoTab; } final JsonArray tabs = initialData.getObject("contents") .getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); - JsonObject foundVideoTab = null; - for (final Object tab : tabs) { - if (((JsonObject) tab).has("tabRenderer")) { - if (((JsonObject) tab).getObject("tabRenderer").getString("title", - "").equals("Videos")) { - foundVideoTab = ((JsonObject) tab).getObject("tabRenderer"); - break; - } - } - } + final JsonObject foundVideoTab = tabs.stream() + .filter(Objects::nonNull) + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(tab -> tab.has("tabRenderer") + && tab.getObject("tabRenderer") + .getString("title", "") + .equals("Videos")) + .findFirst() + .map(tab -> tab.getObject("tabRenderer")) + .orElseThrow( + () -> new ContentNotSupportedException("This channel has no Videos tab")); - if (foundVideoTab == null) { - throw new ContentNotSupportedException("This channel has no Videos tab"); - } - - final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("messageRenderer").getObject("text")); + final String messageRendererText = getTextFromObject( + foundVideoTab.getObject("content") + .getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) + .getObject("itemSectionRenderer") + .getArray("contents") + .getObject(0) + .getObject("messageRenderer") + .getObject("text")); if (messageRendererText != null && messageRendererText.equals("This channel has no videos.")) { return null; } - this.videoTab = foundVideoTab; + videoTab = foundVideoTab; return foundVideoTab; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 2a9393bc..96d1924a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -38,6 +38,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -91,6 +92,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final Map> headers = new HashMap<>(); // Cookie is required due to consent addYouTubeHeaders(headers); + headers.put("Content-Type", Collections.singletonList("application/json")); final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization); @@ -224,6 +226,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final Map> headers = new HashMap<>(); // Cookie is required due to consent addYouTubeHeaders(headers); + headers.put("Content-Type", Collections.singletonList("application/json")); final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), getExtractorLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index b7bcb5b6..a4c91de3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -5,6 +5,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; @@ -39,9 +40,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -116,15 +115,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, - json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(url, + getYoutubeMusicHeaders(), json)); try { initialData = JsonParser.object().from(responseBody); @@ -251,15 +243,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), - headers, json)); + getYoutubeMusicHeaders(), json)); final JsonObject ajaxJson; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index bfdadc73..2ad98408 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -2,25 +2,21 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; - import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -31,17 +27,12 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; public class YoutubePlaylistExtractor extends PlaylistExtractor { // Names of some objects in JSON response frequently used in this class @@ -349,12 +340,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); - final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), getExtractorLocalization()); - final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions") .getObject(0) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 1489963e..7d374e9e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -5,7 +5,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -13,8 +12,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; @@ -34,10 +31,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.List; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /* * Created by Christian Schabesberger on 22.07.2018 @@ -105,10 +102,14 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { final JsonObject itemSectionRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) .getObject("itemSectionRenderer"); - final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0) .getObject("didYouMeanRenderer"); final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents") .getObject(0) @@ -138,8 +139,10 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public List getMetaInfo() throws ParsingException { return YoutubeParsingHelper.getMetaInfo( - initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer") + initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") .getArray("contents")); } @@ -149,20 +152,23 @@ public class YoutubeSearchExtractor extends SearchExtractor { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final JsonArray sections = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents"); + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") + .getArray("contents"); Page nextPage = null; for (final Object section : sections) { - if (((JsonObject) section).has("itemSectionRenderer")) { - final JsonObject itemSectionRenderer = ((JsonObject) section) - .getObject("itemSectionRenderer"); + final JsonObject sectionJsonObject = (JsonObject) section; + if (sectionJsonObject.has("itemSectionRenderer")) { + final JsonObject itemSectionRenderer = + sectionJsonObject.getObject("itemSectionRenderer"); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - } else if (((JsonObject) section).has("continuationItemRenderer")) { - nextPage = getNextPageFrom(((JsonObject) section) - .getObject("continuationItemRenderer")); + } else if (sectionJsonObject.has("continuationItemRenderer")) { + nextPage = getNextPageFrom( + sectionJsonObject.getObject("continuationItemRenderer")); } } @@ -187,22 +193,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getBytes(StandardCharsets.UTF_8); // @formatter:on - final String responseBody = getValidJsonResponseBody(getDownloader().post( - page.getUrl(), new HashMap<>(), json)); - - final JsonObject ajaxJson; - try { - ajaxJson = JsonParser.object().from(responseBody); - } catch (final JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } + final JsonObject ajaxJson = getJsonPostResponse("search", json, localization); final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands") - .getObject(0).getObject("appendContinuationItemsAction") + .getObject(0) + .getObject("appendContinuationItemsAction") .getArray("continuationItems"); final JsonArray contents = continuationItems.getObject(0) - .getObject("itemSectionRenderer").getArray("contents"); + .getObject("itemSectionRenderer") + .getArray("contents"); collectStreamsFrom(collector, contents); return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1) @@ -210,28 +210,30 @@ public class YoutubeSearchExtractor extends SearchExtractor { } private void collectStreamsFrom(final MultiInfoItemsCollector collector, - final JsonArray contents) throws NothingFoundException, - ParsingException { + @Nonnull final JsonArray contents) + throws NothingFoundException, ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (final Object content : contents) { final JsonObject item = (JsonObject) content; if (item.has("backgroundPromoRenderer")) { - throw new NothingFoundException(getTextFromObject( - item.getObject("backgroundPromoRenderer").getObject("bodyText"))); + throw new NothingFoundException( + getTextFromObject(item.getObject("backgroundPromoRenderer") + .getObject("bodyText"))); } else if (item.has("videoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(item - .getObject("videoRenderer"), timeAgoParser)); + collector.commit(new YoutubeStreamInfoItemExtractor( + item.getObject("videoRenderer"), timeAgoParser)); } else if (item.has("channelRenderer")) { - collector.commit(new YoutubeChannelInfoItemExtractor(item - .getObject("channelRenderer"))); + collector.commit(new YoutubeChannelInfoItemExtractor( + item.getObject("channelRenderer"))); } else if (item.has("playlistRenderer")) { - collector.commit(new YoutubePlaylistInfoItemExtractor(item - .getObject("playlistRenderer"))); + collector.commit(new YoutubePlaylistInfoItemExtractor( + item.getObject("playlistRenderer"))); } } } + @Nullable private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException { if (isNullOrEmpty(continuationItemRenderer)) { @@ -239,7 +241,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { } final String token = continuationItemRenderer.getObject("continuationEndpoint") - .getObject("continuationCommand").getString("token"); + .getObject("continuationCommand") + .getString("token"); final String url = YOUTUBEI_V1_URL + "search?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER;