From e9a0d3bd95074ab057c0674e49f2dd333600347f Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 18 Jun 2022 16:07:32 +0200 Subject: [PATCH] [YouTube] Send Content-Type header in all POST requests This header was not sent partially before and was added and guessed by OkHttp. This can create issues when using other HTTP clients than OkHttp, such as Cronet. Some code in the modified classes has been improved and / or deduplicated, and usages of the UTF_8 constant of the Utils class has been replaced by StandardCharsets.UTF_8 where possible. Note that this header has been not added in except in YoutubeDashManifestCreatorsUtils, as an empty body is sent in the POST requests made by this class. --- .../youtube/YoutubeParsingHelper.java | 34 ++++-- .../YoutubeDashManifestCreatorsUtils.java | 4 +- .../extractors/YoutubeChannelExtractor.java | 110 ++++++++---------- .../YoutubeMixPlaylistExtractor.java | 3 + .../YoutubeMusicSearchExtractor.java | 23 +--- .../extractors/YoutubePlaylistExtractor.java | 18 +-- .../extractors/YoutubeSearchExtractor.java | 81 ++++++------- 7 files changed, 127 insertions(+), 146 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index b460448d..280babee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -558,6 +558,7 @@ public final class YoutubeParsingHelper { headers.put("X-YouTube-Client-Name", singletonList("1")); headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_CLIENT_VERSION)); + headers.put("Content-Type", Collections.singletonList("application/json")); // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) @@ -805,7 +806,7 @@ public final class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", singletonList( HARDCODED_YOUTUBE_MUSIC_KEY[2])); headers.put("Origin", singletonList("https://music.youtube.com")); - headers.put("Referer", singletonList("music.youtube.com")); + headers.put("Referer", singletonList("https://music.youtube.com")); headers.put("Content-Type", singletonList("application/json")); final Response response = getDownloader().post(url, headers, json); @@ -1148,13 +1149,12 @@ public final class YoutubeParsingHelper { final Localization localization) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + addYouTubeHeaders(headers); headers.put("Content-Type", singletonList("application/json")); - final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" - + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization); - - return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" + getKey() + + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } public static JsonObject getJsonAndroidPostResponse( @@ -1183,17 +1183,18 @@ public final class YoutubeParsingHelper { @Nonnull final String innerTubeApiKey, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); - headers.put("Content-Type", singletonList("application/json")); headers.put("User-Agent", singletonList(userAgent)); headers.put("X-Goog-Api-Format-Version", singletonList("2")); + headers.put("Content-Type", singletonList("application/json")); final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey + DISABLE_PRETTY_PRINT_PARAMETER; - final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) - ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest, - headers, body, localization); - return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) + ? baseEndpointUrl + : baseEndpointUrl + endPartOfUrlRequest, + headers, body, localization))); } @Nonnull @@ -1396,6 +1397,17 @@ public final class YoutubeParsingHelper { + ")"; } + @Nonnull + public static Map> getYoutubeMusicHeaders() { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1])); + headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2])); + headers.put("Origin", Collections.singletonList("https://music.youtube.com")); + headers.put("Referer", Collections.singletonList("https://music.youtube.com")); + headers.put("Content-Type", Collections.singletonList("application/json")); + return headers; + } + /** * Add required headers and cookies to an existing headers Map. * @see #addClientInfoHeaders(Map) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 045e5dda..46bd3242 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -35,7 +35,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; @@ -707,7 +706,8 @@ public final class YoutubeDashManifestCreatorsUtils { throws CreationException { try { final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); + headers.put("Origin", Collections.singletonList("https://www.youtube.com")); + headers.put("Referer", Collections.singletonList("https://www.youtube.com")); String responseMimeType = ""; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 86a16409..00d1bb75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,12 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -19,7 +17,6 @@ import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -30,15 +27,13 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -85,8 +80,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, - ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { final String channelPath = super.getId(); final String[] channelId = channelPath.split("/"); String id = ""; @@ -103,17 +98,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url", body, getExtractorLocalization()); - if (!isNullOrEmpty(jsonResponse.getObject("error"))) { - final JsonObject errorJsonObject = jsonResponse.getObject("error"); - final int errorCode = errorJsonObject.getInt("code"); - if (errorCode == 404) { - throw new ContentNotAvailableException("This channel doesn't exist."); - } else { - throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); - } - } + checkIfChannelResponseIsValid(jsonResponse); final JsonObject endpoint = jsonResponse.getObject("endpoint"); @@ -151,17 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final JsonObject jsonResponse = getJsonPostResponse("browse", body, getExtractorLocalization()); - if (!isNullOrEmpty(jsonResponse.getObject("error"))) { - final JsonObject errorJsonObject = jsonResponse.getObject("error"); - final int errorCode = errorJsonObject.getInt("code"); - if (errorCode == 404) { - throw new ContentNotAvailableException("This channel doesn't exist."); - } else { - throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); - } - } + checkIfChannelResponseIsValid(jsonResponse); final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions") .getObject(0) @@ -199,6 +174,21 @@ public class YoutubeChannelExtractor extends ChannelExtractor { YoutubeParsingHelper.defaultAlertsCheck(initialData); } + private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse) + throws ContentNotAvailableException { + if (!isNullOrEmpty(jsonResponse.getObject("error"))) { + final JsonObject errorJsonObject = jsonResponse.getObject("error"); + final int errorCode = errorJsonObject.getInt("code"); + if (errorCode == 404) { + throw new ContentNotAvailableException("This channel doesn't exist."); + } else { + throw new ContentNotAvailableException("Got error:\"" + + errorJsonObject.getString("status") + "\": " + + errorJsonObject.getString("message")); + } + } + } + @Nonnull @Override public String getUrl() throws ParsingException { @@ -354,8 +344,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, - ExtractionException { + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -363,14 +353,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final List channelIds = page.getIds(); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); - final Response response = getDownloader().post(page.getUrl(), null, page.getBody(), + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), getExtractorLocalization()); - final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); - final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") .getObject(0) .getObject("appendContinuationItemsAction"); @@ -383,8 +369,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nullable private Page getNextPageFrom(final JsonObject continuations, - final List channelIds) throws IOException, - ExtractionException { + final List channelIds) + throws IOException, ExtractionException { if (isNullOrEmpty(continuations)) { return null; } @@ -462,39 +448,43 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nullable private JsonObject getVideoTab() throws ParsingException { - if (this.videoTab != null) { - return this.videoTab; + if (videoTab != null) { + return videoTab; } final JsonArray tabs = initialData.getObject("contents") .getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); - JsonObject foundVideoTab = null; - for (final Object tab : tabs) { - if (((JsonObject) tab).has("tabRenderer")) { - if (((JsonObject) tab).getObject("tabRenderer").getString("title", - "").equals("Videos")) { - foundVideoTab = ((JsonObject) tab).getObject("tabRenderer"); - break; - } - } - } + final JsonObject foundVideoTab = tabs.stream() + .filter(Objects::nonNull) + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(tab -> tab.has("tabRenderer") + && tab.getObject("tabRenderer") + .getString("title", "") + .equals("Videos")) + .findFirst() + .map(tab -> tab.getObject("tabRenderer")) + .orElseThrow( + () -> new ContentNotSupportedException("This channel has no Videos tab")); - if (foundVideoTab == null) { - throw new ContentNotSupportedException("This channel has no Videos tab"); - } - - final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("messageRenderer").getObject("text")); + final String messageRendererText = getTextFromObject( + foundVideoTab.getObject("content") + .getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) + .getObject("itemSectionRenderer") + .getArray("contents") + .getObject(0) + .getObject("messageRenderer") + .getObject("text")); if (messageRendererText != null && messageRendererText.equals("This channel has no videos.")) { return null; } - this.videoTab = foundVideoTab; + videoTab = foundVideoTab; return foundVideoTab; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 2a9393bc..96d1924a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -38,6 +38,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -91,6 +92,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final Map> headers = new HashMap<>(); // Cookie is required due to consent addYouTubeHeaders(headers); + headers.put("Content-Type", Collections.singletonList("application/json")); final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization); @@ -224,6 +226,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final Map> headers = new HashMap<>(); // Cookie is required due to consent addYouTubeHeaders(headers); + headers.put("Content-Type", Collections.singletonList("application/json")); final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), getExtractorLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index b7bcb5b6..a4c91de3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -5,6 +5,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; @@ -39,9 +40,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -116,15 +115,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - - final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, - json)); + final String responseBody = getValidJsonResponseBody(getDownloader().post(url, + getYoutubeMusicHeaders(), json)); try { initialData = JsonParser.object().from(responseBody); @@ -251,15 +243,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on - final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1])); - headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2])); - headers.put("Origin", Collections.singletonList("https://music.youtube.com")); - headers.put("Referer", Collections.singletonList("music.youtube.com")); - headers.put("Content-Type", Collections.singletonList("application/json")); - final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), - headers, json)); + getYoutubeMusicHeaders(), json)); final JsonObject ajaxJson; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index bfdadc73..2ad98408 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -2,25 +2,21 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; - import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -31,17 +27,12 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; public class YoutubePlaylistExtractor extends PlaylistExtractor { // Names of some objects in JSON response frequently used in this class @@ -349,12 +340,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); - final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), getExtractorLocalization()); - final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions") .getObject(0) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 1489963e..7d374e9e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -5,7 +5,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -13,8 +12,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; @@ -34,10 +31,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.List; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /* * Created by Christian Schabesberger on 22.07.2018 @@ -105,10 +102,14 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { final JsonObject itemSectionRenderer = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") + .getArray("contents") + .getObject(0) .getObject("itemSectionRenderer"); - final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0) .getObject("didYouMeanRenderer"); final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents") .getObject(0) @@ -138,8 +139,10 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public List getMetaInfo() throws ParsingException { return YoutubeParsingHelper.getMetaInfo( - initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer") + initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") .getArray("contents")); } @@ -149,20 +152,23 @@ public class YoutubeSearchExtractor extends SearchExtractor { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final JsonArray sections = initialData.getObject("contents") - .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") - .getObject("sectionListRenderer").getArray("contents"); + .getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents") + .getObject("sectionListRenderer") + .getArray("contents"); Page nextPage = null; for (final Object section : sections) { - if (((JsonObject) section).has("itemSectionRenderer")) { - final JsonObject itemSectionRenderer = ((JsonObject) section) - .getObject("itemSectionRenderer"); + final JsonObject sectionJsonObject = (JsonObject) section; + if (sectionJsonObject.has("itemSectionRenderer")) { + final JsonObject itemSectionRenderer = + sectionJsonObject.getObject("itemSectionRenderer"); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - } else if (((JsonObject) section).has("continuationItemRenderer")) { - nextPage = getNextPageFrom(((JsonObject) section) - .getObject("continuationItemRenderer")); + } else if (sectionJsonObject.has("continuationItemRenderer")) { + nextPage = getNextPageFrom( + sectionJsonObject.getObject("continuationItemRenderer")); } } @@ -187,22 +193,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getBytes(StandardCharsets.UTF_8); // @formatter:on - final String responseBody = getValidJsonResponseBody(getDownloader().post( - page.getUrl(), new HashMap<>(), json)); - - final JsonObject ajaxJson; - try { - ajaxJson = JsonParser.object().from(responseBody); - } catch (final JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } + final JsonObject ajaxJson = getJsonPostResponse("search", json, localization); final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands") - .getObject(0).getObject("appendContinuationItemsAction") + .getObject(0) + .getObject("appendContinuationItemsAction") .getArray("continuationItems"); final JsonArray contents = continuationItems.getObject(0) - .getObject("itemSectionRenderer").getArray("contents"); + .getObject("itemSectionRenderer") + .getArray("contents"); collectStreamsFrom(collector, contents); return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1) @@ -210,28 +210,30 @@ public class YoutubeSearchExtractor extends SearchExtractor { } private void collectStreamsFrom(final MultiInfoItemsCollector collector, - final JsonArray contents) throws NothingFoundException, - ParsingException { + @Nonnull final JsonArray contents) + throws NothingFoundException, ParsingException { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (final Object content : contents) { final JsonObject item = (JsonObject) content; if (item.has("backgroundPromoRenderer")) { - throw new NothingFoundException(getTextFromObject( - item.getObject("backgroundPromoRenderer").getObject("bodyText"))); + throw new NothingFoundException( + getTextFromObject(item.getObject("backgroundPromoRenderer") + .getObject("bodyText"))); } else if (item.has("videoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(item - .getObject("videoRenderer"), timeAgoParser)); + collector.commit(new YoutubeStreamInfoItemExtractor( + item.getObject("videoRenderer"), timeAgoParser)); } else if (item.has("channelRenderer")) { - collector.commit(new YoutubeChannelInfoItemExtractor(item - .getObject("channelRenderer"))); + collector.commit(new YoutubeChannelInfoItemExtractor( + item.getObject("channelRenderer"))); } else if (item.has("playlistRenderer")) { - collector.commit(new YoutubePlaylistInfoItemExtractor(item - .getObject("playlistRenderer"))); + collector.commit(new YoutubePlaylistInfoItemExtractor( + item.getObject("playlistRenderer"))); } } } + @Nullable private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException { if (isNullOrEmpty(continuationItemRenderer)) { @@ -239,7 +241,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { } final String token = continuationItemRenderer.getObject("continuationEndpoint") - .getObject("continuationCommand").getString("token"); + .getObject("continuationCommand") + .getString("token"); final String url = YOUTUBEI_V1_URL + "search?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER;