CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[YouTube] Send Content-Type header in all POST requests 

This header was not sent partially before and was added and guessed by OkHttp. This can create issues when using other HTTP clients than OkHttp, such as Cronet.

Some code in the modified classes has been improved and / or deduplicated, and usages of the UTF_8 constant of the Utils class has been replaced by StandardCharsets.UTF_8 where possible.

Note that this header has been not added in except in YoutubeDashManifestCreatorsUtils, as an empty body is sent in the POST requests made by this class.
This commit is contained in:
AudricV 2022-06-18 16:07:32 +02:00 committed by Stypox
parent b9e463de49
commit e9a0d3bd95
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23
7 changed files with 127 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -558,6 +558,7 @@ public final class YoutubeParsingHelper {
headers.put("X-YouTube-Client-Name", singletonList("1"));
headers.put("X-YouTube-Client-Version",
singletonList(HARDCODED_CLIENT_VERSION));
headers.put("Content-Type", Collections.singletonList("application/json"));
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
@ -805,7 +806,7 @@ public final class YoutubeParsingHelper {
headers.put("X-YouTube-Client-Version", singletonList(
HARDCODED_YOUTUBE_MUSIC_KEY[2]));
headers.put("Origin", singletonList("https://music.youtube.com"));
headers.put("Referer", singletonList("music.youtube.com"));
headers.put("Referer", singletonList("https://music.youtube.com"));
headers.put("Content-Type", singletonList("application/json"));
final Response response = getDownloader().post(url, headers, json);
@ -1148,13 +1149,12 @@ public final class YoutubeParsingHelper {
final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
addYouTubeHeaders(headers);
headers.put("Content-Type", singletonList("application/json"));
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
}
public static JsonObject getJsonAndroidPostResponse(
@ -1183,17 +1183,18 @@ public final class YoutubeParsingHelper {
@Nonnull final String innerTubeApiKey,
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Content-Type", singletonList("application/json"));
headers.put("User-Agent", singletonList(userAgent));
headers.put("X-Goog-Api-Format-Version", singletonList("2"));
headers.put("Content-Type", singletonList("application/json"));
final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
+ DISABLE_PRETTY_PRINT_PARAMETER;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl
: baseEndpointUrl + endPartOfUrlRequest,
headers, body, localization)));
}
@Nonnull
@ -1396,6 +1397,17 @@ public final class YoutubeParsingHelper {
+ ")";
}
@Nonnull
public static Map<String, List<String>> getYoutubeMusicHeaders() {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
return headers;
}
/**
* Add required headers and cookies to an existing headers Map.
* @see #addClientInfoHeaders(Map)

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
View File

@ -35,7 +35,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
@ -707,7 +706,8 @@ public final class YoutubeDashManifestCreatorsUtils {
throws CreationException {
try {
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
String responseMimeType = "";

110
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -2,12 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -19,7 +17,6 @@ import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -30,15 +27,13 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -85,8 +80,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String channelPath = super.getId();
final String[] channelId = channelPath.split("/");
String id = "";
@ -103,17 +98,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
body, getExtractorLocalization());
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getObject("endpoint");
@ -151,17 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
getExtractorLocalization());
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
checkIfChannelResponseIsValid(jsonResponse);
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
@ -199,6 +174,21 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
YoutubeParsingHelper.defaultAlertsCheck(initialData);
}
private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
throws ContentNotAvailableException {
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
@ -354,8 +344,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@ -363,14 +353,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final List<String> channelIds = page.getIds();
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
@ -383,8 +369,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds) throws IOException,
ExtractionException {
final List<String> channelIds)
throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
@ -462,39 +448,43 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (this.videoTab != null) {
return this.videoTab;
if (videoTab != null) {
return videoTab;
}
final JsonArray tabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
JsonObject foundVideoTab = null;
for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title",
"").equals("Videos")) {
foundVideoTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}
}
}
final JsonObject foundVideoTab = tabs.stream()
.filter(Objects::nonNull)
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(tab -> tab.has("tabRenderer")
&& tab.getObject("tabRenderer")
.getString("title", "")
.equals("Videos"))
.findFirst()
.map(tab -> tab.getObject("tabRenderer"))
.orElseThrow(
() -> new ContentNotSupportedException("This channel has no Videos tab"));
if (foundVideoTab == null) {
throw new ContentNotSupportedException("This channel has no Videos tab");
}
final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("messageRenderer").getObject("text"));
final String messageRendererText = getTextFromObject(
foundVideoTab.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0)
.getObject("messageRenderer")
.getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
}
this.videoTab = foundVideoTab;
videoTab = foundVideoTab;
return foundVideoTab;
}
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -38,6 +38,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -91,6 +92,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final Map<String, List<String>> headers = new HashMap<>();
// Cookie is required due to consent
addYouTubeHeaders(headers);
headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
@ -224,6 +226,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final Map<String, List<String>> headers = new HashMap<>();
// Cookie is required due to consent
addYouTubeHeaders(headers);
headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
getExtractorLocalization());

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -5,6 +5,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
@ -39,9 +40,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
@ -116,15 +115,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers,
json));
final String responseBody = getValidJsonResponseBody(getDownloader().post(url,
getYoutubeMusicHeaders(), json));
try {
initialData = JsonParser.object().from(responseBody);
@ -251,15 +243,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(),
headers, json));
getYoutubeMusicHeaders(), json));
final JsonObject ajaxJson;
try {

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -2,25 +2,21 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -31,17 +27,12 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
public class YoutubePlaylistExtractor extends PlaylistExtractor {
// Names of some objects in JSON response frequently used in this class
@ -349,12 +340,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)

81
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -5,7 +5,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -13,8 +12,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
@ -34,10 +31,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/*
* Created by Christian Schabesberger on 22.07.2018
@ -105,10 +102,14 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public String getSearchSuggestion() throws ParsingException {
final JsonObject itemSectionRenderer = initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer");
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0)
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
.getObject(0)
.getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
.getObject(0)
@ -138,8 +139,10 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer")
initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents")
.getObject("sectionListRenderer")
.getArray("contents"));
}
@ -149,20 +152,23 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonArray sections = initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents");
.getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents")
.getObject("sectionListRenderer")
.getArray("contents");
Page nextPage = null;
for (final Object section : sections) {
if (((JsonObject) section).has("itemSectionRenderer")) {
final JsonObject itemSectionRenderer = ((JsonObject) section)
.getObject("itemSectionRenderer");
final JsonObject sectionJsonObject = (JsonObject) section;
if (sectionJsonObject.has("itemSectionRenderer")) {
final JsonObject itemSectionRenderer =
sectionJsonObject.getObject("itemSectionRenderer");
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
} else if (((JsonObject) section).has("continuationItemRenderer")) {
nextPage = getNextPageFrom(((JsonObject) section)
.getObject("continuationItemRenderer"));
} else if (sectionJsonObject.has("continuationItemRenderer")) {
nextPage = getNextPageFrom(
sectionJsonObject.getObject("continuationItemRenderer"));
}
}
@ -187,22 +193,16 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
final String responseBody = getValidJsonResponseBody(getDownloader().post(
page.getUrl(), new HashMap<>(), json));
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(responseBody);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
final JsonObject ajaxJson = getJsonPostResponse("search", json, localization);
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
.getObject(0).getObject("appendContinuationItemsAction")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems");
final JsonArray contents = continuationItems.getObject(0)
.getObject("itemSectionRenderer").getArray("contents");
.getObject("itemSectionRenderer")
.getArray("contents");
collectStreamsFrom(collector, contents);
return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1)
@ -210,28 +210,30 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
private void collectStreamsFrom(final MultiInfoItemsCollector collector,
final JsonArray contents) throws NothingFoundException,
ParsingException {
@Nonnull final JsonArray contents)
throws NothingFoundException, ParsingException {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object content : contents) {
final JsonObject item = (JsonObject) content;
if (item.has("backgroundPromoRenderer")) {
throw new NothingFoundException(getTextFromObject(
item.getObject("backgroundPromoRenderer").getObject("bodyText")));
throw new NothingFoundException(
getTextFromObject(item.getObject("backgroundPromoRenderer")
.getObject("bodyText")));
} else if (item.has("videoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(item
.getObject("videoRenderer"), timeAgoParser));
collector.commit(new YoutubeStreamInfoItemExtractor(
item.getObject("videoRenderer"), timeAgoParser));
} else if (item.has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(item
.getObject("channelRenderer")));
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("channelRenderer")));
} else if (item.has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(item
.getObject("playlistRenderer")));
collector.commit(new YoutubePlaylistInfoItemExtractor(
item.getObject("playlistRenderer")));
}
}
}
@Nullable
private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuationItemRenderer)) {
@ -239,7 +241,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
final String token = continuationItemRenderer.getObject("continuationEndpoint")
.getObject("continuationCommand").getString("token");
.getObject("continuationCommand")
.getString("token");
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER;