From 3d38459cf37ff3d34decec4f01035861706f43b6 Mon Sep 17 00:00:00 2001
From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com>
Date: Tue, 15 Mar 2022 11:46:12 +0100
Subject: [PATCH] [YouTube] Reduce InnerTube response sizes by adding the
prettyPrint parameter with the false value
InnerTube responses return pretty printed responses, which increase responses' size for nothing.
By using the prettyPrint parameter on requests and setting its value to false, responses are not pretty printed anymore, which reduces responses size, and so data transfer and processing times.
This usage has been recently deployed by YouTube on their websites.
---
.../youtube/YoutubeParsingHelper.java | 20 ++++++++++++++-----
.../extractors/YoutubeChannelExtractor.java | 3 ++-
.../YoutubeMixPlaylistExtractor.java | 4 ++--
.../YoutubeMusicSearchExtractor.java | 2 +-
.../extractors/YoutubePlaylistExtractor.java | 3 ++-
.../extractors/YoutubeSearchExtractor.java | 3 ++-
.../YoutubeMixPlaylistExtractorTest.java | 16 ++++++++-------
7 files changed, 33 insertions(+), 18 deletions(-)
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
index a36c511d..eb67a5bb 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@@ -81,6 +81,16 @@ public final class YoutubeParsingHelper {
}
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
+
+ /**
+ * A parameter to disable pretty-printed response of InnerTube requests, to reduce response
+ * sizes.
+ *
+ *
+ * Sent in query parameters of the requests, after the API key.
+ *
+ **/
+ public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false";
public static final String CPN = "cpn";
public static final String VIDEO_ID = "videoId";
@@ -495,7 +505,7 @@ public final class YoutubeParsingHelper {
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
- + HARDCODED_KEY, headers, body);
+ + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();
@@ -674,7 +684,7 @@ public final class YoutubeParsingHelper {
ReCaptchaException {
final String url =
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
- + HARDCODED_YOUTUBE_MUSIC_KEY[0];
+ + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
// @formatter:off
final byte[] json = JsonWriter.string()
@@ -953,7 +963,7 @@ public final class YoutubeParsingHelper {
headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
- + getKey(), headers, body, localization);
+ + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
@@ -972,7 +982,7 @@ public final class YoutubeParsingHelper {
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
- + "?key=" + ANDROID_YOUTUBE_KEY;
+ + "?key=" + ANDROID_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
@@ -995,7 +1005,7 @@ public final class YoutubeParsingHelper {
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
- + "?key=" + IOS_YOUTUBE_KEY;
+ + "?key=" + IOS_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
index ec04e358..6209317d 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
@@ -395,7 +395,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.done())
.getBytes(UTF_8);
- return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
+ return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
index 327c1077..788d0b4d 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
@@ -90,8 +90,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final Map> headers = new HashMap<>();
addClientInfoHeaders(headers);
- final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
- headers, body, localization);
+ final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
index 5f45a335..022ea167 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
@@ -60,7 +60,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
- + youtubeMusicKeys[0];
+ + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
final String params;
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
index f52608df..d2f2c877 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
@@ -317,7 +317,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.done())
.getBytes(StandardCharsets.UTF_8);
- return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
+ return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER, body);
} else {
return null;
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
index 895a8594..b985f7bc 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
@@ -239,7 +239,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final String token = continuationItemRenderer.getObject("continuationEndpoint")
.getObject("continuationCommand").getString("token");
- final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
+ final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
+ + DISABLE_PRETTY_PRINT_PARAMETER;
return new Page(url, token);
}
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
index 27d7d98e..b562d659 100644
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
@@ -6,9 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import com.grack.nanojson.JsonWriter;
@@ -93,7 +91,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@@ -183,7 +182,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@@ -270,7 +270,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@@ -389,7 +390,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage streams = extractor.getPage(new Page(
- YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+ YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
+ null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}