CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[YouTube] Reduce InnerTube response sizes by adding the prettyPrint parameter with the false value 

InnerTube responses return pretty printed responses, which increase responses' size for nothing.

By using the prettyPrint parameter on requests and setting its value to false, responses are not pretty printed anymore, which reduces responses size, and so data transfer and processing times.
This usage has been recently deployed by YouTube on their websites.
This commit is contained in:
TiA4f8R 2022-03-15 11:46:12 +01:00
parent 349ba8db7f
commit 3d38459cf3
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
7 changed files with 33 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -81,6 +81,16 @@ public final class YoutubeParsingHelper {
}
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
/**
* A parameter to disable pretty-printed response of InnerTube requests, to reduce response
* sizes.
*
* <p>
* Sent in query parameters of the requests, <b>after</b> the API key.
* </p>
**/
public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false";
public static final String CPN = "cpn";
public static final String VIDEO_ID = "videoId";
@ -495,7 +505,7 @@ public final class YoutubeParsingHelper {
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
+ HARDCODED_KEY, headers, body);
+ HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();
@ -674,7 +684,7 @@ public final class YoutubeParsingHelper {
ReCaptchaException {
final String url =
"https://music.youtube.com/youtubei/v1/music/get_search_suggestions?alt=json&key="
+ HARDCODED_YOUTUBE_MUSIC_KEY[0];
+ HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
// @formatter:off
final byte[] json = JsonWriter.string()
@ -953,7 +963,7 @@ public final class YoutubeParsingHelper {
headers.put("Content-Type", Collections.singletonList("application/json"));
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
+ getKey(), headers, body, localization);
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
@ -972,7 +982,7 @@ public final class YoutubeParsingHelper {
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
+ "?key=" + ANDROID_YOUTUBE_KEY;
+ "?key=" + ANDROID_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
@ -995,7 +1005,7 @@ public final class YoutubeParsingHelper {
headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2"));
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
+ "?key=" + IOS_YOUTUBE_KEY;
+ "?key=" + IOS_YOUTUBE_KEY + DISABLE_PRETTY_PRINT_PARAMETER;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -395,7 +395,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.done())
.getBytes(UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -90,8 +90,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
headers, body, localization);
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -60,7 +60,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
+ youtubeMusicKeys[0];
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
final String params;

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -317,7 +317,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, body);
} else {
return null;
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -239,7 +239,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final String token = continuationItemRenderer.getObject("continuationEndpoint")
.getObject("continuationCommand").getString("token");
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER;
return new Page(url, token);
}

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View File

@ -6,9 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import com.grack.nanojson.JsonWriter;
@ -93,7 +91,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@ -183,7 +182,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@ -270,7 +270,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@ -389,7 +390,8 @@ public class YoutubeMixPlaylistExtractorTest {
.getBytes(StandardCharsets.UTF_8);
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}