CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Reformat some code and don't use the clickTrackingParams in continuations of YouTube Music search results

Browse source 
The clickTrackingParams of YouTube Music search results are not needed to get continuations. This commit removes their use, which may improve privacy.
This commit is contained in:
TiA4f8R 2021-08-01 13:16:46 +02:00
parent f8197da09e
commit b74a39c176
No known key found for this signature in database
GPG key ID: E6D3E7F5949450DD
3 changed files with 111 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java
View file

@ -79,11 +79,13 @@ public class YoutubeThrottlingDecrypter {
} }
} }
@Nonnull
private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) { private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
final String functionBase = functionName + "=function"; final String functionBase = functionName + "=function";
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";"; return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";";
} }
@Nonnull
private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException { private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n", Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
Pattern.DOTALL); Pattern.DOTALL);

172
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View file

@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -33,16 +34,18 @@ import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubeMusicSearchExtractor extends SearchExtractor { public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData; private JsonObject initialData;
public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { public YoutubeMusicSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, public void onFetchPage(@Nonnull final Downloader downloader)
ExtractionException { throws IOException, ExtractionException {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0]; final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
+ youtubeMusicKeys[0];
final String params; final String params;
@ -68,7 +71,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
} }
// @formatter:off // @formatter:off
byte[] json = JsonWriter.string() final byte[] json = JsonWriter.string()
.object() .object()
.object("context") .object("context")
.object("client") .object("client")
@ -104,11 +107,12 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json")); headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json)); final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers,
json));
try { try {
initialData = JsonParser.object().from(responseBody); initialData = JsonParser.object().from(responseBody);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON", e); throw new ParsingException("Could not parse JSON", e);
} }
} }
@ -122,20 +126,26 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Nonnull @Nonnull
@Override @Override
public String getSearchSuggestion() throws ParsingException { public String getSearchSuggestion() throws ParsingException {
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents").getObject(0).getObject("itemSectionRenderer"); final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
"tabRenderer.content.sectionListRenderer.contents")
.getObject(0)
.getObject("itemSectionRenderer");
if (itemSectionRenderer.isEmpty()) { if (itemSectionRenderer.isEmpty()) {
return ""; return "";
} }
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
.getObject(0).getObject("didYouMeanRenderer"); .getObject(0).getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
.getObject(0)
.getObject("showingResultsForRenderer"); .getObject("showingResultsForRenderer");
if (!didYouMeanRenderer.isEmpty()) { if (!didYouMeanRenderer.isEmpty()) {
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
} else if (!showingResultsForRenderer.isEmpty()) { } else if (!showingResultsForRenderer.isEmpty()) {
return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); return JsonUtils.getString(showingResultsForRenderer,
"correctedQueryEndpoint.searchEndpoint.query");
} else { } else {
return ""; return "";
} }
@ -143,16 +153,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public boolean isCorrectedSearch() throws ParsingException { public boolean isCorrectedSearch() throws ParsingException {
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents").getObject(0).getObject("itemSectionRenderer"); final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
"tabRenderer.content.sectionListRenderer.contents")
.getObject(0)
.getObject("itemSectionRenderer");
if (itemSectionRenderer.isEmpty()) { if (itemSectionRenderer.isEmpty()) {
return false; return false;
} }
JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0); JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0);
final boolean corrected = firstContent return firstContent.has("didYouMeanRenderer")
.has("didYouMeanRenderer") || firstContent.has("showingResultsForRenderer"); || firstContent.has("showingResultsForRenderer");
return corrected;
} }
@Nonnull @Nonnull
@ -163,16 +176,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException, IOException { public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents"); final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData,
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
"tabRenderer.content.sectionListRenderer.contents");
Page nextPage = null; Page nextPage = null;
for (Object content : contents) { for (final Object content : contents) {
if (((JsonObject) content).has("musicShelfRenderer")) { if (((JsonObject) content).has("musicShelfRenderer")) {
final JsonObject musicShelfRenderer = ((JsonObject) content).getObject("musicShelfRenderer"); final JsonObject musicShelfRenderer = ((JsonObject) content)
.getObject("musicShelfRenderer");
collectMusicStreamsFrom(collector, musicShelfRenderer.getArray("contents")); collectMusicStreamsFrom(collector, musicShelfRenderer.getArray("contents"));
@ -229,16 +245,18 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json")); headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json)); final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(),
headers, json));
final JsonObject ajaxJson; final JsonObject ajaxJson;
try { try {
ajaxJson = JsonParser.object().from(responseBody); ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON", e); throw new ParsingException("Could not parse JSON", e);
} }
final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation"); final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents")
.getObject("musicShelfContinuation");
collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents"));
final JsonArray continuations = musicShelfContinuation.getArray("continuations"); final JsonArray continuations = musicShelfContinuation.getArray("continuations");
@ -246,31 +264,32 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
} }
private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) { private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector,
@Nonnull final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) { for (final Object item : videos) {
final JsonObject info = ((JsonObject) item) final JsonObject info = ((JsonObject) item)
.getObject("musicResponsiveListItemRenderer", null); .getObject("musicResponsiveListItemRenderer", null);
if (info != null) { if (info != null) {
final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", EMPTY_STRING); final String displayPolicy = info.getString("musicItemRendererDisplayPolicy",
EMPTY_STRING);
if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) {
continue; // no info about video URL available continue; // No info about video URL available
} }
final JsonObject flexColumnRenderer = info final JsonObject flexColumnRenderer = info.getArray("flexColumns")
.getArray("flexColumns")
.getObject(1) .getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer"); .getObject("musicResponsiveListItemFlexColumnRenderer");
final JsonArray descriptionElements = flexColumnRenderer final JsonArray descriptionElements = flexColumnRenderer.getObject("text")
.getObject("text")
.getArray("runs"); .getArray("runs");
final String searchType = getLinkHandler().getContentFilters().get(0); final String searchType = getLinkHandler().getContentFilters().get(0);
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) { if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) { collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
final String id = info.getObject("playlistItemData").getString("videoId"); final String id = info.getObject("playlistItemData")
.getString("videoId");
if (!isNullOrEmpty(id)) { if (!isNullOrEmpty(id)) {
return "https://music.youtube.com/watch?v=" + id; return "https://music.youtube.com/watch?v=" + id;
} }
@ -279,8 +298,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) final String name = getTextFromObject(info.getArray("flexColumns")
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); .getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(name)) { if (!isNullOrEmpty(name)) {
return name; return name;
} }
@ -310,24 +331,34 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
if (searchType.equals(MUSIC_VIDEOS)) { if (searchType.equals(MUSIC_VIDEOS)) {
JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items"); JsonArray items = info.getObject("menu").getObject("menuRenderer")
for (Object item : items) { .getArray("items");
final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer"); for (final Object item : items) {
if (menuNavigationItemRenderer.getObject("icon").getString("iconType", EMPTY_STRING).equals("ARTIST")) { final JsonObject menuNavigationItemRenderer =
return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint")); ((JsonObject) item).getObject(
"menuNavigationItemRenderer");
if (menuNavigationItemRenderer.getObject("icon")
.getString("iconType", EMPTY_STRING)
.equals("ARTIST")) {
return getUrlFromNavigationEndpoint(
menuNavigationItemRenderer
.getObject("navigationEndpoint"));
} }
} }
return null; return null;
} else { } else {
final JsonObject navigationEndpointHolder = info.getArray("flexColumns") final JsonObject navigationEndpointHolder = info
.getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer") .getArray("flexColumns")
.getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0); .getObject("text").getArray("runs").getObject(0);
if (!navigationEndpointHolder.has("navigationEndpoint")) if (!navigationEndpointHolder.has("navigationEndpoint"))
return null; return null;
final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint")); final String url = getUrlFromNavigationEndpoint(
navigationEndpointHolder.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) { if (!isNullOrEmpty(url)) {
return url; return url;
@ -369,13 +400,15 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") final JsonArray thumbnails = info.getObject("thumbnail")
.getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails"); .getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution // the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); final String url = thumbnails.getObject(thumbnails.size() - 1)
.getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
} }
@ -385,21 +418,25 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") final JsonArray thumbnails = info.getObject("thumbnail")
.getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails"); .getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution // the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); final String url = thumbnails.getObject(thumbnails.size() - 1)
.getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
} }
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) final String name = getTextFromObject(info.getArray("flexColumns")
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); .getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(name)) { if (!isNullOrEmpty(name)) {
return name; return name;
} }
@ -408,7 +445,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint")); final String url = getUrlFromNavigationEndpoint(info
.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) { if (!isNullOrEmpty(url)) {
return url; return url;
} }
@ -417,8 +455,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public long getSubscriberCount() throws ParsingException { public long getSubscriberCount() throws ParsingException {
final String subscriberCount = getTextFromObject(info.getArray("flexColumns").getObject(2) final String subscriberCount = getTextFromObject(info
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); .getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(subscriberCount)) { if (!isNullOrEmpty(subscriberCount)) {
try { try {
return Utils.mixedNumberWordToLong(subscriberCount); return Utils.mixedNumberWordToLong(subscriberCount);
@ -445,21 +485,25 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer") final JsonArray thumbnails = info.getObject("thumbnail")
.getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails"); .getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution // the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); final String url = thumbnails.getObject(thumbnails.size() - 1)
.getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
} }
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0) final String name = getTextFromObject(info.getArray("flexColumns")
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text")); .getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text"));
if (!isNullOrEmpty(name)) { if (!isNullOrEmpty(name)) {
return name; return name;
} }
@ -512,7 +556,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
if (searchType.equals(MUSIC_ALBUMS)) { if (searchType.equals(MUSIC_ALBUMS)) {
return ITEM_COUNT_UNKNOWN; return ITEM_COUNT_UNKNOWN;
} }
final String count = descriptionElements.getObject(2).getString("text"); final String count = descriptionElements.getObject(2)
.getString("text");
if (!isNullOrEmpty(count)) { if (!isNullOrEmpty(count)) {
if (count.contains("100+")) { if (count.contains("100+")) {
return ITEM_COUNT_MORE_THAN_100; return ITEM_COUNT_MORE_THAN_100;
@ -528,18 +573,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
} }
} }
private Page getNextPageFrom(final JsonArray continuations) throws ParsingException, @Nullable
IOException, ReCaptchaException { private Page getNextPageFrom(final JsonArray continuations)
throws IOException, ParsingException, ReCaptchaException {
if (isNullOrEmpty(continuations)) { if (isNullOrEmpty(continuations)) {
return null; return null;
} }
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); final JsonObject nextContinuationData = continuations.getObject(0)
.getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation"); final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams + "&alt=json" + "&continuation=" + continuation + "&alt=json" + "&key="
+ "&key=" + YoutubeParsingHelper.getYoutubeMusicKey()[0]); + YoutubeParsingHelper.getYoutubeMusicKey()[0]);
} }
} }

1
extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java
View file

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.utils; package org.schabi.newpipe.extractor.utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
public class StringUtils { public class StringUtils {