From 883f16e0ad943570bbc9613949ef144e5962386d Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 7 Apr 2021 12:25:59 +0200 Subject: [PATCH] [YouTube] Set CONSENT cookie --- .../youtube/YoutubeParsingHelper.java | 104 ++++++++++++------ .../YoutubeMixPlaylistExtractor.java | 9 +- .../YoutubeSuggestionExtractor.java | 16 ++- .../YoutubeMixPlaylistExtractorTest.java | 13 ++- 4 files changed, 96 insertions(+), 46 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 377cec9a..befed0aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -28,12 +28,7 @@ import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -79,6 +74,17 @@ public class YoutubeParsingHelper { private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"}; private static String[] youtubeMusicKeys; + /** + * PENDING+ means that the user did not yet submit their choices. + * Therefore, YouTube & Google should not track the user, because they did not give consent. + * The three digits at the end can be random, but are required. + */ + public static final String CONSENT_COOKIE_VALUE = "PENDING+" + (100 + new Random().nextInt(900)); + /** + * Youtube CONSENT cookie. Should prevent redirect to consent.youtube.com + */ + public static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -427,6 +433,7 @@ public class YoutubeParsingHelper { headers.put("Origin", Collections.singletonList("https://music.youtube.com")); headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Content-Type", Collections.singletonList("application/json")); + addCookieHeader(headers); final String response = getDownloader().post(url, headers, json).responseBody(); @@ -629,8 +636,7 @@ public class YoutubeParsingHelper { public static Response getResponse(final String url, final Localization localization) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + addYouTubeHeaders(headers); final Response response = getDownloader().get(url, headers, localization); getValidJsonResponseBody(response); @@ -638,6 +644,64 @@ public class YoutubeParsingHelper { return response; } + public static JsonArray getJsonResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + addYouTubeHeaders(headers); + + final Response response = getDownloader().get(url, headers, localization); + + return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); + } + + public static JsonArray getJsonResponse(final Page page, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + addYouTubeHeaders(headers); + + final Response response = getDownloader().get(page.getUrl(), headers, localization); + + return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); + } + + /** + * Add required headers and cookies to an existing headers Map. + * @see #addClientInfoHeaders(Map) + * @see #addCookieHeader(Map) + */ + public static void addYouTubeHeaders(final Map> headers) + throws IOException, ExtractionException { + addClientInfoHeaders(headers); + addCookieHeader(headers); + } + + /** + * Add the X-YouTube-Client-Name and X-YouTube-Client-Version headers. + * @param headers The headers which should be completed + */ + public static void addClientInfoHeaders(final Map> headers) + throws IOException, ExtractionException { + if (headers.get("X-YouTube-Client-Name") == null) { + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + } + if (headers.get("X-YouTube-Client-Version") == null) { + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + } + } + + /** + * Add the CONSENT cookie to prevent redirect to consent.youtube.com + * @see #CONSENT_COOKIE + * @param headers the headers which should be completed + */ + public static void addCookieHeader(final Map> headers) { + if (headers.get("Cookie") == null) { + headers.put("Cookie", Arrays.asList(CONSENT_COOKIE)); + } else { + headers.get("Cookie").add(CONSENT_COOKIE); + } + } + public static String extractCookieValue(final String cookieName, final Response response) { final List cookies = response.responseHeaders().get("set-cookie"); int startIndex; @@ -652,30 +716,6 @@ public class YoutubeParsingHelper { return result; } - public static JsonArray getJsonResponse(final String url, final Localization localization) - throws IOException, ExtractionException { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); - final Response response = getDownloader().get(url, headers, localization); - - return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); - } - - public static JsonArray getJsonResponse(final Page page, final Localization localization) - throws IOException, ExtractionException { - final Map> headers = new HashMap<>(); - if (!isNullOrEmpty(page.getCookies())) { - headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies()))); - } - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); - - final Response response = getDownloader().get(page.getUrl(), headers, localization); - - return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); - } - /** * Shared alert detection function, multiple endpoints return the error similarly structured. *

diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 140392cc..9fdc0ab4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -20,7 +20,9 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -130,8 +132,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { public InfoItemsPage getInitialPage() throws ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); - return new InfoItemsPage<>(collector, - new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue))); + + final Map cookies = new HashMap<>(); + cookies.put(COOKIE_NAME, cookieValue); + + return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies)); } private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index a98d9c92..d78cd373 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -12,9 +12,10 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.io.IOException; import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.List; +import java.util.*; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONSENT_COOKIE_VALUE; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; /* @@ -45,17 +46,20 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { @Override public List suggestionList(String query) throws IOException, ExtractionException { - Downloader dl = NewPipe.getDownloader(); - List suggestions = new ArrayList<>(); + final Downloader dl = NewPipe.getDownloader(); + final List suggestions = new ArrayList<>(); - String url = "https://suggestqueries.google.com/complete/search" + final String url = "https://suggestqueries.google.com/complete/search" + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml + "&jsonp=" + "JP" + "&ds=" + "yt" + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8) + "&q=" + URLEncoder.encode(query, UTF_8); - String response = dl.get(url, getExtractorLocalization()).responseBody(); + final Map> headers = new HashMap<>(); + addCookieHeader(headers); + + String response = dl.get(url, headers, getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" response = response.substring(3, response.length() - 1); try { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index fa9c7ebf..d2051b9f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -21,10 +21,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlayli import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.startsWith; @@ -44,8 +41,7 @@ public class YoutubeMixPlaylistExtractorTest { private static final String VIDEO_TITLE = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/"; - private static final Map dummyCookie - = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); + private static final Map dummyCookie = new HashMap<>(); private static YoutubeMixPlaylistExtractor extractor; @@ -55,6 +51,7 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws Exception { YoutubeParsingHelper.resetClientVersionAndKey(); NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix")); + dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); @@ -133,6 +130,7 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws Exception { YoutubeParsingHelper.resetClientVersionAndKey(); NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex")); + dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" @@ -203,6 +201,7 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws Exception { YoutubeParsingHelper.resetClientVersionAndKey(); NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix")); + dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM" @@ -277,6 +276,7 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws IOException { YoutubeParsingHelper.resetClientVersionAndKey(); NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "invalid")); + dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); } @Test(expected = IllegalArgumentException.class) @@ -309,6 +309,7 @@ public class YoutubeMixPlaylistExtractorTest { public static void setUp() throws Exception { YoutubeParsingHelper.resetClientVersionAndKey(); NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix")); + dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL