CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[YouTube] Set CONSENT cookie

This commit is contained in:
TobiGr 2021-04-07 12:25:59 +02:00
parent e61ceef005
commit 883f16e0ad
4 changed files with 96 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -28,12 +28,7 @@ import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -79,6 +74,17 @@ public class YoutubeParsingHelper {
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
private static String[] youtubeMusicKeys;
/**
* <code>PENDING+</code> means that the user did not yet submit their choices.
* Therefore, YouTube & Google should not track the user, because they did not give consent.
* The three digits at the end can be random, but are required.
*/
public static final String CONSENT_COOKIE_VALUE = "PENDING+" + (100 + new Random().nextInt(900));
/**
* Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
*/
public static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
@ -427,6 +433,7 @@ public class YoutubeParsingHelper {
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
addCookieHeader(headers);
final String response = getDownloader().post(url, headers, json).responseBody();
@ -629,8 +636,7 @@ public class YoutubeParsingHelper {
public static Response getResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
addYouTubeHeaders(headers);
final Response response = getDownloader().get(url, headers, localization);
getValidJsonResponseBody(response);
@ -638,6 +644,64 @@ public class YoutubeParsingHelper {
return response;
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
final Response response = getDownloader().get(url, headers, localization);
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final Page page, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
final Response response = getDownloader().get(page.getUrl(), headers, localization);
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
/**
* Add required headers and cookies to an existing headers Map.
* @see #addClientInfoHeaders(Map)
* @see #addCookieHeader(Map)
*/
public static void addYouTubeHeaders(final Map<String, List<String>> headers)
throws IOException, ExtractionException {
addClientInfoHeaders(headers);
addCookieHeader(headers);
}
/**
* Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
* @param headers The headers which should be completed
*/
public static void addClientInfoHeaders(final Map<String, List<String>> headers)
throws IOException, ExtractionException {
if (headers.get("X-YouTube-Client-Name") == null) {
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
}
if (headers.get("X-YouTube-Client-Version") == null) {
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
}
}
/**
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
* @see #CONSENT_COOKIE
* @param headers the headers which should be completed
*/
public static void addCookieHeader(final Map<String, List<String>> headers) {
if (headers.get("Cookie") == null) {
headers.put("Cookie", Arrays.asList(CONSENT_COOKIE));
} else {
headers.get("Cookie").add(CONSENT_COOKIE);
}
}
public static String extractCookieValue(final String cookieName, final Response response) {
final List<String> cookies = response.responseHeaders().get("set-cookie");
int startIndex;
@ -652,30 +716,6 @@ public class YoutubeParsingHelper {
return result;
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final Page page, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
if (!isNullOrEmpty(page.getCookies())) {
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
}
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(page.getUrl(), headers, localization);
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
/**
* Shared alert detection function, multiple endpoints return the error similarly structured.
* <p>

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -20,7 +20,9 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -130,8 +132,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, playlistData.getArray("contents"));
return new InfoItemsPage<>(collector,
new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue)));
final Map<String, String> cookies = new HashMap<>();
cookies.put(COOKIE_NAME, cookieValue);
return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
}
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -12,9 +12,10 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONSENT_COOKIE_VALUE;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/*
@ -45,17 +46,20 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
@Override
public List<String> suggestionList(String query) throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
List<String> suggestions = new ArrayList<>();
final Downloader dl = NewPipe.getDownloader();
final List<String> suggestions = new ArrayList<>();
String url = "https://suggestqueries.google.com/complete/search"
final String url = "https://suggestqueries.google.com/complete/search"
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+ "&jsonp=" + "JP"
+ "&ds=" + "yt"
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
+ "&q=" + URLEncoder.encode(query, UTF_8);
String response = dl.get(url, getExtractorLocalization()).responseBody();
final Map<String, List<String>> headers = new HashMap<>();
addCookieHeader(headers);
String response = dl.get(url, headers, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)"
response = response.substring(3, response.length() - 1);
try {

13
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View File

@ -21,10 +21,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlayli
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.*;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.startsWith;
@ -44,8 +41,7 @@ public class YoutubeMixPlaylistExtractorTest {
private static final String VIDEO_TITLE =
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
private static final Map<String, String> dummyCookie
= Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
private static final Map<String, String> dummyCookie = new HashMap<>();
private static YoutubeMixPlaylistExtractor extractor;
@ -55,6 +51,7 @@ public class YoutubeMixPlaylistExtractorTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
@ -133,6 +130,7 @@ public class YoutubeMixPlaylistExtractorTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
@ -203,6 +201,7 @@ public class YoutubeMixPlaylistExtractorTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
@ -277,6 +276,7 @@ public class YoutubeMixPlaylistExtractorTest {
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "invalid"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
}
@Test(expected = IllegalArgumentException.class)
@ -309,6 +309,7 @@ public class YoutubeMixPlaylistExtractorTest {
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL