From ecfc3706858b0b8da4a0284a02b85556f00e46c8 Mon Sep 17 00:00:00 2001
From: litetex <40789489+litetex@users.noreply.github.com>
Date: Sat, 30 Jul 2022 16:05:52 +0200
Subject: [PATCH] Fixed all YTMixPlaylists
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Added option to choose if you want to consent or not - currently this is done by a static variable in ``YoutubeParsingHelper`` - may not be the best long-term solution but for now the tests work again (in EU countries) 🥳
---
.../exceptions/ConsentRequiredException.java | 12 +++++
.../youtube/YoutubeParsingHelper.java | 45 +++++++++-------
.../YoutubeMixPlaylistExtractor.java | 52 +++++++++++--------
.../YoutubeMixPlaylistExtractorTest.java | 30 +++++++----
.../services/youtube/YoutubeTestsUtils.java | 1 +
5 files changed, 89 insertions(+), 51 deletions(-)
create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java
new file mode 100644
index 00000000..90feacc0
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java
@@ -0,0 +1,12 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+public class ConsentRequiredException extends ParsingException {
+
+ public ConsentRequiredException(final String message) {
+ super(message);
+ }
+
+ public ConsentRequiredException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
index 2458a97c..05231628 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@@ -27,7 +27,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
import static java.util.Collections.singletonList;
import com.grack.nanojson.JsonArray;
@@ -245,16 +244,19 @@ public final class YoutubeParsingHelper {
* The three digits at the end can be random, but are required.
*
*/
- private static final String CONSENT_COOKIE_VALUE = "PENDING+";
-
+ private static final String CONSENT_COOKIE_PENDING_VALUE = "PENDING+";
/**
- * YouTube {@code CONSENT} cookie.
+ * {@code YES+} means that the user did submit their choices and accepted all cookies.
*
*
- * Should prevent redirect to {@code consent.youtube.com}.
+ * Therefore, YouTube & Google can track the user, because they did give consent.
+ *
+ *
+ *
+ * The three digits at the end can be random, but are required.
*
*/
- private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
+ private static final String CONSENT_COOKIE_YES_VALUE = "YES+";
private static final String FEED_BASE_CHANNEL_ID =
"https://www.youtube.com/feeds/videos.xml?channel_id=";
@@ -265,6 +267,13 @@ public final class YoutubeParsingHelper {
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
+ /**
+ * {@code false} (default) will use {@link #CONSENT_COOKIE_PENDING_VALUE}.
+ *
+ * {@code true} will use {@link #CONSENT_COOKIE_YES_VALUE}.
+ */
+ private static boolean consentAccepted = false;
+
private static boolean isGoogleURL(final String url) {
final String cachedUrl = extractCachedUrlIfNeeded(url);
try {
@@ -1378,7 +1387,6 @@ public final class YoutubeParsingHelper {
/**
* Add the CONSENT
cookie to prevent redirect to consent.youtube.com
- * @see #CONSENT_COOKIE
* @param headers the headers which should be completed
*/
public static void addCookieHeader(@Nonnull final Map> headers) {
@@ -1391,8 +1399,9 @@ public final class YoutubeParsingHelper {
@Nonnull
public static String generateConsentCookie() {
- final int statusCode = 100 + numberGenerator.nextInt(900);
- return CONSENT_COOKIE + statusCode;
+ return "CONSENT="
+ + (isConsentAccepted() ? CONSENT_COOKIE_YES_VALUE : CONSENT_COOKIE_PENDING_VALUE)
+ + (100 + numberGenerator.nextInt(900));
}
public static String extractCookieValue(final String cookieName,
@@ -1612,16 +1621,6 @@ public final class YoutubeParsingHelper {
return false;
}
- @Nonnull
- public static String unescapeDocument(@Nonnull final String doc) {
- return doc
- .replaceAll("\\\\x22", "\"")
- .replaceAll("\\\\x7b", "{")
- .replaceAll("\\\\x7d", "}")
- .replaceAll("\\\\x5b", "[")
- .replaceAll("\\\\x5d", "]");
- }
-
/**
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
* playback requests (and also for some clients, in the player request body).
@@ -1692,4 +1691,12 @@ public final class YoutubeParsingHelper {
public static boolean isIosStreamingUrl(@Nonnull final String url) {
return Parser.isMatch(C_IOS_PATTERN, url);
}
+
+ public static void setConsentAccepted(final boolean accepted) {
+ consentAccepted = accepted;
+ }
+
+ public static boolean isConsentAccepted() {
+ return consentAccepted;
+ }
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
index 49225330..dc7179e4 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
@@ -1,28 +1,15 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
-import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
-
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
-
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ConsentRequiredException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -35,6 +22,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
@@ -43,8 +32,18 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
/**
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@@ -89,16 +88,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
final Map> headers = new HashMap<>();
- addClientInfoHeaders(headers);
+ // Cookie is required due to consent
+ addYouTubeHeaders(headers);
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
- playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
- .getObject("playlist").getObject("playlist");
+ playlistData = initialData
+ .getObject("contents")
+ .getObject("twoColumnWatchNextResults")
+ .getObject("playlist")
+ .getObject("playlist");
if (isNullOrEmpty(playlistData)) {
- throw new ExtractionException("Could not get playlistData");
+ final ExtractionException ex = new ExtractionException("Could not get playlistData");
+ if (!YoutubeParsingHelper.isConsentAccepted()) {
+ throw new ConsentRequiredException(
+ "Consent is required in some countries to view Mix playlists",
+ ex);
+ }
+ throw ex;
}
cookieValue = extractCookieValue(COOKIE_NAME, response);
}
@@ -212,7 +221,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map> headers = new HashMap<>();
- addClientInfoHeaders(headers);
+ // Cookie is required due to consent
+ addYouTubeHeaders(headers);
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
getExtractorLocalization());
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
index b562d659..89906770 100644
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
@@ -1,15 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-
import com.grack.nanojson.JsonWriter;
-
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -31,6 +22,17 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+
public class YoutubeMixPlaylistExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
@@ -45,6 +47,7 @@ public class YoutubeMixPlaylistExtractorTest {
@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
+ YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -140,6 +143,7 @@ public class YoutubeMixPlaylistExtractorTest {
@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
+ YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -221,11 +225,12 @@ public class YoutubeMixPlaylistExtractorTest {
}
public static class MyMix {
- private static final String VIDEO_ID = "_AzeUSL9lZc";
+ private static final String VIDEO_ID = "YVkUvmDQ3HY";
@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
+ YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -249,7 +254,7 @@ public class YoutubeMixPlaylistExtractorTest {
void getThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
- assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
+ assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID));
}
@Test
@@ -316,6 +321,7 @@ public class YoutubeMixPlaylistExtractorTest {
@BeforeAll
public static void setUp() throws IOException {
YoutubeTestsUtils.ensureStateless();
+ YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
}
@@ -350,6 +356,7 @@ public class YoutubeMixPlaylistExtractorTest {
@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
+ YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -414,6 +421,7 @@ public class YoutubeMixPlaylistExtractorTest {
@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
+ YoutubeParsingHelper.setConsentAccepted(true);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
extractor = (YoutubeMixPlaylistExtractor) YouTube
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java
index cc2b3111..a9803987 100644
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java
@@ -21,6 +21,7 @@ public final class YoutubeTestsUtils {
*
*/
public static void ensureStateless() {
+ YoutubeParsingHelper.setConsentAccepted(false);
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeStreamExtractor.resetDeobfuscationCode();