Merge pull request #567 from XiangRongLin/playlist_continuations

Playlist continuations

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1,6 +1,11 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import com.grack.nanojson.*;
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import com.grack.nanojson.JsonWriter;
+
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -10,6 +15,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -33,7 +39,12 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
-import static org.schabi.newpipe.extractor.utils.Utils.*;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
+import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import static org.schabi.newpipe.extractor.utils.Utils.join;
 
 /*
  * Created by Christian Schabesberger on 02.03.16.
@@ -638,7 +649,7 @@ public class YoutubeParsingHelper {
         headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
         final Response response = getDownloader().get(url, headers, localization);
 
-        return toJsonArray(getValidJsonResponseBody(response));
+        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
     }
 
     public static JsonArray getJsonResponse(final Page page, final Localization localization)
@@ -652,15 +663,7 @@ public class YoutubeParsingHelper {
 
         final Response response = getDownloader().get(page.getUrl(), headers, localization);
 
-        return toJsonArray(getValidJsonResponseBody(response));
+        return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
-    }
-
-    public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
-        try {
-            return JsonParser.array().from(responseBody);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse JSON", e);
-        }
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
+
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -14,14 +15,19 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /**
@@ -51,7 +57,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
             throws IOException, ExtractionException {
         final String url = getUrl() + "&pbj=1";
         final Response response = getResponse(url, getExtractorLocalization());
-        final JsonArray ajaxJson = toJsonArray(response.responseBody());
+        final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
         initialData = ajaxJson.getObject(3).getObject("response");
         playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
                 .getObject("playlist").getObject("playlist");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -2,9 +2,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonWriter;
+
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -19,11 +22,20 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.IOException;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.utils.JsonUtils.toJsonObject;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 @SuppressWarnings("WeakerAccess")
@@ -168,7 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
     @Nonnull
     @Override
-    public InfoItemsPage<StreamInfoItem> getInitialPage() {
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         Page nextPage = null;
 
@@ -205,12 +217,27 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
             throw new IllegalArgumentException("Page doesn't contain an URL");
         }
 
-        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        // @formatter:off
-        final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
+        byte[] json = JsonWriter.string()
+                .object()
+                    .object("context")
+                        .object("client")
+                            .value("clientName", "1")
+                            .value("clientVersion", getClientVersion())
+                        .end()
+                    .end()
+                    .value("continuation", page.getId())
+                .end()
+                .done()
+                .getBytes(UTF_8);
+        // @formatter:on
 
-        final JsonArray continuation = ajaxJson.getObject(1)
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-                .getObject("response")
+        final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
-                .getArray("onResponseReceivedActions")
+
+        final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
+
+        final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
                 .getObject(0)
                 .getObject("appendContinuationItemsAction")
                 .getArray("continuationItems");
@@ -220,7 +247,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
     }
 
-    private Page getNextPageFrom(final JsonArray contents) {
+    private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException {
         if (isNullOrEmpty(contents)) {
             return null;
         }
@@ -232,7 +259,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                     .getObject("continuationEndpoint")
                     .getObject("continuationCommand")
                     .getString("token");
-            return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
+            return new Page(
+                    "https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
+                    continuation);
         } else {
             return null;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java

@@ -2,14 +2,18 @@ package org.schabi.newpipe.extractor.utils;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 public class JsonUtils {
     public static final JsonObject EMPTY_OBJECT = new JsonObject();
     public static final JsonArray EMPTY_ARRAY = new JsonArray();
@@ -99,4 +103,19 @@ public class JsonUtils {
         return result;
     }
 
+    public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
+        try {
+            return JsonParser.array().from(responseBody);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse JSON", e);
+        }
+    }
+
+    public static JsonObject toJsonObject(final String responseBody) throws ParsingException {
+        try {
+            return JsonParser.object().from(responseBody);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse JSON", e);
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -13,7 +13,11 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.*;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
@@ -410,7 +414,7 @@ public class YoutubePlaylistExtractorTest {
         public void testOnlySingleContinuation() throws Exception {
             final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
                     .getPlaylistExtractor(
-                            "https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX");
+                            "https://www.youtube.com/playlist?list=PLoumn5BIsUDeGF1vy5Nylf_RJKn5aL_nr");
             extractor.fetchPage();
 
             final ListExtractor.InfoItemsPage<StreamInfoItem> page = defaultTestMoreItems(