CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #465 from XiangRongLin/playlist_continuation

Browse source 
[YouTube] Fix playlist continuations extraction
This commit is contained in:
Tobias Groza 2020-11-20 20:48:20 +01:00 committed by GitHub
commit 650f0920fe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 70 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View file

@ -190,9 +190,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, null);
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer");
collectStreamsFrom(collector, videos.getArray("contents"));
final JsonArray videosArray = videos.getArray("contents");
collectStreamsFrom(collector, videosArray);
nextPage = getNextPageFrom(videos.getArray("continuations"));
nextPage = getNextPageFrom(videosArray);
}
return new InfoItemsPage<>(collector, nextPage);
@ -207,24 +208,34 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation");
final JsonArray continuation = ajaxJson.getObject(1)
.getObject("response")
.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems");
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
collectStreamsFrom(collector, continuation);
return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations")));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray continuations) {
if (isNullOrEmpty(continuations)) {
private Page getNextPageFrom(final JsonArray contents) {
if (isNullOrEmpty(contents)) {
return null;
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams);
final JsonObject lastElement = contents.getObject(contents.size() - 1);
if (lastElement.has("continuationItemRenderer")) {
final String continuation = lastElement
.getObject("continuationItemRenderer")
.getObject("continuationEndpoint")
.getObject("continuationCommand")
.getString("token");
return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
} else {
return null;
}
}
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {

49
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View file

@ -3,6 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -11,10 +14,17 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -23,6 +33,9 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link YoutubePlaylistExtractor}
*/
@RunWith(Suite.class)
@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class,
LearningPlaylist.class, ContinuationsTests.class})
public class YoutubePlaylistExtractorTest {
public static class NotAvailable {
@ -114,7 +127,7 @@ public class YoutubePlaylistExtractorTest {
@Ignore
@Test
public void testBannerUrl() throws Exception {
public void testBannerUrl() {
final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt"));
@ -227,7 +240,7 @@ public class YoutubePlaylistExtractorTest {
@Ignore
@Test
public void testBannerUrl() throws Exception {
public void testBannerUrl() {
final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt"));
@ -324,7 +337,7 @@ public class YoutubePlaylistExtractorTest {
@Ignore
@Test
public void testBannerUrl() throws Exception {
public void testBannerUrl() {
final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt"));
@ -352,4 +365,34 @@ public class YoutubePlaylistExtractorTest {
assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
}
}
public static class ContinuationsTests {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void testNoContinuations() throws Exception {
final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO");
extractor.fetchPage();
assertNoMoreItems(extractor);
}
@Test
public void testOnlySingleContinuation() throws Exception {
final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX");
extractor.fetchPage();
final ListExtractor.InfoItemsPage<StreamInfoItem> page = defaultTestMoreItems(
extractor);
assertFalse("More items available when it shouldn't", page.hasNextPage());
}
}
}