[Youtube] Add tests and take thumbnail image always from first video of mix

Also fix getThumbnailUrl for "My Mix"
This commit is contained in:
Xiang Rong Lin 2020-03-07 16:03:12 +01:00 committed by XiangRongLin
parent 68a3948af6
commit df38b1926c
4 changed files with 518 additions and 133 deletions

View file

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import com.grack.nanojson.JsonArray;
@ -8,28 +7,28 @@ import com.grack.nanojson.JsonObject;
import java.io.IOException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
/**
* A YoutubePlaylistExtractor for a mix (auto-generated playlist).
* It handles urls in the format of "youtube.com/watch?v=videoId&list=playlistId"
* A YoutubePlaylistExtractor for a mix (auto-generated playlist). It handles urls in the format of
* "youtube.com/watch?v=videoId&list=playlistId"
*/
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
private JsonObject initialData;
private final static String CONTENTS = "contents";
private final static String RESPONSE = "response";
private final static String PLAYLIST = "playlist";
private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults";
private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer";
private JsonObject playlistData;
public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
@ -37,12 +36,18 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull Downloader downloader)
throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(3).getObject("response");
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist");
System.out.println();
JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE);
try {
playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS)
.getObject(PLAYLIST).getObject(PLAYLIST);
} catch (NullPointerException e) {
throw new ExtractionException(e);
}
}
@Nonnull
@ -50,8 +55,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
public String getName() throws ParsingException {
try {
final String name = playlistData.getString("title");
if (name!= null) return name;
else return "";
if (name != null) {
return name;
} else {
return "";
}
} catch (Exception e) {
throw new ParsingException("Could not get playlist name", e);
}
@ -60,9 +68,16 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final String videoId = playlistData.getArray("contents").getObject(0)
.getObject("playlistPanelVideoRenderer").getString("videoId");
if (videoId == null || videoId.isEmpty()) throw new ParsingException("");
final String playlistId = playlistData.getString("playlistId");
final String videoId;
if (playlistId.startsWith("RDMM")) {
videoId = playlistId.substring(4);
} else {
videoId = playlistId.substring(2);
}
if (videoId.isEmpty()) {
throw new ParsingException("");
}
return getThumbnailUrlFromId(videoId);
} catch (Exception e) {
throw new ParsingException("Could not get playlist thumbnail", e);
@ -100,36 +115,39 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, playlistData.getArray("contents"));
collectStreamsFrom(collector, playlistData.getArray(CONTENTS));
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@Override
public String getNextPageUrl() {
final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents")
.get(playlistData.getArray("contents").size() - 1));
final String lastStreamId = lastStream.getObject("playlistPanelVideoRenderer")
.getString("videoId");
return "https://youtube.com" + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")
.getObject("commandMetadata").getObject("webCommandMetadata").getString("url") + "&pbj=1";
public String getNextPageUrl() throws ExtractionException {
final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS)
.get(playlistData.getArray(CONTENTS).size() - 1));
if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) {
throw new ExtractionException("Could not extract next page url");
}
return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER)
.getObject("navigationEndpoint").getObject("commandMetadata")
.getObject("webCommandMetadata").getString("url") + "&pbj=1";
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl)
throws ExtractionException, IOException {
if (pageUrl == null || pageUrl.isEmpty()) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
throw new ExtractionException(
new IllegalArgumentException("Page url is empty or null"));
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
playlistData =
ajaxJson.getObject(3).getObject("response").getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("playlist")
.getObject("playlist");
final JsonArray streams = playlistData.getArray("contents");
ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS)
.getObject(TWO_COLUMNS_WATCH_NEXT_RESULTS).getObject(PLAYLIST)
.getObject(PLAYLIST);
final JsonArray streams = playlistData.getArray(CONTENTS);
//Because continuation requests are created with the last video of previous request as start
streams.remove(0);
collectStreamsFrom(collector, streams);
@ -149,7 +167,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
for (Object stream : streams) {
if (stream instanceof JsonObject) {
JsonObject streamInfo = ((JsonObject) stream).getObject("playlistPanelVideoRenderer");
JsonObject streamInfo = ((JsonObject) stream)
.getObject(PLAYLIST_PANEL_VIDEO_RENDERER);
if (streamInfo != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
}

View file

@ -0,0 +1,323 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
public class YoutubeMixPlaylistExtractorTest {
private static YoutubeMixPlaylistExtractor extractor;
private static String videoId = "_AzeUSL9lZc";
private static String videoTitle = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
public static class Mix {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
extractor.fetchPage();
}
@Test
public void getServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void getName() throws Exception {
String name = extractor.getName();
assertThat(name, startsWith("Mix"));
assertThat(name, containsString(videoTitle));
}
@Test
public void getThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertThat(thumbnailUrl, containsString("yt"));
assertThat(thumbnailUrl, containsString(videoId));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RD" + videoId));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPageMultipleTimes() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
//Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
streams = extractor.getPage(streams.getNextPageUrl());
}
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
}
@Test
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
public static class MixWithIndex {
private static String index = "&index=13";
private static String videoIdNumber13 = "qHtzO49SDmk";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoIdNumber13 + "&list=RD" + videoId
+ index);
extractor.fetchPage();
}
@Test
public void getName() throws Exception {
String name = extractor.getName();
assertThat(name, startsWith("Mix"));
assertThat(name, containsString(videoTitle));
}
@Test
public void getThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertThat(thumbnailUrl, containsString("yt"));
assertThat(thumbnailUrl, containsString(videoId));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RD" + videoId));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPageMultipleTimes() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
//Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
streams = extractor.getPage(streams.getNextPageUrl());
}
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
}
@Test
public void getStreamCount() {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
public static class MyMix {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId);
extractor.fetchPage();
}
@Test
public void getServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void getName() throws Exception {
String name = extractor.getName();
assertEquals("My Mix", name);
}
@Test
public void getThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RDMM" + videoId));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPageMultipleTimes() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
//Should work infinitely, but for testing purposes only 3 times
for (int i = 0; i < 3; i++) {
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
streams = extractor.getPage(streams.getNextPageUrl());
}
assertTrue(streams.hasNextPage());
assertFalse(streams.getItems().isEmpty());
}
@Test
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
public static class Invalid {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = ExtractionException.class)
public void getPageEmptyUrl() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
extractor.fetchPage();
extractor.getPage("");
}
@Test(expected = NullPointerException.class)
public void invalidVideoId() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
extractor.fetchPage();
}
}
public static class ChannelMix {
private static String channelId = "UCXuqSBlHAE6Xw-yeJA0Tunw";
private static String videoIdOfChannel = "mnk6gnOBYIo";
private static String channelTitle = "Linus Tech Tips";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoIdOfChannel + "&list=RDCM" + channelId);
extractor.fetchPage();
}
@Test
public void getName() throws Exception {
String name = extractor.getName();
assertThat(name, startsWith("Mix"));
assertThat(name, containsString(channelTitle));
}
@Test
public void getThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertThat(thumbnailUrl, containsString("yt"));
}
@Test
public void getNextPageUrl() throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
assertIsSecureUrl(nextPageUrl);
assertThat(nextPageUrl, containsString("list=RDCM" + channelId));
}
@Test
public void getInitialPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getPage() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getPage(extractor.getNextPageUrl());
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getStreamCount() throws Exception {
assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
}
}
}

View file

@ -105,4 +105,23 @@ public class YoutubePlaylistLinkHandlerFactoryTest {
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
}
@Test
public void fromUrlIsMixVideo() throws Exception {
final String videoId = "_AzeUSL9lZc";
String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId;
assertEquals(url, linkHandler.fromUrl(url).getUrl());
final String mixVideoId = "qHtzO49SDmk";
url = "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId;
assertEquals(url, linkHandler.fromUrl(url).getUrl());
}
@Test
public void fromUrlIsMixPlaylist() throws Exception {
final String videoId = "_AzeUSL9lZc";
final String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId;
assertEquals(url,
linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl());
}
}

View file

@ -26,9 +26,13 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
@ -54,4 +58,24 @@ public class YoutubeServiceTest {
public void testGetDefaultKiosk() throws Exception {
assertEquals(kioskList.getDefaultKioskExtractor(null).getId(), "Trending");
}
@Test
public void getPlayListExtractorIsNormalPlaylist() throws Exception {
PlaylistExtractor extractor = service.getPlaylistExtractor(
"https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS");
assertTrue(extractor instanceof YoutubePlaylistExtractor);
}
@Test
public void getPlaylistExtractorIsMix() throws Exception {
String videoId = "_AzeUSL9lZc";
PlaylistExtractor extractor = YouTube.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
assertTrue(extractor instanceof YoutubeMixPlaylistExtractor);
extractor = YouTube.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId);
assertTrue(extractor instanceof YoutubeMixPlaylistExtractor);
}
}