CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Test if services recognizes their own items urls

Browse source
This commit is contained in:
Mauricio Colli 2020-02-27 00:08:23 -03:00 committed by TobiGr
parent 35252235b0
commit e590417cc4
9 changed files with 91 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View file

@ -2,20 +2,29 @@ package org.schabi.newpipe.extractor.services;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.util.Calendar;
import java.util.List;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.*;
import static org.schabi.newpipe.extractor.StreamingService.*;
public final class DefaultTests {
public static void defaultTestListOfItems(int expectedServiceId, List<? extends InfoItem> itemsList, List<Throwable> errors) {
assertTrue("List of items is empty", !itemsList.isEmpty());
public static void defaultTestListOfItems(StreamingService expectedService, List<? extends InfoItem> itemsList, List<Throwable> errors) throws ParsingException {
assertFalse("List of items is empty", itemsList.isEmpty());
assertFalse("List of items contains a null element", itemsList.contains(null));
assertEmptyErrors("Errors during stream list extraction", errors);
assertEmptyErrors("Errors during extraction", errors);
for (InfoItem item : itemsList) {
assertIsSecureUrl(item.getUrl());
@ -23,12 +32,17 @@ public final class DefaultTests {
assertIsSecureUrl(item.getThumbnailUrl());
}
assertNotNull("InfoItem type not set: " + item, item.getInfoType());
assertEquals("Service id doesn't match: " + item, expectedServiceId, item.getServiceId());
assertEquals("Unexpected item service id", expectedService.getServiceId(), item.getServiceId());
assertNotEmpty("Item name not set: " + item, item.getName());
if (item instanceof StreamInfoItem) {
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
assertIsSecureUrl(streamInfoItem.getUploaderUrl());
assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM);
assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL);
final String textualUploadDate = streamInfoItem.getTextualUploadDate();
if (textualUploadDate != null && !textualUploadDate.isEmpty()) {
@ -37,34 +51,56 @@ public final class DefaultTests {
assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance()));
}
} else if (item instanceof ChannelInfoItem) {
final ChannelInfoItem channelInfoItem = (ChannelInfoItem) item;
assertExpectedLinkType(expectedService, channelInfoItem.getUrl(), LinkType.CHANNEL);
} else if (item instanceof PlaylistInfoItem) {
final PlaylistInfoItem playlistInfoItem = (PlaylistInfoItem) item;
assertExpectedLinkType(expectedService, playlistInfoItem.getUrl(), LinkType.PLAYLIST);
}
}
}
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestRelatedItems(ListExtractor<T> extractor, int expectedServiceId) throws Exception {
private static void assertExpectedLinkType(StreamingService expectedService, String url, LinkType expectedLinkType) throws ParsingException {
final LinkType linkTypeByUrl = expectedService.getLinkTypeByUrl(url);
assertNotEquals("Url is not recognized by its own service: \"" + url + "\"",
LinkType.NONE, linkTypeByUrl);
assertEquals("Service returned wrong link type for: \"" + url + "\"",
expectedLinkType, linkTypeByUrl);
}
public static <T extends InfoItem> void assertNoMoreItems(ListExtractor<T> extractor) throws Exception {
assertFalse("More items available when it shouldn't", extractor.hasNextPage());
final String nextPageUrl = extractor.getNextPageUrl();
assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty());
}
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestRelatedItems(ListExtractor<T> extractor) throws Exception {
final ListExtractor.InfoItemsPage<T> page = extractor.getInitialPage();
final List<T> itemsList = page.getItems();
List<Throwable> errors = page.getErrors();
defaultTestListOfItems(expectedServiceId, itemsList, errors);
defaultTestListOfItems(extractor.getService(), itemsList, errors);
return page;
}
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor, int expectedServiceId) throws Exception {
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor) throws Exception {
assertTrue("Doesn't have more items", extractor.hasNextPage());
ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(extractor.getNextPageUrl());
final List<T> items = nextPage.getItems();
assertTrue("Next page is empty", !items.isEmpty());
assertFalse("Next page is empty", items.isEmpty());
assertEmptyErrors("Next page have errors", nextPage.getErrors());
defaultTestListOfItems(expectedServiceId, nextPage.getItems(), nextPage.getErrors());
defaultTestListOfItems(extractor.getService(), nextPage.getItems(), nextPage.getErrors());
return nextPage;
}
public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor, int expectedServiceId) throws Exception {
public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor) throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPageUrl);
defaultTestListOfItems(expectedServiceId, page.getItems(), page.getErrors());
defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors());
}
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View file

@ -68,12 +68,12 @@ public class PeertubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, PeerTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, PeerTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -127,7 +127,7 @@ public class PeertubeChannelExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -165,12 +165,12 @@ public class PeertubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, PeerTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, PeerTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View file

@ -64,12 +64,12 @@ public class SoundcloudChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, SoundCloud.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -120,7 +120,7 @@ public class SoundcloudChannelExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -158,12 +158,12 @@ public class SoundcloudChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, SoundCloud.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View file

@ -7,7 +7,6 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -67,13 +66,13 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() {
try {
defaultTestMoreItems(extractor, SoundCloud.getServiceId());
defaultTestMoreItems(extractor);
} catch (Throwable ignored) {
return;
}
@ -165,12 +164,12 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, SoundCloud.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -226,11 +225,10 @@ public class SoundcloudPlaylistExtractorTest {
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Ignore
@Test
public void testGetPageInNewExtractor() throws Exception {
final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -269,18 +267,18 @@ public class SoundcloudPlaylistExtractorTest {
@Ignore
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
defaultTestRelatedItems(extractor);
}
//TODO: FUCK THIS: This triggers a 500 at sever
@Ignore
@Test
public void testMoreRelatedItems() throws Exception {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId());
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor);
// Test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
defaultTestListOfItems(SoundCloud.getServiceId(), currentPage.getItems(), currentPage.getErrors());
defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors());
}
}

27
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View file

@ -4,7 +4,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -88,12 +87,12 @@ public class YoutubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -178,12 +177,12 @@ public class YoutubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -241,7 +240,7 @@ public class YoutubeChannelExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -280,12 +279,12 @@ public class YoutubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -371,12 +370,12 @@ public class YoutubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -461,12 +460,12 @@ public class YoutubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, YouTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -553,13 +552,13 @@ public class YoutubeChannelExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() {
try {
defaultTestMoreItems(extractor, YouTube.getServiceId());
defaultTestMoreItems(extractor);
} catch (Throwable ignored) {
return;
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java
View file

@ -49,7 +49,7 @@ public class YoutubeChannelLocalizationTest {
final ChannelExtractor extractor = YouTube.getChannelExtractor(channelUrl);
extractor.forceLocalization(currentLocalization);
extractor.fetchPage();
itemsPage = defaultTestRelatedItems(extractor, YouTube.getServiceId());
itemsPage = defaultTestRelatedItems(extractor);
} catch (Throwable e) {
System.out.println("[!] " + currentLocalization + " → failed");
throw e;

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View file

@ -84,7 +84,7 @@ public class YoutubeCommentsExtractorTest {
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractorYT.getInitialPage();
DefaultTests.defaultTestListOfItems(YouTube.getServiceId(), comments.getItems(), comments.getErrors());
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
assertFalse(StringUtil.isBlank(c.getAuthorName()));

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java
View file

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtra
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
public class YoutubeFeedExtractorTest {
@ -60,13 +61,12 @@ public class YoutubeFeedExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() {
assertFalse(extractor.hasNextPage());
assertNull(extractor.getNextPageUrl());
public void testMoreRelatedItems() throws Exception {
assertNoMoreItems(extractor);
}
}
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View file

@ -6,7 +6,7 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -93,12 +93,12 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -161,7 +161,7 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
@ -200,18 +200,17 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, YouTube.getServiceId());
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage
= defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId());
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor);
// test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors());
defaultTestListOfItems(YouTube, currentPage.getItems(), currentPage.getErrors());
}
}