From 3a865749e89b213538f5073b2161d154173f67e1 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 5 Oct 2018 16:01:10 +0200 Subject: [PATCH 01/16] fix issues Kiosk issue about localization, nad rename to preferred localization --- .../org/schabi/newpipe/extractor/NewPipe.java | 2 +- .../newpipe/extractor/StreamingService.java | 24 ++++++++---------- .../extractor/channel/ChannelInfo.java | 3 +-- .../newpipe/extractor/kiosk/KioskInfo.java | 13 +++------- .../newpipe/extractor/kiosk/KioskList.java | 25 ++++++++++++++----- .../soundcloud/SoundcloudService.java | 4 +-- .../services/youtube/YoutubeService.java | 4 +-- .../YoutubeStreamExtractorDefaultTest.java | 2 +- .../youtube/YoutubeTrendingKioskInfoTest.java | 2 +- 9 files changed, 41 insertions(+), 38 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index fcbde0f9..cf4a2643 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -103,7 +103,7 @@ public class NewPipe { NewPipe.localization = localization; } - public static Localization getLocalization() { + public static Localization getPreferredLocalization() { return localization; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 55efd1f5..a85d8f3a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -80,7 +80,7 @@ public abstract class StreamingService { public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization); public abstract SuggestionExtractor getSuggestionExtractor(Localization localization); public abstract SubscriptionExtractor getSubscriptionExtractor(); - public abstract KioskList getKioskList(Localization localization) throws ExtractionException; + public abstract KioskList getKioskList() throws ExtractionException; public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) throws ExtractionException; @@ -93,27 +93,23 @@ public abstract class StreamingService { //////////////////////////////////////////// public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { - return getSearchExtractor(queryHandler, NewPipe.getLocalization()); + return getSearchExtractor(queryHandler, NewPipe.getPreferredLocalization()); } public SuggestionExtractor getSuggestionExtractor() { - return getSuggestionExtractor(NewPipe.getLocalization()); - } - - public KioskList getKioskList() throws ExtractionException { - return getKioskList(NewPipe.getLocalization()); + return getSuggestionExtractor(NewPipe.getPreferredLocalization()); } public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return getChannelExtractor(linkHandler, NewPipe.getLocalization()); + return getChannelExtractor(linkHandler, NewPipe.getPreferredLocalization()); } public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException { - return getPlaylistExtractor(linkHandler, NewPipe.getLocalization()); + return getPlaylistExtractor(linkHandler, NewPipe.getPreferredLocalization()); } public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException { - return getStreamExtractor(linkHandler, NewPipe.getLocalization()); + return getStreamExtractor(linkHandler, NewPipe.getPreferredLocalization()); } //////////////////////////////////////////// @@ -154,19 +150,19 @@ public abstract class StreamingService { //////////////////////////////////////////// public SearchExtractor getSearchExtractor(String query) throws ExtractionException { - return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getLocalization()); + return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getPreferredLocalization()); } public ChannelExtractor getChannelExtractor(String url) throws ExtractionException { - return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getLocalization()); + return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getPreferredLocalization()); } public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException { - return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getLocalization()); + return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getPreferredLocalization()); } public StreamExtractor getStreamExtractor(String url) throws ExtractionException { - return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getLocalization()); + return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getPreferredLocalization()); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 0aed6e11..20abdfe4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -51,8 +51,7 @@ public class ChannelInfo extends ListInfo { public static InfoItemsPage getMoreItems(StreamingService service, String url, - String pageUrl, - Localization localization) throws IOException, ExtractionException { + String pageUrl) throws IOException, ExtractionException { return service.getChannelExtractor(url).getPage(pageUrl); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 141b480c..9ebb0cd9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -37,25 +37,20 @@ public class KioskInfo extends ListInfo { public static ListExtractor.InfoItemsPage getMoreItems(StreamingService service, String url, - String pageUrl, - String contentCountry) throws IOException, ExtractionException { + String pageUrl) throws IOException, ExtractionException { KioskList kl = service.getKioskList(); KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl); - extractor.setContentCountry(contentCountry); return extractor.getPage(pageUrl); } - public static KioskInfo getInfo(String url, - String contentCountry) throws IOException, ExtractionException { - return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry); + public static KioskInfo getInfo(String url) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); } public static KioskInfo getInfo(StreamingService service, - String url, - String contentCountry) throws IOException, ExtractionException { + String url) throws IOException, ExtractionException { KioskList kl = service.getKioskList(); KioskExtractor extractor = kl.getExtractorByUrl(url, null); - extractor.setContentCountry(contentCountry); extractor.fetchPage(); return getInfo(extractor); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index b37b7c04..22a9f2ca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -23,7 +23,6 @@ public class KioskList { private final int service_id; private final HashMap kioskList = new HashMap<>(); private String defaultKiosk = null; - private final Localization localization; private class KioskEntry { public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) { @@ -34,9 +33,8 @@ public class KioskList { final ListLinkHandlerFactory handlerFactory; } - public KioskList(int service_id, Localization localization) { + public KioskList(int service_id) { this.service_id = service_id; - this.localization = localization; } public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id) @@ -53,13 +51,18 @@ public class KioskList { public KioskExtractor getDefaultKioskExtractor(String nextPageUrl) throws ExtractionException, IOException { + return getDefaultKioskExtractor(nextPageUrl, NewPipe.getPreferredLocalization()); + } + + public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization) + throws ExtractionException, IOException { if(defaultKiosk != null && !defaultKiosk.equals("")) { - return getExtractorById(defaultKiosk, nextPageUrl); + return getExtractorById(defaultKiosk, nextPageUrl, localization); } else { if(!kioskList.isEmpty()) { // if not set get any entry Object[] keySet = kioskList.keySet().toArray(); - return getExtractorById(keySet[0].toString(), nextPageUrl); + return getExtractorById(keySet[0].toString(), nextPageUrl, localization); } else { return null; } @@ -72,6 +75,11 @@ public class KioskList { public KioskExtractor getExtractorById(String kioskId, String nextPageUrl) throws ExtractionException, IOException { + return getExtractorById(kioskId, nextPageUrl, NewPipe.getPreferredLocalization()); + } + + public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization) + throws ExtractionException, IOException { KioskEntry ke = kioskList.get(kioskId); if(ke == null) { throw new ExtractionException("No kiosk found with the type: " + kioskId); @@ -86,11 +94,16 @@ public class KioskList { } public KioskExtractor getExtractorByUrl(String url, String nextPageUrl) + throws ExtractionException, IOException{ + return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization()); + } + + public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization) throws ExtractionException, IOException { for(Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); if(ke.handlerFactory.acceptUrl(url)) { - return getExtractorById(e.getKey(), nextPageUrl); + return getExtractorById(e.getKey(), nextPageUrl, localization); } } throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index c43d34ce..0202bba5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -68,7 +68,7 @@ public class SoundcloudService extends StreamingService { } @Override - public KioskList getKioskList(Localization localization) throws ExtractionException { + public KioskList getKioskList() throws ExtractionException { KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() { @Override public KioskExtractor createNewKiosk(StreamingService streamingService, @@ -81,7 +81,7 @@ public class SoundcloudService extends StreamingService { } }; - KioskList list = new KioskList(getServiceId(), localization); + KioskList list = new KioskList(getServiceId()); // add kiosks here e.g.: final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index d9187ca4..57cd3f85 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -90,8 +90,8 @@ public class YoutubeService extends StreamingService { } @Override - public KioskList getKioskList(final Localization localization) throws ExtractionException { - KioskList list = new KioskList(getServiceId(), localization); + public KioskList getKioskList() throws ExtractionException { + KioskList list = new KioskList(getServiceId()); // add kiosks here e.g.: try { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index ea88e350..2c026f36 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -177,7 +177,7 @@ public class YoutubeStreamExtractorDefaultTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=LzR8Sf5PK2Q"); + .getStreamExtractor("https://www.youtube.com/watch?v=fBc4Q_htqPg"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java index 85480eaf..68a5bae0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java @@ -46,7 +46,7 @@ public class YoutubeTrendingKioskInfoTest { StreamingService service = YouTube; LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending"); - kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl(), null); + kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl()); } @Test From 0c3dc86b07e5d57970cd7e994af0ee62b49b8484 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 6 Oct 2018 12:22:37 +0200 Subject: [PATCH 02/16] add support for yt content country --- .../extractor/kiosk/KioskExtractor.java | 17 ----------------- .../extractors/YoutubeSearchExtractor.java | 18 ++++++------------ .../extractors/YoutubeTrendingExtractor.java | 2 +- .../youtube/YoutubeTrendingExtractorTest.java | 1 - 4 files changed, 7 insertions(+), 31 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 8c003b01..481bb63c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -30,7 +30,6 @@ import org.schabi.newpipe.extractor.utils.Localization; import javax.annotation.Nonnull; public abstract class KioskExtractor extends ListExtractor { - private String contentCountry = null; private final String id; public KioskExtractor(StreamingService streamingService, @@ -41,17 +40,6 @@ public abstract class KioskExtractor extends ListExtractor { this.id = kioskId; } - /** - * For certain Websites the content of a kiosk will be different depending - * on the country you want to poen the website in. Therefore you should - * set the contentCountry. - * @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm - */ - public void setContentCountry(String contentCountry) { - this.contentCountry = contentCountry; - } - - @Nonnull @Override public String getId() { @@ -69,9 +57,4 @@ public abstract class KioskExtractor extends ListExtractor { @Nonnull @Override public abstract String getName() throws ParsingException; - - - public String getContentCountry() { - return contentCountry; - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 1c1bad6b..2d411fa7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -53,22 +53,16 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String site; - final String url = getUrl(); - final String contentCountry = getLocalization().getCountry(); + final String url = getUrl() + "?gl="+ getLocalization().getCountry(); //String url = builder.build().toString(); //if we've been passed a valid language code, append it to the URL - if (!contentCountry.isEmpty()) { - //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); - site = downloader.download(url, getLocalization()); - } else { - site = downloader.download(url); - } + site = downloader.download(url, getLocalization()); doc = Jsoup.parse(site, url); } @Override - public String getSearchSuggestion() throws ParsingException { + public String getSearchSuggestion() { final Element el = doc.select("div[class*=\"spell-correction\"]").first(); if (el != null) { return el.select("a").first().text(); @@ -79,13 +73,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() throws ExtractionException { return new InfoItemsPage<>(collectItems(doc), getNextPageUrl()); } @Override public String getNextPageUrl() throws ExtractionException { - return getUrl() + "&page=" + Integer.toString( 2); + return getUrl() + "&page=" + 2; } @Override @@ -104,7 +98,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getQuery()) .get("page")); - return currentUrl.replace("&page=" + Integer.toString( pageNr), + return currentUrl.replace("&page=" + pageNr, "&page=" + Integer.toString(pageNr + 1)); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 9dd0a637..f33f7b8a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -50,7 +50,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String contentCountry = getContentCountry(); + final String contentCountry = getLocalization().getCountry(); String url = getUrl(); if(contentCountry != null && !contentCountry.isEmpty()) { url += "?gl=" + contentCountry; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index fb61797b..8dee92e0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -51,7 +51,6 @@ public class YoutubeTrendingExtractorTest { .getKioskList() .getExtractorById("Trending", null); extractor.fetchPage(); - extractor.setContentCountry("de"); } @Test From 0920680708b79a59d351f64ef57e6fb3cdcd98b0 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 6 Oct 2018 17:03:55 +0200 Subject: [PATCH 03/16] fix search country for next page --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 2 +- .../youtube/search/YoutubeSearchExtractorChannelOnlyTest.java | 4 ++-- .../youtube/search/YoutubeSearchExtractorDefaultTest.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 2d411fa7..ddac5499 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -79,7 +79,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getNextPageUrl() throws ExtractionException { - return getUrl() + "&page=" + 2; + return getUrl() + "&page=" + 2 + "&gl=" + getLocalization().getCountry(); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 3b86042d..937cee5e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -45,12 +45,12 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } assertFalse("First and second page are equal", equals); - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=3", secondPage.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=3&gl=GB", secondPage.getNextPageUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=2", extractor.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=2&gl=GB", extractor.getNextPageUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 234ea5f9..6b0d09c7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -53,7 +53,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&page=2", extractor.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&page=2&gl=GB", extractor.getNextPageUrl()); } @Test @@ -96,7 +96,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas } assertFalse("First and second page are equal", equals); - assertEquals("https://www.youtube.com/results?q=pewdiepie&page=3", secondPage.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&page=3&gl=GB", secondPage.getNextPageUrl()); } @Test From 24cb684e235314bdf2bb45be7d8e07f01bae6fd2 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 6 Oct 2018 17:11:12 +0200 Subject: [PATCH 04/16] fix failing test --- .../youtube/search/YoutubeSearchExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 6b0d09c7..3df6434e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -58,7 +58,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(0); + InfoItem firstInfoItem = itemsPage.getItems().get(1); // THe channel should be the first item assertTrue(firstInfoItem instanceof ChannelInfoItem); From 99a0134b1cf031f96a9bf6fedda627bcc69fa957 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 19 Oct 2018 19:33:36 +0530 Subject: [PATCH 05/16] removed unchecked cast --- .../extractors/YoutubeCommentsExtractor.java | 14 +++---- .../YoutubeCommentsInfoItemExtractor.java | 24 ++++++------ .../newpipe/extractor/utils/JsonUtils.java | 39 +++++++++++++++++++ 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index cede2a4d..dee0b21c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -62,7 +62,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { JsonArray arr; try { - arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); + arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); } catch (Exception e) { return ""; } @@ -71,7 +71,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } String continuation; try { - continuation = (String) JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation"); + continuation = JsonUtils.getString(arr.getObject(0), "nextContinuationData.continuation"); } catch (Exception e) { return ""; } @@ -111,7 +111,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { JsonArray contents; try { - contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.items"); + contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items"); }catch(Exception e) { //no comments return; @@ -135,7 +135,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { private void fetchTitle(JsonArray contents) { if(null == title) { try { - title = getYoutubeText((JsonObject) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); + title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); } catch (Exception e) { title = "Youtube Comments"; } @@ -196,13 +196,13 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException { try { - return (String) JsonUtils.getValue(object, "simpleText"); + return JsonUtils.getString(object, "simpleText"); } catch (Exception e1) { try { - JsonArray arr = (JsonArray) JsonUtils.getValue(object, "runs"); + JsonArray arr = JsonUtils.getArray(object, "runs"); String result = ""; for(int i=0; i getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { From f58c914e7330f24a04cbc1894c3f9d6fecc615ae Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 19 Oct 2018 20:17:26 +0530 Subject: [PATCH 06/16] merge fix --- .../java/org/schabi/newpipe/extractor/StreamingService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 539492cc..0921e67a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -122,7 +122,7 @@ public abstract class StreamingService { } public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException { - return getCommentsExtractor(urlIdHandler, NewPipe.getLocalization()); + return getCommentsExtractor(urlIdHandler, NewPipe.getPreferredLocalization()); } //////////////////////////////////////////// @@ -183,7 +183,7 @@ public abstract class StreamingService { if(null == llhf) { return null; } - return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getLocalization()); + return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getPreferredLocalization()); } public abstract boolean isCommentsSupported(); From 32d316330c26f7a2611721701158248c0dfd583c Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Thu, 25 Oct 2018 15:46:47 +0200 Subject: [PATCH 07/16] fix broken search results --- .../youtube/extractors/YoutubeSearchExtractor.java | 9 +++++++-- .../services/youtube/YoutubePlaylistExtractorTest.java | 2 +- .../search/YoutubeSearchExtractorChannelOnlyTest.java | 4 ++-- .../search/YoutubeSearchExtractorDefaultTest.java | 10 +++++++--- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index ddac5499..709e5f57 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -53,7 +53,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String site; - final String url = getUrl() + "?gl="+ getLocalization().getCountry(); + final String url = getUrl(); //String url = builder.build().toString(); //if we've been passed a valid language code, append it to the URL site = downloader.download(url, getLocalization()); @@ -61,6 +61,11 @@ public class YoutubeSearchExtractor extends SearchExtractor { doc = Jsoup.parse(site, url); } + @Override + public String getUrl() throws ParsingException { + return super.getUrl() + "&gl="+ getLocalization().getCountry(); + } + @Override public String getSearchSuggestion() { final Element el = doc.select("div[class*=\"spell-correction\"]").first(); @@ -79,7 +84,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getNextPageUrl() throws ExtractionException { - return getUrl() + "&page=" + 2 + "&gl=" + getLocalization().getCountry(); + return getUrl() + "&page=" + 2; } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 1cb6388a..9cfd6c00 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -219,7 +219,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testUploaderName() throws Exception { - assertEquals("Tomas Nilsson", extractor.getUploaderName()); + assertEquals("Tomas Nilsson TOMPA571", extractor.getUploaderName()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 937cee5e..14b94b51 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -45,12 +45,12 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } assertFalse("First and second page are equal", equals); - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=3&gl=GB", secondPage.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=2&gl=GB", extractor.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 3df6434e..172ca62c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -49,16 +49,20 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas itemsPage = extractor.getInitialPage(); } + @Test + public void testGetUrl() throws Exception { + assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl()); + } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&page=2&gl=GB", extractor.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl()); } @Test public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(1); + InfoItem firstInfoItem = itemsPage.getItems().get(0); // THe channel should be the first item assertTrue(firstInfoItem instanceof ChannelInfoItem); @@ -96,7 +100,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas } assertFalse("First and second page are equal", equals); - assertEquals("https://www.youtube.com/results?q=pewdiepie&page=3&gl=GB", secondPage.getNextPageUrl()); + assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test From 4de99ae28f28c52eab0584c62c93f7de25fe4f92 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 6 Nov 2018 14:24:47 +0100 Subject: [PATCH 08/16] add support for segmented streams --- .../newpipe/extractor/stream/StreamInfo.java | 32 +++++++ .../extractor/utils/DashMpdParser.java | 91 +++++++++++++++---- .../YoutubeStreamExtractorDASHTest.java | 10 +- .../search/YoutubeSearchCountTest.java | 2 +- 4 files changed, 115 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 5ab2c2e8..65a02498 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -152,6 +152,9 @@ public class StreamInfo extends Info { streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams()); streamInfo.getAudioStreams().addAll(result.getAudioStreams()); streamInfo.getVideoStreams().addAll(result.getVideoStreams()); + streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams(); + streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams(); + streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams(); } catch (Exception e) { // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl), // just skip the exception (but store it somewhere), as we later check if we have streams anyway. @@ -271,6 +274,11 @@ public class StreamInfo extends Info { private List videoOnlyStreams; private String dashMpdUrl; + private List segmentedVideoStreams; + private List segmentedAudioStreams; + private List segmentedVideoOnlyStreams; + + private String hlsUrl; private StreamInfoItem nextVideo; private List relatedStreams; @@ -431,6 +439,30 @@ public class StreamInfo extends Info { this.dashMpdUrl = dashMpdUrl; } + public List getSegmentedVideoStreams() { + return segmentedVideoStreams; + } + + public void setSegmentedVideoStreams(List segmentedVideoStreams) { + this.segmentedVideoStreams = segmentedVideoStreams; + } + + public List getSegmentedAudioStreams() { + return segmentedAudioStreams; + } + + public void setSegmentedAudioStreams(List segmentedAudioStreams) { + this.segmentedAudioStreams = segmentedAudioStreams; + } + + public List getSegmentedVideoOnlyStreams() { + return segmentedVideoOnlyStreams; + } + + public void setSegmentedVideoOnlyStreams(List segmentedVideoOnlyStreams) { + this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams; + } + public String getHlsUrl() { return hlsUrl; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 6c656d77..8a994cbd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -59,10 +59,23 @@ public class DashMpdParser { private final List audioStreams; private final List videoOnlyStreams; - public ParserResult(List videoStreams, List audioStreams, List videoOnlyStreams) { + private final List segmentedVideoStreams; + private final List segmentedAudioStreams; + private final List segmentedVideoOnlyStreams; + + + public ParserResult(List videoStreams, + List audioStreams, + List videoOnlyStreams, + List segmentedVideoStreams, + List segmentedAudioStreams, + List segmentedVideoOnlyStreams) { this.videoStreams = videoStreams; this.audioStreams = audioStreams; this.videoOnlyStreams = videoOnlyStreams; + this.segmentedVideoStreams = segmentedVideoStreams; + this.segmentedAudioStreams = segmentedAudioStreams; + this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams; } public List getVideoStreams() { @@ -76,10 +89,22 @@ public class DashMpdParser { public List getVideoOnlyStreams() { return videoOnlyStreams; } + + public List getSegmentedVideoStreams() { + return segmentedVideoStreams; + } + + public List getSegmentedAudioStreams() { + return segmentedAudioStreams; + } + + public List getSegmentedVideoOnlyStreams() { + return segmentedVideoOnlyStreams; + } } /** - * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest, + * Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest, * then it will search for any stream that the ItagItem has (by the id). *

* It has video, video only and audio streams and will only add to the list if it don't @@ -90,7 +115,8 @@ public class DashMpdParser { * * @param streamInfo where the parsed streams will be added */ - public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { + public static ParserResult getStreams(final StreamInfo streamInfo) + throws DashMpdParsingException, ReCaptchaException { String dashDoc; Downloader downloader = NewPipe.getDownloader(); try { @@ -113,6 +139,10 @@ public class DashMpdParser { final List audioStreams = new ArrayList<>(); final List videoOnlyStreams = new ArrayList<>(); + final List segmentedVideoStreams = new ArrayList<>(); + final List segmentedAudioStreams = new ArrayList<>(); + final List segmentedVideoOnlyStreams = new ArrayList<>(); + for (int i = 0; i < representationList.getLength(); i++) { final Element representation = (Element) representationList.item(i); try { @@ -126,34 +156,61 @@ public class DashMpdParser { // instead we need to add the "media=" value from the tags inside the // tag in order to get a full working url. However each of these is just pointing to a part of the // video, so we can not return a URL with a working stream here. - // We decided not to ignore such streams for the moment. - if (itag != null && segmentationList == null) { + // Instead of putting those streams into the list of regular stream urls wie put them in a + // for example "segmentedVideoStreams" list. + if (itag != null) { final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); - - if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { - audioStreams.add(audioStream); + if(segmentationList == null) { + final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); + if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { + audioStreams.add(audioStream); + } + } else { + segmentedAudioStreams.add( + new AudioStream(id, mediaFormat, itag.avgBitrate)); } } else { boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly); - if (isVideoOnly) { - if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) { - streamInfo.getVideoOnlyStreams().add(videoStream); - videoOnlyStreams.add(videoStream); + if(segmentationList == null) { + final VideoStream videoStream = new VideoStream(url, + mediaFormat, + itag.resolutionString, + isVideoOnly); + + if (isVideoOnly) { + if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) { + videoOnlyStreams.add(videoStream); + } + } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) { + videoStreams.add(videoStream); + } + } else { + final VideoStream videoStream = new VideoStream(id, + mediaFormat, + itag.resolutionString, + isVideoOnly); + + if(isVideoOnly) { + segmentedVideoOnlyStreams.add(videoStream); + } else { + segmentedVideoStreams.add(videoStream); } - } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) { - videoStreams.add(videoStream); } } } } catch (Exception ignored) { } } - return new ParserResult(videoStreams, audioStreams, videoOnlyStreams); + return new ParserResult( + videoStreams, + audioStreams, + videoOnlyStreams, + segmentedVideoStreams, + segmentedAudioStreams, + segmentedVideoOnlyStreams); } catch (Exception e) { throw new DashMpdParsingException("Could not parse Dash mpd", e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java index 0d4013e9..ceba6092 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java @@ -47,15 +47,21 @@ public class YoutubeStreamExtractorDASHTest { @Test public void testGetDashMpd() { - System.out.println(info.getDashMpdUrl()); assertTrue(info.getDashMpdUrl(), info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty()); } @Test - public void testDashMpdParser() { + public void testRegularStreams() { assertEquals(0, info.getAudioStreams().size()); assertEquals(0, info.getVideoOnlyStreams().size()); assertEquals(4, info.getVideoStreams().size()); } + + @Test + public void testSegmentedStreams() { + assertEquals(2, info.getSegmentedAudioStreams().size()); + assertEquals(3, info.getSegmentedVideoOnlyStreams().size()); + assertEquals(0, info.getSegmentedVideoStreams().size()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java index af314e63..92349c85 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java @@ -28,7 +28,7 @@ public class YoutubeSearchCountTest { public void testViewCount() { ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()), - 65043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 68043316); + 69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 73043316); } } } From 7129d6db55d92fed6de5365c272b6b7a6610681d Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 7 Nov 2018 18:28:44 +0100 Subject: [PATCH 09/16] add inline documentation for StreamExtractor --- .../soundcloud/SoundcloudStreamExtractor.java | 4 +- .../extractors/YoutubeStreamExtractor.java | 4 +- .../extractor/stream/StreamExtractor.java | 255 +++++++++++++++--- .../newpipe/extractor/stream/StreamInfo.java | 3 +- .../extractor/{ => stream}/Subtitles.java | 2 +- .../extractor/utils/ExtractorHelper.java | 2 +- .../SoundcloudStreamExtractorDefaultTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 4 +- 8 files changed, 223 insertions(+), 53 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/{ => stream}/Subtitles.java (94%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index f19f46c3..e7fe83d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -188,12 +188,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItem getNextVideo() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() throws IOException, ExtractionException { return null; } @Override - public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 66898557..6e310b13 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -490,7 +490,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItem getNextVideo() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() throws IOException, ExtractionException { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); @@ -504,7 +504,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 6071b2dc..e4a83729 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -22,7 +22,6 @@ package org.schabi.newpipe.extractor.stream; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.Subtitles; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -34,7 +33,7 @@ import java.io.IOException; import java.util.List; /** - * Scrapes information from a video streaming service (eg, YouTube). + * Scrapes information from a video/audio streaming service (eg, YouTube). */ public abstract class StreamExtractor extends Extractor { @@ -44,22 +43,234 @@ public abstract class StreamExtractor extends Extractor { super(service, linkHandler, localization); } + /** + * The day on which the stream got uploaded/created. The return information should be in the format + * dd.mm.yyyy, however it NewPipe will not crash if its not. + * @return The day on which the stream got uploaded. + * @throws ParsingException + */ @Nonnull public abstract String getUploadDate() throws ParsingException; + + /** + * This will return the url to the thumbnail of the stream. Try to return the medium resolution here. + * @return The url of the thumbnail. + * @throws ParsingException + */ @Nonnull public abstract String getThumbnailUrl() throws ParsingException; + + /** + * This is the stream description. On YouTube this is the video description. You can return simple HTML here. + * @return The description of the stream/video. + * @throws ParsingException + */ @Nonnull public abstract String getDescription() throws ParsingException; /** - * Get the age limit + * Get the age limit. * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit * @throws ParsingException if an error occurs while parsing */ public abstract int getAgeLimit() throws ParsingException; + /** + * This should return the length of a video in seconds. + * @return The length of the stream in seconds. + * @throws ParsingException + */ public abstract long getLength() throws ParsingException; + + /** + * If the url you are currently handling contains a time stamp/seek, you can return the + * position it represents here. + * If the url has no time stamp simply return zero. + * @return the timestamp in seconds + * @throws ParsingException + */ public abstract long getTimeStamp() throws ParsingException; + + /** + * The count of how many people have watched the video/listened to the audio stream. + * If the current stream has no view count or its not available simply return -1 + * @return amount of views. + * @throws ParsingException + */ + public abstract long getViewCount() throws ParsingException; + + /** + * The Amount of likes a video/audio stream got. + * If the current stream has no likes or its not available simply return -1 + * @return the amount of likes the stream got + * @throws ParsingException + */ + public abstract long getLikeCount() throws ParsingException; + + /** + * The Amount of dislikes a video/audio stream got. + * If the current stream has no dislikes or its not available simply return -1 + * @return the amount of likes the stream got + * @throws ParsingException + */ + public abstract long getDislikeCount() throws ParsingException; + + /** + * The Url to the page of the creator/uploader of the stream. This must not be a homepage, + * but the page offered by the service the extractor handles. This url will be handled by the + * ChannelExtractor, + * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects + * this url. + * @return the url to the page of the creator/uploader of the stream or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getUploaderUrl() throws ParsingException; + + /** + * The name of the creator/uploader of the stream. + * If the name is not available you can simply return an empty string. + * @return the name of the creator/uploader of the stream or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getUploaderName() throws ParsingException; + + /** + * The url to the image file/profile picture/avatar of the creator/uploader of the stream. + * If the url is not available you can return an empty String. + * @return The url of the image file of the uploader or an empty String + * @throws ParsingException + */ + @Nonnull + public abstract String getUploaderAvatarUrl() throws ParsingException; + + /** + * Get the dash mpd url. If you don't know what a dash MPD is you can read about it + * here. + * @return the url as a string or an empty string + * @throws ParsingException if an error occurs while reading + */ + @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + + /** + * I am not sure if this is in use, and how this is used. However the frontend is missing support + * for HLS streams. Prove me if I am wrong. Please open an + * issue, + * or fix this description if you know whats up with this. + * @return The Url to the hls stream. + * @throws ParsingException + */ + @Nonnull public abstract String getHlsUrl() throws ParsingException; + + /** + * This should return a list of available + * AudioStreams + * You can also return null or an empty list, however be aware that if you don't return anything + * in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as + * a failed extraction procedure. + * @return a list of audio only streams in the format of AudioStream + * @throws IOException + * @throws ExtractionException + */ + public abstract List getAudioStreams() throws IOException, ExtractionException; + + /** + * This should return a list of available + * VideoStreams + * Be aware this is the list of video streams which do contain an audio stream. + * You can also return null or an empty list, however be aware that if you don't return anything + * in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as + * a failed extraction procedure. + * @return a list of combined video and streams in the format of AudioStream + * @throws IOException + * @throws ExtractionException + */ + public abstract List getVideoStreams() throws IOException, ExtractionException; + + /** + * This should return a list of available + * VideoStreams. + * Be aware this is the list of video streams which do NOT contain an audio stream. + * You can also return null or an empty list, however be aware that if you don't return anything + * in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as + * a failed extraction procedure. + * @return a list of video and streams in the format of AudioStream + * @throws IOException + * @throws ExtractionException + */ + public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; + + /** + * This will return a list of available + * Subtitless. + * If no subtitles are available an empty list can returned. + * @return a list of available subtitles or an empty list + * @throws IOException + * @throws ExtractionException + */ + @Nonnull + public abstract List getSubtitlesDefault() throws IOException, ExtractionException; + + /** + * This will return a list of available + * Subtitless. + * given by a specific type. + * If no subtitles in that specific format are available an empty list can returned. + * @return a list of available subtitles or an empty list + * @throws IOException + * @throws ExtractionException + */ + @Nonnull + public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException; + + /** + * Get the StreamType. + * @return the type of the stream + * @throws ParsingException + */ + public abstract StreamType getStreamType() throws ParsingException; + + /** + * should return the url of the next stream. NewPipe will automatically play + * the next stream if the user wants that. + * If the next stream is is not available simply return null + * @return the InfoItem of the next stream + * @throws IOException + * @throws ExtractionException + */ + public abstract StreamInfoItem getNextStream() throws IOException, ExtractionException; + + /** + * Should return a list of streams related to the current handled. Many services show suggested + * streams. If you don't like suggested streams you should implement them anyway since they can + * be disabled by the user later in the frontend. + * This list MUST NOT contain the next available video as this should be return through getNextStream() + * If is is not available simply return null + * @return a list of InfoItems showing the related videos/streams + * @throws IOException + * @throws ExtractionException + */ + public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException; + + /** + * Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block) + * + * @return Error message; null if there is no error message. + */ + public abstract String getErrorMessage(); + + ////////////////////////////////////////////////////////////////// + /// Helper + ////////////////////////////////////////////////////////////////// + + /** + * Override this function if the format of time stamp in the url is not the same format as that form youtube. + * Honestly I don't even know the time stamp fromat of youtube. + * @param regexPattern + * @return the sime stamp/seek for the video in seconds + * @throws ParsingException + */ protected long getTimestampSeconds(String regexPattern) throws ParsingException { String timeStamp; try { @@ -104,42 +315,4 @@ public abstract class StreamExtractor extends Extractor { } else { return 0; }}; - - public abstract long getViewCount() throws ParsingException; - public abstract long getLikeCount() throws ParsingException; - public abstract long getDislikeCount() throws ParsingException; - - @Nonnull - public abstract String getUploaderUrl() throws ParsingException; - @Nonnull - public abstract String getUploaderName() throws ParsingException; - @Nonnull - public abstract String getUploaderAvatarUrl() throws ParsingException; - - /** - * Get the dash mpd url - * @return the url as a string or an empty string - * @throws ParsingException if an error occurs while reading - */ - @Nonnull public abstract String getDashMpdUrl() throws ParsingException; - @Nonnull public abstract String getHlsUrl() throws ParsingException; - public abstract List getAudioStreams() throws IOException, ExtractionException; - public abstract List getVideoStreams() throws IOException, ExtractionException; - public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; - - @Nonnull - public abstract List getSubtitlesDefault() throws IOException, ExtractionException; - @Nonnull - public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException; - - public abstract StreamType getStreamType() throws ParsingException; - public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; - public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; - - /** - * Analyses the webpage's document and extracts any error message there might be. - * - * @return Error message; null if there is no error message. - */ - public abstract String getErrorMessage(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 65a02498..f9732f4c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.ExtractorHelper; -import org.schabi.newpipe.extractor.utils.Localization; import java.io.IOException; import java.util.ArrayList; @@ -240,7 +239,7 @@ public class StreamInfo extends Info { streamInfo.addError(e); } try { - streamInfo.setNextVideo(extractor.getNextVideo()); + streamInfo.setNextVideo(extractor.getNextStream()); } catch (Exception e) { streamInfo.addError(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java similarity index 94% rename from extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java index affd854e..81488328 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor; +package org.schabi.newpipe.extractor.stream; import org.schabi.newpipe.extractor.stream.SubtitlesFormat; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index bad59593..445c52c7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -29,7 +29,7 @@ public class ExtractorHelper { public static List getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { try { - InfoItemsCollector collector = extractor.getRelatedVideos(); + InfoItemsCollector collector = extractor.getRelatedStreams(); info.addAllErrors(collector.getErrors()); //noinspection unchecked diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index b6c5b3a2..3970efde 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -101,7 +101,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); assertFalse(relatedVideos.getItems().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 2c026f36..75dc5e00 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.*; -import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Utils; @@ -17,7 +16,6 @@ import java.io.IOException; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeTrendingExtractorTest.extractor; /* * Created by Christian Schabesberger on 30.12.15. @@ -151,7 +149,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty()); From e4afb2186214509997a8768adcdfb899f0e73c9f Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 24 Sep 2018 16:04:22 -0300 Subject: [PATCH 10/16] make Subtitle object Stream compliant * merge SubtitlesFormat into MediaFormat * implement Stream interface on Subtitle class * misc fixes: dont show a snackbar error on age-restricted videos, lint fix. --- .../schabi/newpipe/extractor/MediaFormat.java | 9 ++- .../soundcloud/SoundcloudStreamExtractor.java | 4 +- .../extractors/YoutubeStreamExtractor.java | 26 +++---- .../extractor/stream/StreamExtractor.java | 42 ++++++++++- .../newpipe/extractor/stream/StreamInfo.java | 6 +- .../extractor/stream/SubtitlesStream.java | 73 +++++++++++++++++++ 6 files changed, 140 insertions(+), 20 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index e39e6977..21ead42c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -36,7 +36,14 @@ public enum MediaFormat { M4A (0x3, "m4a", "m4a", "audio/mp4"), WEBMA (0x4, "WebM", "webm", "audio/webm"), MP3 (0x5, "MP3", "mp3", "audio/mpeg"), - OPUS (0x6, "opus", "opus", "audio/opus"); + OPUS (0x6, "opus", "opus", "audio/opus"), + // subtitles formats + VTT (0x7, "WebVTT", "vtt", "text/vtt"), + TTML (0x8, "Timed Text Markup Language", "ttml", "application/ttml+xml"), + TRANSCRIPT1 (0x9, "TranScript v1", "srv1", "text/xml"), + TRANSCRIPT2 (0xA, "TranScript v2", "srv2", "text/xml"), + TRANSCRIPT3 (0xB, "TranScript v3", "srv3", "text/xml"), + SRT (0xC, "SubRip file format", "srt", "text/srt"); public final int id; public final String name; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index e7fe83d9..005722e3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -172,13 +172,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitlesDefault() throws IOException, ExtractionException { return Collections.emptyList(); } @Override @Nonnull - public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { + public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { return Collections.emptyList(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 6e310b13..e5208d5a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -460,15 +461,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { - return getSubtitles(SubtitlesFormat.TTML); + public List getSubtitlesDefault() throws IOException, ExtractionException { + return getSubtitles(MediaFormat.TTML); } @Override @Nonnull - public List getSubtitles(final SubtitlesFormat format) throws IOException, ExtractionException { + public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { assertPageFetched(); - List subtitles = new ArrayList<>(); + List subtitles = new ArrayList<>(); for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) { subtitles.add(subtitlesInfo.getSubtitle(format)); } @@ -494,9 +495,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]") - .first().select("li").first())); + Elements watch = doc.select("div[class=\"watch-sidebar-section\"]"); + if (watch.size() < 1) { + return null;// prevent the snackbar notification "report error" on age-restricted videos + } + + collector.commit(extractVideoPreviewInfo(watch.first().select("li").first())); return collector.getItems().get(0); } catch (Exception e) { throw new ParsingException("Could not get next video", e); @@ -815,21 +820,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String languageCode; final boolean isGenerated; - final Locale locale; - public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) { this.cleanUrl = baseUrl .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists .replaceAll("&tlang=[^&]*", ""); // Remove translation language this.languageCode = languageCode; this.isGenerated = isGenerated; - - final String[] splits = languageCode.split("-"); - this.locale = splits.length == 2 ? new Locale(splits[0], splits[1]) : new Locale(languageCode); } - public Subtitles getSubtitle(final SubtitlesFormat format) { - return new Subtitles(format, locale, cleanUrl + "&fmt=" + format.getExtension(), isGenerated); + public SubtitlesStream getSubtitle(final MediaFormat format) { + return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index e4a83729..7d1a5e69 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.stream; */ import org.schabi.newpipe.extractor.Extractor; +import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -314,5 +315,44 @@ public abstract class StreamExtractor extends Extractor { } } else { return 0; - }}; + } + } + + public abstract long getViewCount() throws ParsingException; + public abstract long getLikeCount() throws ParsingException; + public abstract long getDislikeCount() throws ParsingException; + + @Nonnull + public abstract String getUploaderUrl() throws ParsingException; + @Nonnull + public abstract String getUploaderName() throws ParsingException; + @Nonnull + public abstract String getUploaderAvatarUrl() throws ParsingException; + + /** + * Get the dash mpd url + * @return the url as a string or an empty string + * @throws ParsingException if an error occurs while reading + */ + @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + @Nonnull public abstract String getHlsUrl() throws ParsingException; + public abstract List getAudioStreams() throws IOException, ExtractionException; + public abstract List getVideoStreams() throws IOException, ExtractionException; + public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; + + @Nonnull + public abstract List getSubtitlesDefault() throws IOException, ExtractionException; + @Nonnull + public abstract List getSubtitles(MediaFormat format) throws IOException, ExtractionException; + + public abstract StreamType getStreamType() throws ParsingException; + public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; + public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; + + /** + * Analyses the webpage's document and extracts any error message there might be. + * + * @return Error message; null if there is no error message. + */ + public abstract String getErrorMessage(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index f9732f4c..6b3afb97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -283,7 +283,7 @@ public class StreamInfo extends Info { private List relatedStreams; private long startPosition = 0; - private List subtitles; + private List subtitles; /** * Get the stream type @@ -494,11 +494,11 @@ public class StreamInfo extends Info { this.startPosition = startPosition; } - public List getSubtitles() { + public List getSubtitles() { return subtitles; } - public void setSubtitles(List subtitles) { + public void setSubtitles(List subtitles) { this.subtitles = subtitles; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java new file mode 100644 index 00000000..d0e09ac1 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -0,0 +1,73 @@ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.MediaFormat; + +import java.io.Serializable; +import java.util.Locale; + +public class SubtitlesStream extends Stream implements Serializable { + private final MediaFormat format; + private final Locale locale; + private final String url; + private final boolean autoGenerated; + private final String code; + + public SubtitlesStream(MediaFormat format, String languageCode, String url, boolean autoGenerated) { + super(url, format); + + /* + * Locale.forLanguageTag only for API >= 21 + * Locale.Builder only for API >= 21 + * Country codes doesn't work well without + */ + final String[] splits = languageCode.split("-"); + switch (splits.length) { + default: + this.locale = new Locale(splits[0]); + break; + case 3: + this.locale = new Locale(splits[0], splits[1], splits[2]);// complex variants doesn't work! + break; + case 2: + this.locale = new Locale(splits[0], splits[1]); + break; + } + this.code = languageCode; + this.format = format; + this.url = url; + this.autoGenerated = autoGenerated; + } + + public String getExtension() { + return format.suffix; + } + + public String getURL() { + return url; + } + + public boolean isAutoGenerated() { + return autoGenerated; + } + + @Override + public boolean equalStats(Stream cmp) { + return super.equalStats(cmp)&& + cmp instanceof SubtitlesStream && + code.equals(((SubtitlesStream) cmp).code) && + autoGenerated == ((SubtitlesStream) cmp).autoGenerated; + } + + public String getDisplayLanguageName() { + return locale.getDisplayName(locale); + } + + public String getLanguageTag() { + return code; + } + + public Locale getLocale() { + return locale; + } + +} From 388f4fd38ba09c1afdaf924f440e4ad3a8a5dc6e Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 24 Sep 2018 16:05:07 -0300 Subject: [PATCH 11/16] Delete SubtitlesFormat.java merge into MediaFormat --- .../extractor/stream/SubtitlesFormat.java | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java deleted file mode 100644 index 3a16f105..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -public enum SubtitlesFormat { - // YouTube subtitles formats - // TRANSCRIPT(3) is default YT format based on TTML, - // but unlike VTT or TTML, it is NOT W3 standard - // TRANSCRIPT subtitles are NOT supported by ExoPlayer, only VTT and TTML - VTT (0x0, "vtt"), - TTML (0x1, "ttml"), - TRANSCRIPT1 (0x2, "srv1"), - TRANSCRIPT2 (0x3, "srv2"), - TRANSCRIPT3 (0x4, "srv3"); - - private final int id; - private final String extension; - - SubtitlesFormat(int id, String extension) { - this.id = id; - this.extension = extension; - } - - public String getExtension() { - return extension; - } -} From b40a1f3657c0ea39495c2ab92a591c9da71ac06a Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 24 Sep 2018 16:05:26 -0300 Subject: [PATCH 12/16] Delete Subtitles.java change class name --- .../newpipe/extractor/stream/Subtitles.java | 34 ------------------- ...utubeStreamExtractorAgeRestrictedTest.java | 4 +-- ...utubeStreamExtractorControversialTest.java | 4 +-- 3 files changed, 4 insertions(+), 38 deletions(-) delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java deleted file mode 100644 index 81488328..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Subtitles.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -import org.schabi.newpipe.extractor.stream.SubtitlesFormat; - -import java.io.Serializable; -import java.util.Locale; - -public class Subtitles implements Serializable { - private final SubtitlesFormat format; - private final Locale locale; - private final String URL; - private final boolean autoGenerated; - - public Subtitles(SubtitlesFormat format, Locale locale, String URL, boolean autoGenerated) { - this.format = format; - this.locale = locale; - this.URL = URL; - this.autoGenerated = autoGenerated; - } - - public SubtitlesFormat getFileType() { return format; } - - public Locale getLocale() { - return locale; - } - - public String getURL() { - return URL; - } - - public boolean isAutoGenerated() { - return autoGenerated; - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java index ec8b495f..dbf8d6d6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java @@ -4,13 +4,13 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Localization; @@ -131,6 +131,6 @@ public class YoutubeStreamExtractorAgeRestrictedTest { @Test public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); + assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java index a5ac490e..a300d622 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java @@ -4,13 +4,13 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Localization; @@ -124,6 +124,6 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(!extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); + assertTrue(!extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } From a78ad162355d1e25b5646ed36319fb5c241abca3 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 9 Nov 2018 19:55:40 +0100 Subject: [PATCH 13/16] rebase correctly --- .../extractor/stream/StreamExtractor.java | 45 ++----------------- .../YoutubeStreamExtractorDefaultTest.java | 3 +- .../YoutubeSearchExtractorDefaultTest.java | 4 +- 3 files changed, 8 insertions(+), 44 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 7d1a5e69..3b9f0653 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.stream; /* - * Created by Christian Schabesberger on 10.08.15. + * Created by Christian Schabesberger on 10.08.18. * * Copyright (C) Christian Schabesberger 2016 * StreamExtractor.java is part of NewPipe. @@ -211,19 +211,20 @@ public abstract class StreamExtractor extends Extractor { * @throws ExtractionException */ @Nonnull - public abstract List getSubtitlesDefault() throws IOException, ExtractionException; + public abstract List getSubtitlesDefault() throws IOException, ExtractionException; /** * This will return a list of available * Subtitless. * given by a specific type. * If no subtitles in that specific format are available an empty list can returned. + * @param format the media format by which the subtitles should be filtered * @return a list of available subtitles or an empty list * @throws IOException * @throws ExtractionException */ @Nonnull - public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException; + public abstract List getSubtitles(MediaFormat format) throws IOException, ExtractionException; /** * Get the StreamType. @@ -317,42 +318,4 @@ public abstract class StreamExtractor extends Extractor { return 0; } } - - public abstract long getViewCount() throws ParsingException; - public abstract long getLikeCount() throws ParsingException; - public abstract long getDislikeCount() throws ParsingException; - - @Nonnull - public abstract String getUploaderUrl() throws ParsingException; - @Nonnull - public abstract String getUploaderName() throws ParsingException; - @Nonnull - public abstract String getUploaderAvatarUrl() throws ParsingException; - - /** - * Get the dash mpd url - * @return the url as a string or an empty string - * @throws ParsingException if an error occurs while reading - */ - @Nonnull public abstract String getDashMpdUrl() throws ParsingException; - @Nonnull public abstract String getHlsUrl() throws ParsingException; - public abstract List getAudioStreams() throws IOException, ExtractionException; - public abstract List getVideoStreams() throws IOException, ExtractionException; - public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; - - @Nonnull - public abstract List getSubtitlesDefault() throws IOException, ExtractionException; - @Nonnull - public abstract List getSubtitles(MediaFormat format) throws IOException, ExtractionException; - - public abstract StreamType getStreamType() throws ParsingException; - public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; - public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; - - /** - * Analyses the webpage's document and extracts any error message there might be. - * - * @return Error message; null if there is no error message. - */ - public abstract String getErrorMessage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 75dc5e00..27b54c54 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -164,7 +165,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); + assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 172ca62c..3e7d3311 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -62,9 +62,9 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(0); + InfoItem firstInfoItem = itemsPage.getItems().get(1); - // THe channel should be the first item + // The channel should be the first item assertTrue(firstInfoItem instanceof ChannelInfoItem); assertEquals("name", "PewDiePie", firstInfoItem.getName()); assertEquals("url","https://www.youtube.com/user/PewDiePie", firstInfoItem.getUrl()); From 91b1efc97e904a2323559002580eb7d6d8576acb Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 10 Nov 2018 10:50:13 +0100 Subject: [PATCH 14/16] add documentation of StreamingService --- README.md | 2 +- .../schabi/newpipe/extractor/ServiceList.java | 22 ++++ .../newpipe/extractor/StreamingService.java | 116 +++++++++++++++++- 3 files changed, 137 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 69d897e6..b13f04ff 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # NewPipe Extractor -[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [Documentation](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) +[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [Documentation](https://teamnewpipe.github.io/documentation/) NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 846355ed..36690438 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -8,6 +8,24 @@ import java.util.List; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; +/* + * Copyright (C) Christian Schabesberger 2018 + * ServiceList.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + /** * A list of supported services. */ @@ -19,6 +37,10 @@ public final class ServiceList { public static final YoutubeService YouTube; public static final SoundcloudService SoundCloud; + /** + * When creating a new service, put this service in the end of this list, + * and give it the next free id. + */ private static final List SERVICES = unmodifiableList( asList( YouTube = new YoutubeService(0), diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index a85d8f3a..b798ec92 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -14,11 +14,38 @@ import org.schabi.newpipe.extractor.utils.Localization; import java.util.Collections; import java.util.List; +/* + * Copyright (C) Christian Schabesberger 2018 + * StreamingService.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + public abstract class StreamingService { + + /** + * This class holds meta information about the service implementation. + */ public static class ServiceInfo { private final String name; private final List mediaCapabilities; + /** + * Creates a new instance of a ServiceInfo + * @param name the name of the service + * @param mediaCapabilities the type of media this service can handle + */ public ServiceInfo(String name, List mediaCapabilities) { this.name = name; this.mediaCapabilities = Collections.unmodifiableList(mediaCapabilities); @@ -37,6 +64,10 @@ public abstract class StreamingService { } } + /** + * LinkType will be used to determine which type of URL you are handling, and therefore which part + * of NewPipe should handle a certain URL. + */ public enum LinkType { NONE, STREAM, @@ -47,6 +78,16 @@ public abstract class StreamingService { private final int serviceId; private final ServiceInfo serviceInfo; + /** + * Creates a new Streaming service. + * If you Implement one do not set id within your implementation of this extractor, instead + * set the id when you put the extractor into + * ServiceList. + * All other parameters can be set directly from the overriding constructor. + * @param id the number of the service to identify him within the NewPipe frontend + * @param name the name of the service + * @param capabilities the type of media this service can handle + */ public StreamingService(int id, String name, List capabilities) { this.serviceId = id; this.serviceInfo = new ServiceInfo(name, capabilities); @@ -68,24 +109,93 @@ public abstract class StreamingService { //////////////////////////////////////////// // Url Id handler //////////////////////////////////////////// + + /** + * Must return a new instance of an implementation of LinkHandlerFactory for streams. + * @return an instance of a LinkHandlerFactory for streams + */ public abstract LinkHandlerFactory getStreamLHFactory(); + + /** + * Must return a new instance of an implementation of ListLinkHandlerFactory for channels. + * If support for channels is not given null must be returned. + * @return an instance of a ListLinkHandlerFactory for channels or null + */ public abstract ListLinkHandlerFactory getChannelLHFactory(); + + /** + * Must return a new instance of an implementation of ListLinkHandlerFactory for playlists. + * If support for playlists is not given null must be returned. + * @return an instance of a ListLinkHandlerFactory for playlists or null + */ public abstract ListLinkHandlerFactory getPlaylistLHFactory(); + + /** + * Must return an instance of an implementation of SearchQueryHandlerFactory. + * @return an instance of a SearchQueryHandlerFactory + */ public abstract SearchQueryHandlerFactory getSearchQHFactory(); //////////////////////////////////////////// // Extractor //////////////////////////////////////////// + + /** + * Must create a new instance of a SearchExtractor implementation. + * @param queryHandler specifies the keyword lock for, and the filters which should be applied. + * @param localization specifies the language/country for the extractor. + * @return a new SearchExtractor instance + */ public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization); + + /** + * Must create a new instance of a SuggestionExtractor implementation. + * @param localization specifies the language/country for the extractor. + * @return a new SuggestionExtractor instance + */ public abstract SuggestionExtractor getSuggestionExtractor(Localization localization); + + /** + * Outdated or obsolete. null can be returned. + * @return just null + */ public abstract SubscriptionExtractor getSubscriptionExtractor(); + + /** + * Must create a new instance of a KioskList implementation. + * @return a new KioskList instance + * @throws ExtractionException + */ public abstract KioskList getKioskList() throws ExtractionException; + /** + * Must create a new instance of a ChannelExtractor implementation. + * @param linkHandler is pointing to the channel which should be handled by this new instance. + * @param localization specifies the language used for the request. + * @return a new ChannelExtractor + * @throws ExtractionException + */ public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) throws ExtractionException; + + /** + * Must crete a new instance of a PlaylistExtractor implementation. + * @param linkHandler is pointing to the playlist which should be handled by this new instance. + * @param localization specifies the language used for the request. + * @return a new PlaylistExtractor + * @throws ExtractionException + */ public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) throws ExtractionException; + + /** + * Must create a new instance of a StreamExtractor implementation. + * @param linkHandler is pointing to the stream which should be handled by this new instance. + * @param localization specifies the language used for the request. + * @return a new StreamExtractor + * @throws ExtractionException + */ public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) throws ExtractionException; //////////////////////////////////////////// @@ -165,9 +275,11 @@ public abstract class StreamingService { return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getPreferredLocalization()); } - /** - * figure out where the link is pointing to (a channel, video, playlist, etc.) + * Figures out where the link is pointing to (a channel, a video, a playlist, etc.) + * @param url the url on which it should be decided of which link type it is + * @return the link type of url + * @throws ParsingException */ public final LinkType getLinkTypeByUrl(String url) throws ParsingException { LinkHandlerFactory sH = getStreamLHFactory(); From 9ad102df3fbd599de26357a716dd740b5a797db5 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 21 Dec 2018 09:35:00 +0530 Subject: [PATCH 15/16] fixed fetching youtube client name --- .../extractors/YoutubeCommentsExtractor.java | 6 ++++- .../newpipe/extractor/utils/Parser.java | 22 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index dee0b21c..94beaf88 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -7,6 +7,8 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.annotation.Nonnull; @@ -25,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -34,6 +37,7 @@ import com.grack.nanojson.JsonParser; public class YoutubeCommentsExtractor extends CommentsExtractor { private static final String USER_AGENT = "Mozilla/5.0 (Android 8.1.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0"; + private static final Pattern YT_CLIENT_NAME_PATTERN = Pattern.compile("INNERTUBE_CONTEXT_CLIENT_NAME\\\":(.*?)[,}]"); private String ytClientVersion; private String ytClientName; @@ -150,7 +154,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { DownloadResponse response = downloader.get(getUrl(), request); String responseBody = response.getResponseBody(); ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\""); - ytClientName = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_NAME\":", ","); + ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody); String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}"); String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\""); initPage = getPage(getNextPageUrl(commentsToken)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 42f098dc..6cd93897 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor.utils; -import org.nibor.autolink.LinkExtractor; -import org.nibor.autolink.LinkSpan; -import org.nibor.autolink.LinkType; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; @@ -14,6 +9,11 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.nibor.autolink.LinkExtractor; +import org.nibor.autolink.LinkSpan; +import org.nibor.autolink.LinkType; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + /* * Created by Christian Schabesberger on 02.02.16. * @@ -51,18 +51,26 @@ public class Parser { public static String matchGroup1(String pattern, String input) throws RegexException { return matchGroup(pattern, input, 1); } + + public static String matchGroup1(Pattern pattern, String input) throws RegexException { + return matchGroup(pattern, input, 1); + } public static String matchGroup(String pattern, String input, int group) throws RegexException { Pattern pat = Pattern.compile(pattern); + return matchGroup(pat, input, group); + } + + public static String matchGroup(Pattern pat, String input, int group) throws RegexException { Matcher mat = pat.matcher(input); boolean foundMatch = mat.find(); if (foundMatch) { return mat.group(group); } else { if (input.length() > 1024) { - throw new RegexException("failed to find pattern \"" + pattern); + throw new RegexException("failed to find pattern \"" + pat.pattern()); } else { - throw new RegexException("failed to find pattern \"" + pattern + " inside of " + input + "\""); + throw new RegexException("failed to find pattern \"" + pat.pattern() + " inside of " + input + "\""); } } } From c77050dc70cf1a7d0c7a53de688b9b67a5db79c1 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 21 Dec 2018 11:02:57 +0530 Subject: [PATCH 16/16] handle empty comments --- .../services/youtube/extractors/YoutubeCommentsExtractor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 94beaf88..e0bbaa11 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -7,7 +7,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nonnull; @@ -210,7 +209,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } return result; } catch (Exception e2) { - throw new ParsingException("Could not get text", e2); + return ""; } } }