From 03893abd9134d457a47f16e12fde58d9d235b0fb Mon Sep 17 00:00:00 2001 From: Stypox Date: Fri, 26 Apr 2019 18:54:30 +0200 Subject: [PATCH 1/4] Fixed TeamNewPipe/NewPipe#2226. (in the youtube subscription extractor) Ignore subscriptions that have an empty title instead of throwing an error: the youtube subscription_manager XML file can sometimes contain those (i.e. deleted channels). --- .../youtube/extractors/YoutubeSubscriptionExtractor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java index f6b6b8bd..0fb9a020 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java @@ -63,7 +63,11 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor { String title = outline.attr("title"); String xmlUrl = outline.attr("abs:xmlUrl"); - if (title.isEmpty() || xmlUrl.isEmpty()) { + if (title.isEmpty()) { + continue; + } + + if (xmlUrl.isEmpty()) { throw new InvalidSourceException("document has invalid entries"); } From d5043cdf499db72138ef15e429d040bf3e12a574 Mon Sep 17 00:00:00 2001 From: Stypox Date: Fri, 26 Apr 2019 19:59:23 +0200 Subject: [PATCH 2/4] Add test for subscriptions with empty title. (youtube subscription extractor) --- .../YoutubeSubscriptionExtractorTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index 8d2e6cfa..8b23b129 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -59,6 +59,23 @@ public class YoutubeSubscriptionExtractorTest { assertTrue(items.isEmpty()); } + @Test + public void testSubscriptionWithEmptyTitleInSource() throws Exception { + String channelName = "NAME OF CHANNEL"; + String emptySource = "" + + + "" + + + "" + + + ""; + + List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8"))); + assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1); + assertTrue("Item does not have the right title \"" + channelName + "\" (had \"" + items.get(0).getName() + "\")", items.get(0).getName().equals(channelName)); + } + @Test public void testInvalidSourceException() { List invalidList = Arrays.asList( From 171f2c49fe61ce4c38b25797ca02d3b246660dd5 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 28 Apr 2019 14:17:52 +0200 Subject: [PATCH 3/4] Ignore subscriptions with invalid url and keep ones with empty title. if a channel if deleted (thus it has an empty title), it is imported in NewPipe anyway, so that if it becomes undeleted in the future, it will be shown in the app. --- .../extractors/YoutubeSubscriptionExtractor.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java index 0fb9a020..55a9c7c1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java @@ -63,20 +63,10 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor { String title = outline.attr("title"); String xmlUrl = outline.attr("abs:xmlUrl"); - if (title.isEmpty()) { - continue; - } - - if (xmlUrl.isEmpty()) { - throw new InvalidSourceException("document has invalid entries"); - } - try { String id = Parser.matchGroup1(ID_PATTERN, xmlUrl); result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title)); - } catch (Parser.RegexException e) { - throw new InvalidSourceException("document has invalid entries", e); - } + } catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ } } return result; From 0eaca52c15e480ef6b851d0e31600764f43442cf Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 28 Apr 2019 14:19:33 +0200 Subject: [PATCH 4/4] Add test for subscription with invalid url. Also modified the test for empty title, since now subscriptions with empty title are not ignored anymore. --- .../YoutubeSubscriptionExtractorTest.java | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index 8b23b129..c5739b85 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -61,19 +61,28 @@ public class YoutubeSubscriptionExtractorTest { @Test public void testSubscriptionWithEmptyTitleInSource() throws Exception { - String channelName = "NAME OF CHANNEL"; - String emptySource = "" + - - "" + - - "" + - + String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA"; + String source = "" + + "" + ""; - List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8"))); + List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8"))); assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1); - assertTrue("Item does not have the right title \"" + channelName + "\" (had \"" + items.get(0).getName() + "\")", items.get(0).getName().equals(channelName)); + assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty()); + assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId)); + } + + @Test + public void testSubscriptionWithInvalidUrlInSource() throws Exception { + String source = "" + + "" + + "" + + "" + + "" + + ""; + + List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8"))); + assertTrue(items.isEmpty()); } @Test @@ -82,9 +91,6 @@ public class YoutubeSubscriptionExtractorTest { "", "", "", - "", - "", "", null, "\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28", @@ -95,11 +101,11 @@ public class YoutubeSubscriptionExtractorTest { if (invalidContent != null) { byte[] bytes = invalidContent.getBytes("UTF-8"); subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); + fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); } else { subscriptionExtractor.fromInputStream(null); + fail("Extracting from null String didn't throw an exception"); } - - fail("didn't throw exception"); } catch (Exception e) { // System.out.println(" -> " + e); boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;