From 1ed89aad3eff72231bfec6c0d8d1fcf6b8c2c8e8 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 6 Jan 2020 20:45:57 +0100 Subject: [PATCH 01/40] Use more often playerResponse in yt stream extractor This enhances performance and should make the extractor more reliable since it get info from a stable json structure that shouldn't be subject to many changes. Fallback html methods have been kept. In case of error the thrown exception contains the data about the playerResponse failure, that should be clearer than a NPE caused by not-found html tags. --- .../extractors/YoutubeStreamExtractor.java | 179 ++++++++++-------- 1 file changed, 100 insertions(+), 79 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c05004ed..ef1edad3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -106,19 +106,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getName() throws ParsingException { assertPageFetched(); - String name = getStringFromMetaData("title"); - if(name == null) { - // Fallback to HTML method + try { + return playerResponse.getObject("videoDetails").getString("title"); + + } catch (Exception e) { + // fallback HTML method + String name = null; try { name = doc.select("meta[name=title]").attr(CONTENT); - } catch (Exception e) { - throw new ParsingException("Could not get the title", e); + } catch (Exception ignored) {} + + if (name == null) { + throw new ParsingException("Could not get name", e); } + return name; } - if(name == null || name.isEmpty()) { - throw new ParsingException("Could not get the title"); - } - return name; } @Override @@ -128,9 +130,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - return doc.select("meta[itemprop=datePublished]").attr(CONTENT); - } catch (Exception e) {//todo: add fallback method - throw new ParsingException("Could not get upload date", e); + return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); + } catch (Exception e) { + String uploadDate = null; + try { + uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT); + } catch (Exception ignored) {} + + if (uploadDate == null) { + throw new ParsingException("Could not get upload date", e); + } + return uploadDate; } } @@ -149,24 +159,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getThumbnailUrl() throws ParsingException { assertPageFetched(); - // Try to get high resolution thumbnail first, if it fails, use low res from the player instead try { - return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); - } catch (Exception ignored) { - // Try other method... - } + JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); + // the last thumbnail is the one with the highest resolution + return thumbnails.getObject(thumbnails.size()-1).getString("url"); - try { - if (playerArgs != null && playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url"); - } catch (Exception ignored) { - // Try other method... - } - - try { - return videoInfoPage.get("thumbnail_url"); } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); + String url = null; + try { + url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); + } catch (Exception ignored) {} + + if (url == null) { + throw new ParsingException("Could not get thumbnail url", e); + } + return url; } + } @Nonnull @@ -174,9 +183,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getDescription() throws ParsingException { assertPageFetched(); try { + // first try to get html-formatted description return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); } catch (Exception e) { - throw new ParsingException("Could not get the description", e); + try { + // fallback to raw non-html description + return playerResponse.getObject("videoDetails").getString("shortDescription"); + } catch (Exception ignored) { + throw new ParsingException("Could not get the description", e); + } } } @@ -269,25 +284,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { public long getLength() throws ParsingException { assertPageFetched(); - // try getting duration from playerargs - try { - String durationMs = playerResponse - .getObject("streamingData") - .getArray("formats") - .getObject(0) - .getString("approxDurationMs"); - return Long.parseLong(durationMs)/1000; - } catch (Exception e) { - } - - //try getting value from age gated video try { String duration = playerResponse .getObject("videoDetails") .getString("lengthSeconds"); return Long.parseLong(duration); } catch (Exception e) { - throw new ParsingException("Every methode to get the duration has failed: ", e); + try { + String durationMs = playerResponse + .getObject("streamingData") + .getArray("formats") + .getObject(0) + .getString("approxDurationMs"); + return Math.round(Long.parseLong(durationMs)/1000.0f); + } catch (Exception ignored) { + throw new ParsingException("Could not get duration", e); + } } } @@ -307,11 +319,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { if (getStreamType().equals(StreamType.LIVE_STREAM)) { return getLiveStreamWatchingCount(); + } else { + return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount")); + } + } catch (Exception e) { + try { + return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT)); + } catch (Exception ignored) { + throw new ParsingException("Could not get view count", e); } - - return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT)); - } catch (Exception e) {//todo: find fallback method - throw new ParsingException("Could not get number of views", e); } } @@ -373,7 +389,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { likesString = button.select("span.yt-uix-button-content").first().text(); } catch (NullPointerException e) { - //if this kicks in our button has no content and therefore likes/dislikes are disabled + //if this kicks in our button has no content and therefore ratings must be disabled + if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { + throw new ParsingException("Ratings are enabled even though the like button is missing", e); + } return -1; } return Integer.parseInt(Utils.removeNonDigitCharacters(likesString)); @@ -393,7 +412,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { dislikesString = button.select("span.yt-uix-button-content").first().text(); } catch (NullPointerException e) { - //if this kicks in our button has no content and therefore likes/dislikes are disabled + //if this kicks in our button has no content and therefore ratings must be disabled + if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { + throw new ParsingException("Ratings are enabled even though the dislike button is missing", e); + } return -1; } return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString)); @@ -409,60 +431,59 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { assertPageFetched(); try { - return doc.select("div[class=\"yt-user-info\"]").first().children() - .select("a").first().attr("abs:href"); + return "https://www.youtube.com/channel/" + + playerResponse.getObject("videoDetails").getString("channelId"); } catch (Exception e) { - throw new ParsingException("Could not get channel link", e); - } - } + String uploaderUrl = null; + try { + uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children() + .select("a").first().attr("abs:href"); + } catch (Exception ignored) {} - - @Nullable - private String getStringFromMetaData(String field) { - assertPageFetched(); - String value = null; - if(playerArgs != null) { - // This can not fail - value = playerArgs.getString(field); + if (uploaderUrl == null) { + throw new ParsingException("Could not get channel link", e); + } + return uploaderUrl; } - if(value == null) { - // This can not fail too - value = videoInfoPage.get(field); - } - return value; } @Nonnull @Override public String getUploaderName() throws ParsingException { assertPageFetched(); - String name = getStringFromMetaData("author"); - - if(name == null) { + try { + return playerResponse.getObject("videoDetails").getString("author"); + } catch (Exception e) { + String name = null; try { - // Fallback to HTML method name = doc.select("div.yt-user-info").first().text(); - } catch (Exception e) { - throw new ParsingException("Could not get uploader name", e); + } catch (Exception ignored) {} + + if (name == null) { + throw new ParsingException("Could not get uploader name"); } + return name; } - if(name == null || name.isEmpty()) { - throw new ParsingException("Could not get uploader name"); - } - return name; } @Nonnull @Override public String getUploaderAvatarUrl() throws ParsingException { assertPageFetched(); + + String uploaderAvatarUrl = null; try { - return doc.select("a[class*=\"yt-user-photo\"]").first() + uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first() .select("img").first() .attr("abs:data-thumb"); } catch (Exception e) {//todo: add fallback method - throw new ParsingException("Could not get uploader thumbnail URL.", e); + throw new ParsingException("Could not get uploader avatar url", e); } + + if (uploaderAvatarUrl == null) { + throw new ParsingException("Could not get uploader avatar url"); + } + return uploaderAvatarUrl; } @Nonnull @@ -590,12 +611,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { public StreamType getStreamType() throws ParsingException { assertPageFetched(); try { - if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") || - (!playerResponse.getObject("streamingData").has(FORMATS)))) { + if (!playerResponse.getObject("streamingData").has(FORMATS) || + (playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) { return StreamType.LIVE_STREAM; } } catch (Exception e) { - throw new ParsingException("Could not get hls manifest url", e); + throw new ParsingException("Could not get stream type", e); } return StreamType.VIDEO_STREAM; } From 1689037b0115d440bce5ee5e612ab646cafa496f Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 6 Jan 2020 20:49:08 +0100 Subject: [PATCH 02/40] Fix getLength() tests now that the value is rounded (not floor-ed) In yt stream extractor --- .../youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java | 2 +- .../youtube/stream/YoutubeStreamExtractorDefaultTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 992a8cd9..b9da6d17 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -75,7 +75,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { @Test public void testGetLength() throws ParsingException { - assertEquals(1789, extractor.getLength()); + assertEquals(1790, extractor.getLength()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index e715144a..14b78501 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -101,7 +101,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetLength() throws ParsingException { - assertEquals(366, extractor.getLength()); + assertEquals(367, extractor.getLength()); } @Test From d5ca02f3f69690b1bab065639db139747a4997b4 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 6 Jan 2020 21:41:15 +0100 Subject: [PATCH 03/40] Fix test failing because music channel could be Vevo In yt stream extractor test. The Vevo and the normal music channel are equivalent, so Youtube picks one of them at random, and in playerResponse the channel id is Vevo's one. --- .../youtube/stream/YoutubeStreamExtractorDefaultTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 14b78501..8dd35ce7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -124,7 +124,11 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetUploaderUrl() throws ParsingException { - assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl()); + String url = extractor.getUploaderUrl(); + if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") && + !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { + fail("Uploader url is neither the music channel one nor the Vevo one"); + } } @Test From 1bb6cdee225a8544bdc87311523f4abb859efcd6 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 6 Jan 2020 21:43:24 +0100 Subject: [PATCH 04/40] Enable commented-out test for yt stream extractor --- .../youtube/stream/YoutubeStreamExtractorControversialTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index 7faec805..5a063a93 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); -// assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().isEmpty()); } @Test From 7ba04836ebe15268e12c5055d0919d61a8dfe1f4 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 6 Jan 2020 23:57:08 +0100 Subject: [PATCH 05/40] Add tests for ratings in yt stream extractor tests --- .../YoutubeStreamExtractorDefaultTest.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 8dd35ce7..5f21df55 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -187,6 +187,18 @@ public class YoutubeStreamExtractorDefaultTest { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } + + @Test + public void testGetLikeCount() throws ParsingException { + long likeCount = extractor.getLikeCount(); + assertTrue("" + likeCount, likeCount >= 15000000); + } + + @Test + public void testGetDislikeCount() throws ParsingException { + long dislikeCount = extractor.getDislikeCount(); + assertTrue("" + dislikeCount, dislikeCount >= 818000); + } } public static class DescriptionTestPewdiepie { @@ -249,6 +261,29 @@ public class YoutubeStreamExtractorDefaultTest { } } + public static class RatingsDisabledTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=HRKu0cvrr_o"); + extractor.fetchPage(); + } + + @Test + public void testGetLikeCount() throws ParsingException { + assertEquals(-1, extractor.getLikeCount()); + } + + @Test + public void testGetDislikeCount() throws ParsingException { + assertEquals(-1, extractor.getDislikeCount()); + } + + } + public static class FramesTest { private static YoutubeStreamExtractor extractor; From a281519e771d139e7d6cd1b7f64dece92cbb5124 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 19 Jan 2020 12:45:52 +0100 Subject: [PATCH 06/40] added metadata, fix descriptions, fix thumbnail, update tests thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails) test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted) metadata: got host, privacy (public, private, unlisted), licence, language, tags --- README.md | 2 +- .../extractors/MediaCCCStreamExtractor.java | 32 +++++++ .../extractors/PeertubeStreamExtractor.java | 50 +++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 31 +++++++ .../extractors/YoutubeStreamExtractor.java | 31 +++++++ .../extractor/stream/StreamExtractor.java | 57 ++++++++++++ .../newpipe/extractor/stream/StreamInfo.java | 88 +++++++++++++++++++ .../schabi/newpipe/extractor/utils/Utils.java | 1 - .../PeertubeCommentsExtractorTest.java | 4 +- .../PeertubeStreamExtractorDefaultTest.java | 21 ++--- 10 files changed, 295 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f8c9c8cf..66515055 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`) 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`) - + > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project. ## Supported sites diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index a1dfd245..84fb7685 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import java.io.IOException; @@ -225,4 +226,35 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public String getOriginalUrl() throws ParsingException { return data.getString("frontend_link"); } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index d8d0de00..0bead848 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -9,7 +9,7 @@ import java.util.List; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -66,16 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(json, "thumbnailPath"); + return baseUrl + JsonUtils.getString(json, "previewPath"); } @Override public String getDescription() throws ParsingException { + String description = ""; + Downloader dl = NewPipe.getDownloader(); try { - return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { - return "No description"; + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + description = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); } + if (description.equals("")) { + //if the request above failed + description = JsonUtils.getString(json, "description"); + } + return description; } @Override @@ -224,8 +232,9 @@ public class PeertubeStreamExtractor extends StreamExtractor { if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } - - private List getTags(){ + + @Override + public List getTags(){ try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -339,4 +348,29 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + "/videos/watch/" + getId(); } + //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it + @Override + public String getHost() throws ParsingException { + return JsonUtils.getString(json, "account.host"); + } + + @Override + public String getPrivacy() throws ParsingException { + return JsonUtils.getString(json, "privacy.label"); + } + + @Override + public String getCategory() throws ParsingException { + return JsonUtils.getString(json, "category.label"); + } + + @Override + public String getLicence() throws ParsingException { + return JsonUtils.getString(json, "licence.label"); + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return JsonUtils.getString(json, "language.label"); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 14f7023f..19ab987c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -254,4 +254,35 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public String getErrorMessage() { return null; } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c05004ed..0e380fbc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1113,4 +1113,35 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ExtractionException(e); } } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index d697dff2..ce18dfa3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; @@ -349,4 +350,60 @@ public abstract class StreamExtractor extends Extractor { return 0; } } + + /** + * The host of the stream (Eg. peertube.cpy.re). + * If the privacy is not available, or if the service doesn't use + * a federated system, but a centralised system, + * you can simply return an empty string. + * @return the host of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getHost() throws ParsingException; + + /** + * The privacy of the stream (Eg. Public, Private, Unlisted…). + * If the privacy is not available you can simply return an empty string. + * @return the privacy of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getPrivacy() throws ParsingException; + + /** + * The name of the category of the stream. + * If the category is not available you can simply return an empty string. + * @return the category of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getCategory() throws ParsingException; + + /** + * The name of the licence of the stream. + * If the licence is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getLicence() throws ParsingException; + + /** + * The language of the stream. + * If the language is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getStreamInfoLanguage() throws ParsingException; + + /** + * The list of tags of the stream. + * If the tag list is not available you can simply return an empty list. + * @return the list of tags of the stream or an empty list. + * @throws ParsingException + */ + @Nonnull + public abstract List getTags() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 34ea703c..111f387d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -270,6 +270,38 @@ public class StreamInfo extends Info { streamInfo.addError(e); } + //additional info + try { + streamInfo.setHost(extractor.getHost()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setPrivacy(extractor.getPrivacy()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setCategory(extractor.getCategory()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLicence(extractor.getLicence()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setTags(extractor.getTags()); + } catch (Exception e) { + streamInfo.addError(e); + } + streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -308,6 +340,13 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles = new ArrayList<>(); + private String host = ""; + private String privacy = ""; + private String category = ""; + private String licence = ""; + private String language = ""; + private List tags = new ArrayList<>(); + /** * Get the stream type * @@ -533,4 +572,53 @@ public class StreamInfo extends Info { this.subtitles = subtitles; } + public String getHost() { + return this.host; + } + + public void setHost(String str) { + this.host = str; + } + + public String getPrivacy() { + return this.privacy; + } + + public void setPrivacy(String str) { + this.privacy = str; + } + + public String getCategory() { + return this.category; + } + + public void setCategory(String cat) { + this.category = cat; + } + + public String getLicence() { + return this.licence; + } + + public void setLicence(String str) { + this.licence = str; + } + + public String getLanguage() { + return this.language; + } + + public void setLanguage(String lang) { + this.language = lang; + } + + public List getTags() { + return this.tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index fd06d40f..3489b6d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -185,5 +185,4 @@ public class Utils { } return uri.getProtocol() + "://" + uri.getAuthority(); } - } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index fef1c3ee..e0a1d189 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (PeertubeCommentsExtractor) PeerTube - .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); } @Test @@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); assertTrue("Comments".equals(commentsInfo.getName())); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 75a692d4..b2175e85 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,13 +29,14 @@ import org.schabi.newpipe.extractor.stream.StreamType; */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; + private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); + PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d"); extractor.fetchPage(); } @@ -47,22 +48,22 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "Power Corrupts the Best"); + assertEquals("What is PeerTube?", extractor.getName()); } @Test public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); + assertEquals(expectedDescription, extractor.getDescription()); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "Rowsedower"); + assertEquals("Framasoft", extractor.getUploaderName()); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 269); + assertEquals(113, extractor.getLength()); } @Test @@ -74,7 +75,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } @@ -82,7 +83,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); + assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); } @Test @@ -115,11 +116,11 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } @Test public void testGetSubtitlesList() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } } From b382416372add1c9420cf1657f67c9cae825751f Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 14:36:12 +0100 Subject: [PATCH 07/40] changed the way to extract peertube description if the description length is above 254, and ends with ..., it means the description we got from the first request is shortened. why above 254: because in fact, shortened descriptions length are between 255 : https://framatube.org/videos/watch/24efbe1b-73c0-4d72-a3ff-77c8b32d3fcd https://framatube.org/videos/watch/1ca99f40-fb5b-4fa4-abe5-4d54325df7fc and 269: https://framatube.org/videos/watch/4d661d5f-a884-4510-bca8-15cb19aa3fe5 also fixed a typo in StreamExtractor.java --- .../extractors/PeertubeStreamExtractor.java | 25 +++++++++++-------- .../extractor/stream/StreamExtractor.java | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0bead848..ac3d6ecb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -70,20 +70,23 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getDescription() throws ParsingException { - String description = ""; - Downloader dl = NewPipe.getDownloader(); + String desc; try { - Response response = dl.get(getUrl() + "/description"); - JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - description = JsonUtils.getString(jsonObject, "description"); - } catch (ReCaptchaException | IOException | JsonParserException e) { - e.printStackTrace(); + desc = JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return "No description"; } - if (description.equals("")) { - //if the request above failed - description = JsonUtils.getString(json, "description"); + if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + Downloader dl = NewPipe.getDownloader(); + try { + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + desc = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); + } } - return description; + return desc; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index ce18dfa3..a638c7b6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -353,7 +353,7 @@ public abstract class StreamExtractor extends Extractor { /** * The host of the stream (Eg. peertube.cpy.re). - * If the privacy is not available, or if the service doesn't use + * If the host is not available, or if the service doesn't use * a federated system, but a centralised system, * you can simply return an empty string. * @return the host of the stream or an empty String. From ad7f97ae836f89a0f555e1062e5cc3b5211b1dc3 Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 15:14:23 +0100 Subject: [PATCH 08/40] fix PeerTube description and add more description tests actually, the max description length is 250 after request with our extractor. during my tests, I made API requests with Firefox, copy/pasted into echo "insert description" | wc, and it was giving a wrong length, maybe due to the escapers, I have no idea anyway, it's now fixed --- .../extractors/PeertubeStreamExtractor.java | 3 ++- .../PeertubeStreamExtractorDefaultTest.java | 22 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index ac3d6ecb..c922b2fe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (ParsingException e) { return "No description"; } - if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { Response response = dl.get(getUrl() + "/description"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index b2175e85..2a4378a2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,7 +29,8 @@ import org.schabi.newpipe.extractor.stream.StreamType; */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; - private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary"; @BeforeClass public static void setUp() throws Exception { @@ -52,8 +53,23 @@ public class PeertubeStreamExtractorDefaultTest { } @Test - public void testGetDescription() throws ParsingException { - assertEquals(expectedDescription, extractor.getDescription()); + public void testGetLargeDescription() throws ParsingException { + assertEquals(expectedLargeDescription, extractor.getDescription()); + } + + @Test + public void testGetEmptyDescription() throws Exception { + PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); + extractorEmpty.fetchPage(); + assertEquals("No description", extractorEmpty.getDescription()); + } + + @Test + public void testGetSmallDescription() throws Exception { + PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); + PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); + extractorSmall.fetchPage(); + assertEquals(expectedSmallDescription, extractorSmall.getDescription()); } @Test From 39de55dcd3ee32b95d476feec3b322759205c0e4 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 20 Jan 2020 21:24:51 +0100 Subject: [PATCH 09/40] [YouTube] Fix and move streamCountTest --- .../search/YoutubeSearchCountTest.java | 36 ------------------- ...YoutubeSearchExtractorChannelOnlyTest.java | 11 ++++-- 2 files changed, 9 insertions(+), 38 deletions(-) delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java deleted file mode 100644 index c9568045..00000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfoItem; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; - -import static java.util.Collections.singletonList; -import static junit.framework.TestCase.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -/** - * Test for {@link YoutubeSearchExtractor} - */ -public class YoutubeSearchCountTest { - public static class YoutubeChannelViewCountTest extends YoutubeSearchExtractorBaseTest { - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - singletonList(YoutubeSearchQueryHandlerFactory.CHANNELS), null); - extractor.fetchPage(); - itemsPage = extractor.getInitialPage(); - } - - @Test - public void testViewCount() { - ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); - assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()), - 69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 103043316); - } - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 12e76a08..91cbebd9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -39,7 +39,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } @@ -58,7 +58,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto @Test public void testOnlyContainChannels() { for(InfoItem item : itemsPage.getItems()) { - if(!(item instanceof ChannelInfoItem)) { + if (!(item instanceof ChannelInfoItem)) { fail("The following item is no channel item: " + item.toString()); } } @@ -78,4 +78,11 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } } } + + @Test + public void testStreamCount() { + ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); + assertTrue("Stream count does not fit: " + ci.getStreamCount(), + 4000 < ci.getStreamCount() && ci.getStreamCount() < 5500); + } } From 79431303079643c49f88a40b92584ce61a50c8ab Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 20 Jan 2020 21:25:16 +0100 Subject: [PATCH 10/40] Throw IllegalArgumentException when url is null in LinkHandlerFactory --- .../extractor/linkhandler/LinkHandlerFactory.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index b9e60ce0..3890c2b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -34,7 +34,7 @@ public abstract class LinkHandlerFactory { public abstract String getUrl(String id) throws ParsingException; public abstract boolean onAcceptUrl(final String url) throws ParsingException; - public String getUrl(String id, String baseUrl) throws ParsingException{ + public String getUrl(String id, String baseUrl) throws ParsingException { return getUrl(id); } @@ -43,13 +43,14 @@ public abstract class LinkHandlerFactory { /////////////////////////////////// public LinkHandler fromUrl(String url) throws ParsingException { + if (url == null) throw new IllegalArgumentException("url can not be null"); final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { - if(url == null) throw new IllegalArgumentException("url can not be null"); - if(!acceptUrl(url)) { + if (url == null) throw new IllegalArgumentException("url can not be null"); + if (!acceptUrl(url)) { throw new ParsingException("Malformed unacceptable url: " + url); } @@ -58,13 +59,13 @@ public abstract class LinkHandlerFactory { } public LinkHandler fromId(String id) throws ParsingException { - if(id == null) throw new IllegalArgumentException("id can not be null"); + if (id == null) throw new IllegalArgumentException("id can not be null"); final String url = getUrl(id); return new LinkHandler(url, url, id); } public LinkHandler fromId(String id, String baseUrl) throws ParsingException { - if(id == null) throw new IllegalArgumentException("id can not be null"); + if (id == null) throw new IllegalArgumentException("id can not be null"); final String url = getUrl(id, baseUrl); return new LinkHandler(url, url, id); } From 69ccb7e69dd0d40840d1dba671d147c738f10b95 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 20 Jan 2020 21:25:47 +0100 Subject: [PATCH 11/40] Update subtitle tests comments --- .../stream/YoutubeStreamExtractorControversialTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index 7faec805..4adcccbd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -122,13 +122,13 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions assertFalse(extractor.getSubtitlesDefault().isEmpty()); } @Test public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } From f403490bd9ada2b19176ffc694d5b5ad405975b6 Mon Sep 17 00:00:00 2001 From: B0pol Date: Mon, 20 Jan 2020 21:28:26 +0100 Subject: [PATCH 12/40] Refactoring Co-Authored-By: Tobias Groza --- .../services/peertube/extractors/PeertubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index c922b2fe..0de72a94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (ParsingException e) { return "No description"; } - if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { From 089dbe8afb6ba9beb21aecce423d12601813e611 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 20 Jan 2020 22:10:18 +0100 Subject: [PATCH 13/40] Fix StreamLinkHandlerFactoryTest by passing baseUrl to vdn.youtube urls --- .../services/youtube/YoutubeStreamLinkHandlerFactoryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index d3f463f3..796238f9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -77,8 +77,8 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", "youtube.com").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI", "youtube.com").getId()); assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); } From 2308b074f0603ec65c1b2c2a5ac0f3285cb3a0e6 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 20 Jan 2020 22:12:39 +0100 Subject: [PATCH 14/40] [MediaCCC] Fix testGetTextualUploadDate() --- .../services/media_ccc/MediaCCCStreamExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index befd840f..71b1e779 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -89,7 +89,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest { @Test public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2018-05-11", extractor.getTextualUploadDate()); + Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); } @Test From 1e1100ef76c2a9cb917759374bc0bc2dddb53c47 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 20 Jan 2020 22:52:36 +0100 Subject: [PATCH 15/40] Update extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java Co-Authored-By: Tobias Groza --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ef1edad3..45874906 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -162,7 +162,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - return thumbnails.getObject(thumbnails.size()-1).getString("url"); + return thumbnails.getObject(thumbnails.size() - 1).getString("url"); } catch (Exception e) { String url = null; From 1cfdc4547a41a70caa6b368b7267b68d04531e53 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 20 Jan 2020 22:52:48 +0100 Subject: [PATCH 16/40] Update extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java Co-Authored-By: Tobias Groza --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 45874906..d0e0ff79 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -296,7 +296,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getArray("formats") .getObject(0) .getString("approxDurationMs"); - return Math.round(Long.parseLong(durationMs)/1000.0f); + return Math.round(Long.parseLong(durationMs) / 1000f); } catch (Exception ignored) { throw new ParsingException("Could not get duration", e); } From 1a15c0e750dc95fa0a92cc1c988e781e66631fa7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Thu, 23 Jan 2020 04:42:54 +0100 Subject: [PATCH 17/40] agelimit now returns 18 if the video is marked as nsfw, 0 otherwise + created getBoolean method in JsonUtils.java --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../org/schabi/newpipe/extractor/utils/JsonUtils.java | 10 ++++++++++ .../peertube/PeertubeStreamExtractorDefaultTest.java | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0de72a94..b841d9fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -92,7 +92,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() throws ParsingException { - return NO_AGE_LIMIT; + boolean isNSFW = JsonUtils.getBoolean(json, "nsfw"); + if (isNSFW) { + return 18; + } else { + return NO_AGE_LIMIT; + } } @Override @@ -352,7 +357,6 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + "/videos/watch/" + getId(); } - //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it @Override public String getHost() throws ParsingException { return JsonUtils.getString(json, "account.host"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index e5d7bb62..ebd251d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -37,6 +37,16 @@ public class JsonUtils { throw new ParsingException("Unable to get " + path); } } + + @Nonnull + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof Boolean) { + return (Boolean) value; + }else { + throw new ParsingException("Unable to get " + path); + } + } @Nonnull public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 2a4378a2..4166b192 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -139,4 +139,12 @@ public class PeertubeStreamExtractorDefaultTest { public void testGetSubtitlesList() throws IOException, ExtractionException { assertFalse(extractor.getSubtitlesDefault().isEmpty()); } + + @Test + public void testGetAgeLimit() throws ExtractionException, IOException { + assertEquals(0, extractor.getAgeLimit()); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7"); + ageLimit.fetchPage(); + assertEquals(18, ageLimit.getAgeLimit()); + } } From 74439f692a234d89b5556e1949192f3f9681f840 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:19:22 +0100 Subject: [PATCH 18/40] add extraction for support info and rename getLanguageInfo function --- .../extractors/MediaCCCStreamExtractor.java | 8 +++++++- .../extractors/PeertubeStreamExtractor.java | 12 ++++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 8 +++++++- .../extractors/YoutubeStreamExtractor.java | 8 +++++++- .../extractor/stream/StreamExtractor.java | 14 +++++++++++++- .../newpipe/extractor/stream/StreamInfo.java | 18 +++++++++++++++--- .../PeertubeStreamExtractorDefaultTest.java | 7 +++++++ 7 files changed, 66 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 84fb7685..9168d6e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -248,7 +248,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -257,4 +257,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index b841d9fd..4dadae77 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -35,6 +35,8 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import javax.annotation.Nonnull; + public class PeertubeStreamExtractor extends StreamExtractor { @@ -250,7 +252,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { return Collections.emptyList(); } } - + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return JsonUtils.getString(json, "support"); + } + private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); @@ -378,7 +386,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return JsonUtils.getString(json, "language.label"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 19ab987c..c5d34bd0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -276,7 +276,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -285,4 +285,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ea9a84d4..74a44a66 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1156,7 +1156,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -1165,4 +1165,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index a638c7b6..31a5a583 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -396,7 +396,7 @@ public abstract class StreamExtractor extends Extractor { * @throws ParsingException */ @Nonnull - public abstract String getStreamInfoLanguage() throws ParsingException; + public abstract String getLanguageInfo() throws ParsingException; /** * The list of tags of the stream. @@ -406,4 +406,16 @@ public abstract class StreamExtractor extends Extractor { */ @Nonnull public abstract List getTags() throws ParsingException; + + /** + * The support information of the stream. + * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37 + * (support button). + * If the support information are not available, + * you can simply return an empty String. + * @return the support information of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getSupportInfo() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 111f387d..e32a133d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -292,7 +292,7 @@ public class StreamInfo extends Info { streamInfo.addError(e); } try { - streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + streamInfo.setLanguageInfo(extractor.getLanguageInfo()); } catch (Exception e) { streamInfo.addError(e); } @@ -301,6 +301,11 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setSupportInfo(extractor.getSupportInfo()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); @@ -345,6 +350,7 @@ public class StreamInfo extends Info { private String category = ""; private String licence = ""; private String language = ""; + private String support = ""; private List tags = new ArrayList<>(); /** @@ -604,11 +610,11 @@ public class StreamInfo extends Info { this.licence = str; } - public String getLanguage() { + public String getLanguageInfo() { return this.language; } - public void setLanguage(String lang) { + public void setLanguageInfo(String lang) { this.language = lang; } @@ -620,5 +626,11 @@ public class StreamInfo extends Info { this.tags = tags; } + public void setSupportInfo(String support) { + this.support = support; + } + public String getSupportInfo() { + return this.support; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 4166b192..82967852 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -147,4 +147,11 @@ public class PeertubeStreamExtractorDefaultTest { ageLimit.fetchPage(); assertEquals(18, ageLimit.getAgeLimit()); } + + @Test + public void testGetSupportInformation() throws ExtractionException, IOException { + PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37"); + supportInfoExtractor.fetchPage(); + assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); + } } From 20da4750f8f779e7fd8efaf9471b890cd47c75a7 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:37:14 +0100 Subject: [PATCH 19/40] empty support returns "", same for empty description --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 4dadae77..a57720e3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { try { desc = JsonUtils.getString(json, "description"); } catch (ParsingException e) { - return "No description"; + return ""; } if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description @@ -256,7 +256,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getSupportInfo() throws ParsingException { - return JsonUtils.getString(json, "support"); + try { + return JsonUtils.getString(json, "support"); + } catch (ParsingException e) { + return ""; + } } private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 82967852..96273506 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -61,7 +61,7 @@ public class PeertubeStreamExtractorDefaultTest { public void testGetEmptyDescription() throws Exception { PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); extractorEmpty.fetchPage(); - assertEquals("No description", extractorEmpty.getDescription()); + assertEquals("", extractorEmpty.getDescription()); } @Test From a691d6d0fc4949ab1392082865416e83a04af341 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:02:40 +0100 Subject: [PATCH 20/40] fix upload date: there was a one hour delay on peertube.co.uk: https://i.imgur.com/8Pai1rb.png on newpipe, before this commit: https://i.imgur.com/NIRbs4Z.png on newpipe, with this commit: https://i.ibb.co/mhKKtRD/Screenshot-20200123-185422-New-Pipe-Debug.png --- .../extractor/services/peertube/PeertubeParsingHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index f518faf9..2483dd94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,6 +4,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.concurrent.TimeUnit; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -27,6 +28,7 @@ public class PeertubeParsingHelper { Date date; try { date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); + date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1)); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } From c790261eed4ba148b603325075bcbbfbd3e9b4c9 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:08:41 +0100 Subject: [PATCH 21/40] update test --- .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 96273506..c6a164f7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -91,7 +91,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T11:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } From 812c8e0ad25de2f1d3f5da16975cae1b145072f1 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 00:22:05 +0100 Subject: [PATCH 22/40] authorName in comments now follow PeerTube website MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea anonyme zirbeldrüse is his displayName, what was displayed in NewPipe. but on the website, it's shown 777@mastodon.xyz --- .../extractors/PeertubeCommentsInfoItemExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index edb59e1f..0bcce9fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -67,7 +67,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac try { Document doc = Jsoup.parse(htmlText); return doc.body().text(); - }catch(Exception e) { + } catch(Exception e) { return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); } } @@ -83,7 +83,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -91,7 +91,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public String getAuthorName() throws ParsingException { - return JsonUtils.getString(item, "account.displayName"); + return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); } @Override From 341372c0d0dadcce1b88333d9c8afc58ec50fff7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 20:16:24 +0100 Subject: [PATCH 23/40] reindenting (ctrl alt l) on JsonUtils and PeertubeStreamExtractor --- .../extractors/PeertubeStreamExtractor.java | 75 ++++++++++--------- .../newpipe/extractor/utils/JsonUtils.java | 46 ++++++------ 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index a57720e3..e6e32d44 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -39,16 +39,16 @@ import javax.annotation.Nonnull; public class PeertubeStreamExtractor extends StreamExtractor { - + + private final String baseUrl; private JsonObject json; private List subtitles = new ArrayList<>(); - private final String baseUrl; - + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); this.baseUrl = getBaseUrl(); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(json, "publishedAt"); @@ -64,7 +64,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public String getThumbnailUrl() throws ParsingException { return baseUrl + JsonUtils.getString(json, "previewPath"); @@ -149,7 +149,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { String value; try { value = JsonUtils.getString(json, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -176,8 +176,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { List videoStreams = new ArrayList<>(); try { JsonArray streams = json.getArray("files", new JsonArray()); - for(Object s: streams) { - if(!(s instanceof JsonObject)) continue; + for (Object s : streams) { + if (!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; String url = JsonUtils.getString(stream, "fileUrl"); String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); @@ -211,8 +211,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { List filteredSubs = new ArrayList<>(); - for(SubtitlesStream sub: subtitles) { - if(sub.getFormat() == format) { + for (SubtitlesStream sub : subtitles) { + if (sub.getFormat() == format) { filteredSubs.add(sub); } } @@ -234,18 +234,18 @@ public class PeertubeStreamExtractor extends StreamExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); List tags = getTags(); String apiUrl = null; - if(!tags.isEmpty()) { + if (!tags.isEmpty()) { apiUrl = getRelatedStreamsUrl(tags); - - }else { + + } else { apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; } - if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); + if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } @Override - public List getTags(){ + public List getTags() { try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -267,7 +267,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); - for(String tag : tags) { + for (String tag : tags) { params.append("&tagsOneOf="); params.append(URLEncoder.encode(tag, "UTF-8")); } @@ -277,38 +277,38 @@ public class PeertubeStreamExtractor extends StreamExtractor { private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { Response response = getDownloader().get(apiUrl); JsonObject relatedVideosJson = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { relatedVideosJson = JsonParser.object().from(response.responseBody()); } catch (JsonParserException e) { throw new ParsingException("Could not parse json data for related videos", e); } } - - if(relatedVideosJson != null) { + + if (relatedVideosJson != null) { collectStreamsFrom(collector, relatedVideosJson); } } - + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract related videos", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); //do not add the same stream in related streams - if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); + if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor); } } - + } - + @Override public String getErrorMessage() { @@ -318,12 +318,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { + } else { throw new ExtractionException("Unable to extract peertube channel data"); } - + loadSubtitles(); } @@ -333,24 +333,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube stream data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + if (null == json) throw new ExtractionException("Unable to extract peertube stream data"); PeertubeParsingHelper.validate(json); } - + private void loadSubtitles() { if (subtitles.isEmpty()) { try { - Response response = getDownloader().get(getUrl() + "/captions"); + Response response = getDownloader().get(getUrl() + "/captions"); JsonObject captionsJson = JsonParser.object().from(response.responseBody()); JsonArray captions = JsonUtils.getArray(captionsJson, "data"); - for(Object c: captions) { - if(c instanceof JsonObject) { - JsonObject caption = (JsonObject)c; + for (Object c : captions) { + if (c instanceof JsonObject) { + JsonObject caption = (JsonObject) c; String url = baseUrl + JsonUtils.getString(caption, "captionPath"); String languageCode = JsonUtils.getString(caption, "language.id"); String ext = url.substring(url.lastIndexOf(".") + 1); MediaFormat fmt = MediaFormat.getFromSuffix(ext); - if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + if (fmt != null && languageCode != null) + subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); } } } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ebd251d7..80e0dd58 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -16,71 +16,71 @@ public class JsonUtils { private JsonUtils() { } - + @Nonnull - public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { List keys = Arrays.asList(path.split("\\.")); object = getObject(object, keys.subList(0, keys.size() - 1)); if (null == object) throw new ParsingException("Unable to get " + path); Object result = object.get(keys.get(keys.size() - 1)); - if(null == result) throw new ParsingException("Unable to get " + path); + if (null == result) throw new ParsingException("Unable to get " + path); return result; } - + @Nonnull - public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof String) { + if (value instanceof String) { return (String) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull - public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Boolean) { + if (value instanceof Boolean) { return (Boolean) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Number) { + if (value instanceof Number) { return (Number) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonObject) { + if (value instanceof JsonObject) { return (JsonObject) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonArray) { + if (value instanceof JsonArray) { return (JsonArray) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { - + List result = new ArrayList<>(); for (int i = 0; i < array.size(); i++) { JsonObject obj = array.getObject(i); From e392b6c68f247e954a2e4eac5c097a4d809d3e1f Mon Sep 17 00:00:00 2001 From: B0pol Date: Sat, 25 Jan 2020 13:16:42 +0100 Subject: [PATCH 24/40] getLanguageInfo returns Locale instead of String so that java can automatically translate with Locale.getDisplayLanguage(), instead of always having English name of the language --- .../extractors/MediaCCCStreamExtractor.java | 5 +++-- .../extractors/PeertubeStreamExtractor.java | 9 +++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 5 +++-- .../youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- .../newpipe/extractor/stream/StreamExtractor.java | 13 ++++++++----- .../schabi/newpipe/extractor/stream/StreamInfo.java | 7 ++++--- .../PeertubeStreamExtractorDefaultTest.java | 6 ++++++ 7 files changed, 33 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 9168d6e6..07c36d12 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -18,6 +18,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; public class MediaCCCStreamExtractor extends StreamExtractor { @@ -248,8 +249,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index e6e32d44..80cae88a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -6,6 +6,7 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; @@ -391,7 +392,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return JsonUtils.getString(json, "language.label"); + public Locale getLanguageInfo() throws ParsingException { + try { + return new Locale(JsonUtils.getString(json, "language.id")); + } catch (ParsingException e) { + return null; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index c5d34bd0..5b1153d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -20,6 +20,7 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -276,8 +277,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index dcf302cb..10ffb143 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1157,8 +1157,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 31a5a583..652e0116 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -35,6 +35,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Locale; /** * Scrapes information from a video/audio streaming service (eg, YouTube). @@ -390,13 +391,15 @@ public abstract class StreamExtractor extends Extractor { public abstract String getLicence() throws ParsingException; /** - * The language of the stream. - * If the language is not available you can simply return an empty string. - * @return the licence of the stream or an empty String. + * The locale language of the stream. + * If the language is not available you can simply return null. + * If the language is provided by a language code, you can return + * new Locale(language_code); + * @return the locale language of the stream or null. * @throws ParsingException */ - @Nonnull - public abstract String getLanguageInfo() throws ParsingException; + @Nullable + public abstract Locale getLanguageInfo() throws ParsingException; /** * The list of tags of the stream. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index e32a133d..64999ce7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; /* * Created by Christian Schabesberger on 26.08.15. @@ -349,8 +350,8 @@ public class StreamInfo extends Info { private String privacy = ""; private String category = ""; private String licence = ""; - private String language = ""; private String support = ""; + private Locale language = null; private List tags = new ArrayList<>(); /** @@ -610,11 +611,11 @@ public class StreamInfo extends Info { this.licence = str; } - public String getLanguageInfo() { + public Locale getLanguageInfo() { return this.language; } - public void setLanguageInfo(String lang) { + public void setLanguageInfo(Locale lang) { this.language = lang; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index c6a164f7..06a8352c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Locale; import org.junit.BeforeClass; import org.junit.Ignore; @@ -154,4 +155,9 @@ public class PeertubeStreamExtractorDefaultTest { supportInfoExtractor.fetchPage(); assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); } + + @Test + public void testGetLanguageInformation() throws ParsingException { + assertEquals(new Locale("en"), extractor.getLanguageInfo()); + } } From 033a9fb8e388e50a93cd5c21de44a832a975048b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 25 Jan 2020 21:08:17 +0100 Subject: [PATCH 25/40] Revert change which I accidentally committed while fixing the decryption function in 7b72fd2a7d3632f3de0c57d525664c4c26c9af19 --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 9ce9d6ab..db167e30 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -720,8 +720,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String playerUrl; // Check if the video is age restricted - Elements e = doc.select("meta[property=\"og:restrictions:age\"]"); - if (!e.isEmpty()) { + if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) { final EmbeddedInfo info = getEmbeddedInfo(); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); From 5c0c35064c8e64d0feb93c1afc0492e883a680f9 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 16 Dec 2019 04:35:42 -0300 Subject: [PATCH 26/40] Use clean url and id when creating a channel info --- .../extractor/channel/ChannelInfo.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index aaa99c7b..0fc44063 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -5,9 +5,7 @@ import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.ExtractorHelper; @@ -35,8 +33,8 @@ import java.io.IOException; public class ChannelInfo extends ListInfo { - public ChannelInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException { - super(serviceId, linkHandler, name); + public ChannelInfo(int serviceId, String id, String url, String originalUrl, String name, ListLinkHandler listLinkHandler) { + super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(), listLinkHandler.getSortFilter()); } public static ChannelInfo getInfo(String url) throws IOException, ExtractionException { @@ -57,15 +55,14 @@ public class ChannelInfo extends ListInfo { public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException { - ChannelInfo info = new ChannelInfo(extractor.getServiceId(), - extractor.getLinkHandler(), - extractor.getName()); + final int serviceId = extractor.getServiceId(); + final String id = extractor.getId(); + final String url = extractor.getUrl(); + final String originalUrl = extractor.getOriginalUrl(); + final String name = extractor.getName(); + + final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler()); - try { - info.setOriginalUrl(extractor.getOriginalUrl()); - } catch (Exception e) { - info.addError(e); - } try { info.setAvatarUrl(extractor.getAvatarUrl()); } catch (Exception e) { From be81f2945cd937467dbdfa1891e585cdd7b1b75e Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 16 Dec 2019 04:35:43 -0300 Subject: [PATCH 27/40] [YouTube] Use correct upload date flag in StreamExtractor --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index db167e30..cee1a4b9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -22,8 +22,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.*; @@ -152,7 +152,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return null; } - return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate)); + return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate), true); } @Nonnull From 26234a1c0fa7cf0a03a47a38948f0ee4f0df29a7 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 16 Dec 2019 04:35:44 -0300 Subject: [PATCH 28/40] Introduce FeedExtractor making fetching from dedicated feeds possible YouTube, for example, has a dedicated feed which was built to be used like this. --- .../newpipe/extractor/StreamingService.java | 22 ++++- .../newpipe/extractor/feed/FeedExtractor.java | 17 ++++ .../newpipe/extractor/feed/FeedInfo.java | 52 ++++++++++ .../services/youtube/YoutubeService.java | 20 ++-- .../extractors/YoutubeChannelExtractor.java | 3 +- .../extractors/YoutubeFeedExtractor.java | 82 ++++++++++++++++ .../YoutubeFeedInfoItemExtractor.java | 94 +++++++++++++++++++ .../linkHandler/YoutubeParsingHelper.java | 13 +++ .../youtube/YoutubeFeedExtractorTest.java | 72 ++++++++++++++ 9 files changed, 361 insertions(+), 14 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index ad22642d..68398506 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; @@ -24,6 +25,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import javax.annotation.Nullable; + /* * Copyright (C) Christian Schabesberger 2018 * StreamingService.java is part of NewPipe. @@ -65,7 +68,7 @@ public abstract class StreamingService { public String getName() { return name; } - + public List getMediaCapabilities() { return mediaCapabilities; } @@ -116,7 +119,7 @@ public abstract class StreamingService { public String toString() { return serviceId + ":" + serviceInfo.getName(); } - + public abstract String getBaseUrl(); /*////////////////////////////////////////////////////////////////////////// @@ -173,6 +176,19 @@ public abstract class StreamingService { */ public abstract SubscriptionExtractor getSubscriptionExtractor(); + /** + * This method decides which strategy will be chosen to fetch the feed. In YouTube, for example, a separate feed + * exists which is lightweight and made specifically to be used like this. + *

+ * In services which there's no other way to retrieve them, null should be returned. + * + * @return a {@link FeedExtractor} instance or null. + */ + @Nullable + public FeedExtractor getFeedExtractor(String url) throws ExtractionException { + return null; + } + /** * Must create a new instance of a KioskList implementation. * @return a new KioskList instance @@ -258,7 +274,7 @@ public abstract class StreamingService { } return getCommentsExtractor(llhf.fromUrl(url)); } - + /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedExtractor.java new file mode 100644 index 00000000..df3e8915 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedExtractor.java @@ -0,0 +1,17 @@ +package org.schabi.newpipe.extractor.feed; + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +/** + * This class helps to extract items from lightweight feeds that the services may provide. + *

+ * YouTube is an example of a service that has this alternative available. + */ +public abstract class FeedExtractor extends ListExtractor { + public FeedExtractor(StreamingService service, ListLinkHandler listLinkHandler) { + super(service, listLinkHandler); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java new file mode 100644 index 00000000..f361cec7 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java @@ -0,0 +1,52 @@ +package org.schabi.newpipe.extractor.feed; + +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; + +import java.io.IOException; +import java.util.List; + +public class FeedInfo extends ListInfo { + + public FeedInfo(int serviceId, String id, String url, String originalUrl, String name, List contentFilter, String sortFilter) { + super(serviceId, id, url, originalUrl, name, contentFilter, sortFilter); + } + + public static FeedInfo getInfo(String url) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); + } + + public static FeedInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException { + final FeedExtractor extractor = service.getFeedExtractor(url); + + if (extractor == null) { + throw new IllegalArgumentException("Service \"" + service.getServiceInfo().getName() + "\" doesn't support FeedExtractor."); + } + + extractor.fetchPage(); + return getInfo(extractor); + } + + public static FeedInfo getInfo(FeedExtractor extractor) throws IOException, ExtractionException { + extractor.fetchPage(); + + final int serviceId = extractor.getServiceId(); + final String id = extractor.getId(); + final String url = extractor.getUrl(); + final String originalUrl = extractor.getOriginalUrl(); + final String name = extractor.getName(); + + final FeedInfo info = new FeedInfo(serviceId, id, url, originalUrl, name, null, null); + + final InfoItemsPage itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor); + info.setRelatedItems(itemsPage.getItems()); + info.setNextPageUrl(itemsPage.getNextPageUrl()); + + return info; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 78c97cfb..6137f029 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -24,14 +25,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.*; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; @@ -42,6 +36,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import javax.annotation.Nonnull; + /* * Created by Christian Schabesberger on 23.08.15. * @@ -72,7 +68,7 @@ public class YoutubeService extends StreamingService { public String getBaseUrl() { return "https://youtube.com"; } - + @Override public LinkHandlerFactory getStreamLHFactory() { return YoutubeStreamLinkHandlerFactory.getInstance(); @@ -147,6 +143,12 @@ public class YoutubeService extends StreamingService { return new YoutubeSubscriptionExtractor(this); } + @Nonnull + @Override + public FeedExtractor getFeedExtractor(final String channelUrl) throws ExtractionException { + return new YoutubeFeedExtractor(this, getChannelLHFactory().fromUrl(channelUrl)); + } + @Override public ListLinkHandlerFactory getCommentsLHFactory() { return YoutubeCommentsLinkHandlerFactory.getInstance(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index adc4705e..d675cb25 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -46,7 +46,6 @@ import java.io.IOException; @SuppressWarnings("WeakerAccess") public class YoutubeChannelExtractor extends ChannelExtractor { /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; - private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; private Document doc; @@ -130,7 +129,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { try { - return CHANNEL_FEED_BASE + getId(); + return YoutubeParsingHelper.getFeedUrlFrom(getId()); } catch (Exception e) { throw new ParsingException("Could not get feed url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java new file mode 100644 index 00000000..6ed91fe4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java @@ -0,0 +1,82 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class YoutubeFeedExtractor extends FeedExtractor { + public YoutubeFeedExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + } + + private Document document; + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String channelIdOrUser = getLinkHandler().getId(); + final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser); + + final Response response = downloader.get(feedUrl); + document = Jsoup.parse(response.responseBody()); + } + + @Nonnull + @Override + public ListExtractor.InfoItemsPage getInitialPage() { + final Elements entries = document.select("feed > entry"); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + for (Element entryElement : entries) { + collector.commit(new YoutubeFeedInfoItemExtractor(entryElement)); + } + + return new InfoItemsPage<>(collector, null); + } + + @Nonnull + @Override + public String getId() { + return document.getElementsByTag("yt:channelId").first().text(); + } + + @Nonnull + @Override + public String getUrl() { + return document.select("feed > author > uri").first().text(); + } + + @Nonnull + @Override + public String getName() { + return document.select("feed > author > name").first().text(); + } + + @Override + public String getNextPageUrl() { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) { + return null; + } + + @Override + public boolean hasNextPage() { + return false; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java new file mode 100644 index 00000000..aadc8022 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -0,0 +1,94 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.TimeZone; + +public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { + private final Element entryElement; + + public YoutubeFeedInfoItemExtractor(Element entryElement) { + this.entryElement = entryElement; + } + + @Override + public StreamType getStreamType() { + // It is not possible to determine the stream type using the feed endpoint. + // All entries are considered a video stream. + return StreamType.VIDEO_STREAM; + } + + @Override + public boolean isAd() { + return false; + } + + @Override + public long getDuration() { + // Not available when fetching through the feed endpoint. + return -1; + } + + @Override + public long getViewCount() { + return Long.parseLong(entryElement.getElementsByTag("media:statistics").first().attr("views")); + } + + @Override + public String getUploaderName() { + return entryElement.select("author > name").first().text(); + } + + @Override + public String getUploaderUrl() { + return entryElement.select("author > uri").first().text(); + } + + @Nullable + @Override + public String getTextualUploadDate() { + return entryElement.getElementsByTag("published").first().text(); + } + + @Nullable + @Override + public DateWrapper getUploadDate() throws ParsingException { + final Date date; + try { + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + date = dateFormat.parse(getTextualUploadDate()); + } catch (ParseException e) { + throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e); + } + + final Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + + return new DateWrapper(calendar); + } + + @Override + public String getName() { + return entryElement.getElementsByTag("title").first().text(); + } + + @Override + public String getUrl() { + return entryElement.getElementsByTag("link").first().attr("href"); + } + + @Override + public String getThumbnailUrl() { + return entryElement.getElementsByTag("media:thumbnail").first().attr("url"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 0e76ddf6..65ec7e3f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -38,6 +38,9 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; + private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; + private static final String[] RECAPTCHA_DETECTION_SELECTORS = { "form[action*=\"/das_captcha\"]", "input[name*=\"action_recaptcha_verify\"]" @@ -118,6 +121,16 @@ public class YoutubeParsingHelper { + Long.parseLong(seconds); } + public static String getFeedUrlFrom(final String channelIdOrUser) { + if (channelIdOrUser.startsWith("user/")) { + return FEED_BASE_USER + channelIdOrUser.replace("user/", ""); + } else if (channelIdOrUser.startsWith("channel/")) { + return FEED_BASE_CHANNEL_ID + channelIdOrUser.replace("channel/", ""); + } else { + return FEED_BASE_CHANNEL_ID + channelIdOrUser; + } + } + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java new file mode 100644 index 00000000..c7f1b1c1 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java @@ -0,0 +1,72 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + +public class YoutubeFeedExtractorTest { + public static class Kurzgesagt implements BaseListExtractorTest { + private static YoutubeFeedExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeFeedExtractor) YouTube + .getFeedExtractor("https://www.youtube.com/user/Kurzgesagt"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() { + String name = extractor.getName(); + assertTrue(name, name.startsWith("Kurzgesagt")); + } + + @Test + public void testId() { + assertEquals("UCsXVk37bltHxD1rDPwtNM8Q", extractor.getId()); + } + + @Test + public void testUrl() { + assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/user/Kurzgesagt", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, YouTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() { + assertFalse(extractor.hasNextPage()); + assertNull(extractor.getNextPageUrl()); + } + } +} \ No newline at end of file From 5756df8dc7e89b7383d1d1e07a91c30bdab6f868 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 6 Feb 2020 22:24:28 +0100 Subject: [PATCH 29/40] Use GMT as base time (actually fix upload date) --- .../services/peertube/PeertubeParsingHelper.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 2483dd94..c84e4e5b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,7 +4,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import java.util.concurrent.TimeUnit; +import java.util.TimeZone; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -13,22 +13,23 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import com.grack.nanojson.JsonObject; public class PeertubeParsingHelper { - + private PeertubeParsingHelper() { } public static void validate(JsonObject json) throws ContentNotAvailableException { String error = json.getString("error"); - if(!StringUtil.isBlank(error)) { + if (!StringUtil.isBlank(error)) { throw new ContentNotAvailableException(error); } } - + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); - date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1)); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } From 26c65b2948dcc58be4a004938f415bbd46380203 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 6 Feb 2020 23:35:46 +0100 Subject: [PATCH 30/40] Create class Description --- .../extractors/MediaCCCStreamExtractor.java | 5 +- .../extractors/PeertubeStreamExtractor.java | 23 ++++------ .../soundcloud/SoundcloudStreamExtractor.java | 5 +- .../extractors/YoutubeStreamExtractor.java | 6 +-- .../newpipe/extractor/stream/Description.java | 46 +++++++++++++++++++ .../extractor/stream/StreamExtractor.java | 6 +-- .../newpipe/extractor/stream/StreamInfo.java | 6 +-- .../PeertubeStreamExtractorDefaultTest.java | 7 +-- ...utubeStreamExtractorAgeRestrictedTest.java | 2 +- ...utubeStreamExtractorControversialTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 38 +++++++-------- .../YoutubeStreamExtractorLivestreamTest.java | 6 +-- 12 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 07c36d12..56036c2e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,7 +12,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; -import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import java.io.IOException; @@ -49,8 +48,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() throws ParsingException { - return data.getString("description"); + public Description getDescription() throws ParsingException { + return new Description(getServiceId(), data.getString("description")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 80cae88a..ac3267f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -21,14 +21,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; import com.grack.nanojson.JsonArray; @@ -72,25 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getDescription() throws ParsingException { - String desc; + public Description getDescription() throws ParsingException { + String text; try { - desc = JsonUtils.getString(json, "description"); + text = JsonUtils.getString(json, "description"); } catch (ParsingException e) { - return ""; + return Description.emptyDescription; } - if (desc.length() == 250 && desc.substring(247).equals("...")) { + if (text.length() == 250 && text.substring(247).equals("...")) { //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { Response response = dl.get(getUrl() + "/description"); JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - desc = JsonUtils.getString(jsonObject, "description"); + text = JsonUtils.getString(jsonObject, "description"); } catch (ReCaptchaException | IOException | JsonParserException e) { e.printStackTrace(); } } - return desc; + return new Description(getServiceId(), text); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 5b1153d5..c2a537a2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -74,10 +74,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return artworkUrlBetterResolution; } - @Nonnull @Override - public String getDescription() { - return track.getString("description"); + public Description getDescription() { + return new Description(getServiceId(), track.getString("description")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f8523ea5..d8ded292 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -180,15 +180,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() throws ParsingException { + public Description getDescription() throws ParsingException { assertPageFetched(); try { // first try to get html-formatted description - return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); + return new Description(getServiceId(), parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html())); } catch (Exception e) { try { // fallback to raw non-html description - return playerResponse.getObject("videoDetails").getString("shortDescription"); + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); } catch (Exception ignored) { throw new ParsingException("Could not get the description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java new file mode 100644 index 00000000..1ce5e948 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.stream; + +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class Description { + private String content; + private int type; + + public static final int HTML = 1; + public static final int MARKDOWN = 2; + public static final int PLAIN_TEXT = 3; + public static final Description emptyDescription = new Description(PLAIN_TEXT, ""); + + public Description(int serviceID, String content) { + if (serviceID == PeerTube.getServiceId()) { + this.type = MARKDOWN; + } else if (serviceID == YouTube.getServiceId()) { + this.type = HTML; + } else { + this.type = PLAIN_TEXT; + } + setContent(content); + } + + private void setContent(String content) { + if (content == null) { + this.content = ""; + } else { + this.content = content; + } + } + + public Description(String content, int type) { + this.type = type; + setContent(content); + } + + public String getContent() { + return content; + } + + public int getType() { + return type; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 652e0116..fdc56972 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -84,12 +84,12 @@ public abstract class StreamExtractor extends Extractor { public abstract String getThumbnailUrl() throws ParsingException; /** - * This is the stream description. On YouTube this is the video description. You can return simple HTML here. - * @return The description of the stream/video. + * This is the stream description. + * @return The description of the stream/video or Description.emptyDescription if the description is empty. * @throws ParsingException */ @Nonnull - public abstract String getDescription() throws ParsingException; + public abstract Description getDescription() throws ParsingException; /** * Get the age limit. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 64999ce7..580f9c21 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -319,7 +319,7 @@ public class StreamInfo extends Info { private DateWrapper uploadDate; private long duration = -1; private int ageLimit = -1; - private String description; + private Description description; private long viewCount = -1; private long likeCount = -1; @@ -417,11 +417,11 @@ public class StreamInfo extends Info { this.ageLimit = ageLimit; } - public String getDescription() { + public Description getDescription() { return description; } - public void setDescription(String description) { + public void setDescription(Description description) { this.description = description; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 06a8352c..794a14a3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -55,14 +55,14 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetLargeDescription() throws ParsingException { - assertEquals(expectedLargeDescription, extractor.getDescription()); + assertEquals(expectedLargeDescription, extractor.getDescription().getContent()); } @Test public void testGetEmptyDescription() throws Exception { PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); extractorEmpty.fetchPage(); - assertEquals("", extractorEmpty.getDescription()); + assertEquals("", extractorEmpty.getDescription().getContent()); } @Test @@ -70,7 +70,7 @@ public class PeertubeStreamExtractorDefaultTest { PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); extractorSmall.fetchPage(); - assertEquals(expectedSmallDescription, extractorSmall.getDescription()); + assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent()); } @Test @@ -89,6 +89,7 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } + @Ignore //fixme @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index b9da6d17..6afc7212 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -64,7 +64,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index f0bd7715..915f42ae 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 5f21df55..e2c0c7bb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -83,13 +83,13 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("http://adele.com")); - assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); + assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); + assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test @@ -215,18 +215,18 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); - assertTrue(extractor.getDescription().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); + assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); + assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); - assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub...")); - assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8...")); - assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/...")); + assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub...")); + assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8...")); + assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/...")); } } @@ -244,20 +244,20 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertFalse(extractor.getDescription().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index eb2541ed..b1d6f53a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -44,13 +44,13 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/")); - assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba...")); + assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/")); + assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba...")); } @Test From 70a40e7388f5bed7ca2019e0d4f82c18631f8902 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 13:28:27 +0100 Subject: [PATCH 31/40] Description: rm constructor by serviceId --- .../extractors/MediaCCCStreamExtractor.java | 2 +- .../extractors/PeertubeStreamExtractor.java | 2 +- .../soundcloud/SoundcloudStreamExtractor.java | 2 +- .../extractors/YoutubeStreamExtractor.java | 2 +- .../newpipe/extractor/stream/Description.java | 27 ++++--------------- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 56036c2e..08f21ef7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -49,7 +49,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public Description getDescription() throws ParsingException { - return new Description(getServiceId(), data.getString("description")); + return new Description(data.getString("description"), Description.PLAIN_TEXT); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index ac3267f1..a4dafbfc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -83,7 +83,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { e.printStackTrace(); } } - return new Description(getServiceId(), text); + return new Description(text, Description.MARKDOWN); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index c2a537a2..18fffa41 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -76,7 +76,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public Description getDescription() { - return new Description(getServiceId(), track.getString("description")); + return new Description(track.getString("description"), Description.PLAIN_TEXT); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index d8ded292..ea870d71 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -184,7 +184,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { // first try to get html-formatted description - return new Description(getServiceId(), parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html())); + return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML); } catch (Exception e) { try { // fallback to raw non-html description diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java index 1ce5e948..cde422bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -1,29 +1,17 @@ package org.schabi.newpipe.extractor.stream; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - public class Description { - private String content; - private int type; public static final int HTML = 1; public static final int MARKDOWN = 2; public static final int PLAIN_TEXT = 3; - public static final Description emptyDescription = new Description(PLAIN_TEXT, ""); + public static final Description emptyDescription = new Description("", PLAIN_TEXT); - public Description(int serviceID, String content) { - if (serviceID == PeerTube.getServiceId()) { - this.type = MARKDOWN; - } else if (serviceID == YouTube.getServiceId()) { - this.type = HTML; - } else { - this.type = PLAIN_TEXT; - } - setContent(content); - } + private String content; + private int type; - private void setContent(String content) { + public Description(String content, int type) { + this.type = type; if (content == null) { this.content = ""; } else { @@ -31,11 +19,6 @@ public class Description { } } - public Description(String content, int type) { - this.type = type; - setContent(content); - } - public String getContent() { return content; } From 0f8a7f9df2138d0e5697cc723655d8ff082699be Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 13:33:50 +0100 Subject: [PATCH 32/40] fix testGetUploadDate for PeerTubeStreamExtractor --- .../PeertubeStreamExtractorDefaultTest.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 794a14a3..a81bf34c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -12,6 +12,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Locale; +import java.util.TimeZone; import org.junit.BeforeClass; import org.junit.Ignore; @@ -42,6 +43,16 @@ public class PeertubeStreamExtractorDefaultTest { extractor.fetchPage(); } + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + + } + @Test public void testGetInvalidTimeStamp() throws ParsingException { assertTrue(extractor.getTimeStamp() + "", @@ -89,15 +100,6 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } - @Ignore //fixme - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T11:52:46.396Z")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - - } - @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); From 11bcc78d9c8eb39e8d61a6f4bc4112025937f087 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 14:10:54 +0100 Subject: [PATCH 33/40] Description implements Serializable. fix NotSerializableException --- .../java/org/schabi/newpipe/extractor/stream/Description.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java index cde422bb..b19b8de3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.stream; -public class Description { +import java.io.Serializable; + +public class Description implements Serializable { public static final int HTML = 1; public static final int MARKDOWN = 2; From 0e33249c85d2f23be9d8aa7fccd4bdc409a8f8ac Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 22:20:20 +0100 Subject: [PATCH 34/40] Fix SoundCloud description test --- .../soundcloud/SoundcloudStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 8a3883fd..0ffe0622 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { - assertEquals("The Perfect LUV Tape®️", extractor.getDescription()); + assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); } @Test From 008b6f505c94a657e9a645e87b84852b432e2462 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 23:08:37 +0100 Subject: [PATCH 35/40] [PeerTube] Fix subscriber count in ChannelExtractorTest --- .../services/peertube/PeertubeChannelExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 3d8bdff3..57b6360b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -204,7 +204,7 @@ public class PeertubeChannelExtractorTest { @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); } } } From 9112a106eab9c05ca5c9e574623d4908ecc15656 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 23:09:03 +0100 Subject: [PATCH 36/40] [SoundCloud] Update client_id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 4617f916..82216075 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "r5ELVSy3RkcjX7ilaL7n2v1Z8irA9SL8"; // Updated on 31/12/19 + private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 private static String clientId; private SoundcloudParsingHelper() { From 030465b5d4d8c738a42cfc52bd1d78f54b0f6bd4 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 23:58:46 +0100 Subject: [PATCH 37/40] Improve code formatting and optimise imports --- .../schabi/newpipe/extractor/Extractor.java | 36 ++--- .../newpipe/extractor/InfoItemsCollector.java | 2 +- .../schabi/newpipe/extractor/MediaFormat.java | 12 +- .../schabi/newpipe/extractor/ServiceList.java | 8 +- .../newpipe/extractor/StreamingService.java | 14 +- .../channel/ChannelInfoItemsCollector.java | 2 +- .../extractor/comments/CommentsInfo.java | 32 ++--- .../extractor/kiosk/KioskExtractor.java | 1 - .../newpipe/extractor/kiosk/KioskList.java | 27 ++-- .../extractor/linkhandler/LinkHandler.java | 8 +- .../linkhandler/LinkHandlerFactory.java | 12 +- .../linkhandler/ListLinkHandlerFactory.java | 33 +++-- .../linkhandler/SearchQueryHandler.java | 1 + .../SearchQueryHandlerFactory.java | 14 +- .../extractor/localization/Localization.java | 5 +- .../extractor/localization/TimeAgoParser.java | 4 +- .../search/InfoItemsSearchCollector.java | 9 +- .../MediaCCCConferenceExtractor.java | 2 +- .../extractors/MediaCCCConferenceKiosk.java | 2 +- .../extractors/MediaCCCSearchExtractor.java | 20 +-- .../extractors/MediaCCCStreamExtractor.java | 18 +-- .../MediaCCCConferenceLinkHandlerFactory.java | 4 +- ...aCCCConferencesListLinkHandlerFactory.java | 1 - .../MediaCCCSearchQueryHandlerFactory.java | 2 +- .../MediaCCCStreamLinkHandlerFactory.java | 4 +- .../services/peertube/PeertubeInstance.java | 31 ++--- .../peertube/PeertubeParsingHelper.java | 11 +- .../services/peertube/PeertubeService.java | 44 ++---- .../extractors/PeertubeChannelExtractor.java | 61 ++++----- .../extractors/PeertubeCommentsExtractor.java | 43 +++--- .../PeertubeCommentsInfoItemExtractor.java | 15 +- .../extractors/PeertubePlaylistExtractor.java | 8 +- .../extractors/PeertubeSearchExtractor.java | 51 ++++--- .../extractors/PeertubeStreamExtractor.java | 24 ++-- .../PeertubeStreamInfoItemExtractor.java | 27 ++-- .../PeertubeSubscriptionExtractor.java | 4 +- .../PeertubeSuggestionExtractor.java | 10 +- .../extractors/PeertubeTrendingExtractor.java | 41 +++--- .../PeertubeChannelLinkHandlerFactory.java | 6 +- .../PeertubeCommentsLinkHandlerFactory.java | 8 +- .../PeertubePlaylistLinkHandlerFactory.java | 8 +- .../PeertubeSearchQueryHandlerFactory.java | 14 +- .../PeertubeStreamLinkHandlerFactory.java | 2 +- .../PeertubeTrendingLinkHandlerFactory.java | 26 ++-- .../SoundcloudChannelExtractor.java | 6 +- .../SoundcloudChannelLinkHandlerFactory.java | 2 +- .../soundcloud/SoundcloudChartsExtractor.java | 10 +- .../soundcloud/SoundcloudParsingHelper.java | 4 +- .../SoundcloudPlaylistExtractor.java | 2 +- .../SoundcloudPlaylistLinkHandlerFactory.java | 2 +- .../soundcloud/SoundcloudSearchExtractor.java | 5 +- .../SoundcloudSearchQueryHandlerFactory.java | 4 +- .../soundcloud/SoundcloudService.java | 27 ++-- .../soundcloud/SoundcloudStreamExtractor.java | 6 +- .../SoundcloudStreamLinkHandlerFactory.java | 2 +- .../SoundcloudSuggestionExtractor.java | 2 +- .../services/youtube/YoutubeService.java | 28 +--- .../extractors/YoutubeChannelExtractor.java | 2 +- .../extractors/YoutubeCommentsExtractor.java | 29 ++-- .../YoutubeCommentsInfoItemExtractor.java | 2 +- .../extractors/YoutubePlaylistExtractor.java | 9 +- .../YoutubePlaylistInfoItemExtractor.java | 2 +- .../extractors/YoutubeSearchExtractor.java | 2 +- .../extractors/YoutubeStreamExtractor.java | 128 +++++++++--------- .../YoutubeStreamInfoItemExtractor.java | 9 +- .../YoutubeSuggestionExtractor.java | 6 +- .../extractors/YoutubeTrendingExtractor.java | 4 +- .../YoutubePlaylistLinkHandlerFactory.java | 2 +- .../YoutubeSearchQueryHandlerFactory.java | 4 +- .../newpipe/extractor/stream/Frameset.java | 102 +++++++------- .../newpipe/extractor/stream/Stream.java | 14 +- .../extractor/stream/StreamExtractor.java | 37 ++++- .../stream/StreamInfoItemExtractor.java | 11 +- .../stream/StreamInfoItemsCollector.java | 4 +- .../extractor/stream/SubtitlesStream.java | 4 +- .../newpipe/extractor/stream/VideoStream.java | 6 +- .../extractor/utils/DashMpdParser.java | 12 +- .../extractor/utils/ExtractorHelper.java | 5 +- .../newpipe/extractor/utils/JsonUtils.java | 16 +-- .../newpipe/extractor/utils/Parser.java | 16 +-- .../schabi/newpipe/extractor/utils/Utils.java | 11 +- .../MediaCCCConferenceExtractorTest.java | 2 +- .../MediaCCCConferenceListExtractorTest.java | 6 +- .../services/media_ccc/MediaCCCOggTest.java | 4 +- .../MediaCCCSearchExtractorAllTest.java | 10 +- ...ediaCCCSearchExtractorConferencesTest.java | 4 +- .../MediaCCCSearchExtractorEventsTest.java | 4 +- .../MediaCCCStreamExtractorTest.java | 2 +- .../PeertubeChannelExtractorTest.java | 16 +-- ...PeertubeChannelLinkHandlerFactoryTest.java | 6 +- .../PeertubeCommentsExtractorTest.java | 22 +-- ...eertubeCommentsLinkHandlerFactoryTest.java | 6 +- ...eertubePlaylistLinkHandlerFactoryTest.java | 6 +- .../PeertubeStreamExtractorDefaultTest.java | 26 ++-- .../PeertubeStreamLinkHandlerFactoryTest.java | 6 +- .../PeertubeTrendingExtractorTest.java | 17 +-- ...eertubeTrendingLinkHandlerFactoryTest.java | 14 +- .../PeertubeSearchExtractorBaseTest.java | 4 +- .../PeertubeSearchExtractorDefaultTest.java | 18 ++- .../peertube/search/PeertubeSearchQHTest.java | 10 +- .../SoundcloudChartsExtractorTest.java | 5 +- .../SoundcloudParsingHelperTest.java | 6 +- ...ndcloudSearchExtractorChannelOnlyTest.java | 6 +- .../SoundcloudSearchExtractorDefaultTest.java | 6 +- .../youtube/YoutubeChannelExtractorTest.java | 16 +-- .../YoutubeChannelLinkHandlerFactoryTest.java | 8 +- .../YoutubeStreamLinkHandlerFactoryTest.java | 4 +- .../youtube/YoutubeTrendingKioskInfoTest.java | 38 +++--- .../YoutubeSearchExtractorBaseTest.java | 2 - ...YoutubeSearchExtractorChannelOnlyTest.java | 4 +- .../YoutubeSearchExtractorDefaultTest.java | 7 +- .../search/YoutubeSearchPagingTest.java | 18 ++- .../youtube/search/YoutubeSearchQHTest.java | 7 +- ...utubeStreamExtractorAgeRestrictedTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 4 +- .../extractor/utils/JsonUtilsTest.java | 25 ++-- 116 files changed, 792 insertions(+), 793 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index b6e90d53..df1c2261 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor; -import java.io.IOException; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -13,7 +8,11 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -public abstract class Extractor{ +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; + +public abstract class Extractor { /** * {@link StreamingService} currently related to this extractor.
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls). @@ -21,19 +20,21 @@ public abstract class Extractor{ private final StreamingService service; private final LinkHandler linkHandler; - @Nullable private Localization forcedLocalization = null; - @Nullable private ContentCountry forcedContentCountry = null; + @Nullable + private Localization forcedLocalization = null; + @Nullable + private ContentCountry forcedContentCountry = null; private boolean pageFetched = false; private final Downloader downloader; public Extractor(final StreamingService service, final LinkHandler linkHandler) { - if(service == null) throw new NullPointerException("service is null"); - if(linkHandler == null) throw new NullPointerException("LinkHandler is null"); + if (service == null) throw new NullPointerException("service is null"); + if (linkHandler == null) throw new NullPointerException("LinkHandler is null"); this.service = service; this.linkHandler = linkHandler; this.downloader = NewPipe.getDownloader(); - if(downloader == null) throw new NullPointerException("downloader is null"); + if (downloader == null) throw new NullPointerException("downloader is null"); } /** @@ -46,11 +47,12 @@ public abstract class Extractor{ /** * Fetch the current page. - * @throws IOException if the page can not be loaded + * + * @throws IOException if the page can not be loaded * @throws ExtractionException if the pages content is not understood */ public void fetchPage() throws IOException, ExtractionException { - if(pageFetched) return; + if (pageFetched) return; onFetchPage(downloader); pageFetched = true; } @@ -65,8 +67,9 @@ public abstract class Extractor{ /** * Fetch the current page. + * * @param downloader the download to use - * @throws IOException if the page can not be loaded + * @throws IOException if the page can not be loaded * @throws ExtractionException if the pages content is not understood */ public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException; @@ -78,6 +81,7 @@ public abstract class Extractor{ /** * Get the name + * * @return the name * @throws ParsingException if the name cannot be extracted */ @@ -93,10 +97,10 @@ public abstract class Extractor{ public String getUrl() throws ParsingException { return linkHandler.getUrl(); } - + @Nonnull public String getBaseUrl() throws ParsingException { - return linkHandler.getBaseUrl(); + return linkHandler.getBaseUrl(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java index 2a3cb5f3..72eca16e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java @@ -27,7 +27,7 @@ import java.util.List; * along with NewPipe. If not, see . */ -public abstract class InfoItemsCollector implements Collector { +public abstract class InfoItemsCollector implements Collector { private final List itemList = new ArrayList<>(); private final List errors = new ArrayList<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index c65283a5..0b6052f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -116,25 +116,27 @@ public enum MediaFormat { /** * Get the media format by it's id. + * * @param id the id * @return the id of the media format or null. */ public static MediaFormat getFormatById(int id) { - for (MediaFormat vf: values()) { + for (MediaFormat vf : values()) { if (vf.id == id) return vf; } return null; } public static MediaFormat getFromSuffix(String suffix) { - for (MediaFormat vf: values()) { + for (MediaFormat vf : values()) { if (vf.suffix.equals(suffix)) return vf; } return null; } - + /** * Get the name of the format + * * @return the name of the format */ public String getName() { @@ -143,6 +145,7 @@ public enum MediaFormat { /** * Get the filename extension + * * @return the filename extension */ public String getSuffix() { @@ -151,10 +154,11 @@ public enum MediaFormat { /** * Get the mime type + * * @return the mime type */ public String getMimeType() { return mimeType; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 6be1cea4..83d8522f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /* * Copyright (C) Christian Schabesberger 2018 * ServiceList.java is part of NewPipe. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 68398506..110efe04 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -1,20 +1,12 @@ package org.schabi.newpipe.extractor; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -26,6 +18,8 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; /* * Copyright (C) Christian Schabesberger 2018 @@ -269,7 +263,7 @@ public abstract class StreamingService { public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException { ListLinkHandlerFactory llhf = getCommentsLHFactory(); - if(null == llhf) { + if (null == llhf) { return null; } return getCommentsExtractor(llhf.fromUrl(url)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java index a60636b2..19abb1c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java @@ -33,7 +33,7 @@ public class ChannelInfoItemsCollector extends InfoItemsCollector{ +import java.io.IOException; - private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { - super(serviceId, listUrlIdHandler, name); - // TODO Auto-generated constructor stub - } - - public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { +public class CommentsInfo extends ListInfo { + + private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { + super(serviceId, listUrlIdHandler, name); + // TODO Auto-generated constructor stub + } + + public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { return getInfo(NewPipe.getServiceByUrl(url), url); } - public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException { - return getInfo(serviceByUrl.getCommentsExtractor(url)); - } + public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException { + return getInfo(serviceByUrl.getCommentsExtractor(url)); + } private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException { // for services which do not have a comments extractor @@ -44,21 +44,21 @@ public class CommentsInfo extends ListInfo{ return commentsInfo; } - + public static InfoItemsPage getMoreItems(CommentsInfo commentsInfo, String pageUrl) throws ExtractionException, IOException { return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl); } - + public static InfoItemsPage getMoreItems(StreamingService service, CommentsInfo commentsInfo, - String pageUrl) throws IOException, ExtractionException { + String pageUrl) throws IOException, ExtractionException { if (null == commentsInfo.getCommentsExtractor()) { commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl())); commentsInfo.getCommentsExtractor().fetchPage(); } return commentsInfo.getCommentsExtractor().getPage(pageUrl); } - + private transient CommentsExtractor commentsExtractor; public CommentsExtractor getCommentsExtractor() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 7aef494d..44890524 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -25,7 +25,6 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; import javax.annotation.Nonnull; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 5a3be079..878fa8ce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -13,27 +13,30 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -public class KioskList { +public class KioskList { public interface KioskExtractorFactory { KioskExtractor createNewKiosk(final StreamingService streamingService, final String url, final String kioskId) - throws ExtractionException, IOException; + throws ExtractionException, IOException; } private final StreamingService service; private final HashMap kioskList = new HashMap<>(); private String defaultKiosk = null; - @Nullable private Localization forcedLocalization; - @Nullable private ContentCountry forcedContentCountry; + @Nullable + private Localization forcedLocalization; + @Nullable + private ContentCountry forcedContentCountry; private class KioskEntry { public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) { extractorFactory = ef; handlerFactory = h; } + final KioskExtractorFactory extractorFactory; final ListLinkHandlerFactory handlerFactory; } @@ -43,8 +46,8 @@ public class KioskList { } public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id) - throws Exception { - if(kioskList.get(id) != null) { + throws Exception { + if (kioskList.get(id) != null) { throw new Exception("Kiosk with type " + id + " already exists."); } kioskList.put(id, new KioskEntry(extractorFactory, handlerFactory)); @@ -66,10 +69,10 @@ public class KioskList { public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization) throws ExtractionException, IOException { - if(defaultKiosk != null && !defaultKiosk.equals("")) { + if (defaultKiosk != null && !defaultKiosk.equals("")) { return getExtractorById(defaultKiosk, nextPageUrl, localization); } else { - if(!kioskList.isEmpty()) { + if (!kioskList.isEmpty()) { // if not set get any entry Object[] keySet = kioskList.keySet().toArray(); return getExtractorById(keySet[0].toString(), nextPageUrl, localization); @@ -91,7 +94,7 @@ public class KioskList { public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization) throws ExtractionException, IOException { KioskEntry ke = kioskList.get(kioskId); - if(ke == null) { + if (ke == null) { throw new ExtractionException("No kiosk found with the type: " + kioskId); } else { final KioskExtractor kioskExtractor = ke.extractorFactory.createNewKiosk(service, @@ -109,15 +112,15 @@ public class KioskList { } public KioskExtractor getExtractorByUrl(String url, String nextPageUrl) - throws ExtractionException, IOException{ + throws ExtractionException, IOException { return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization()); } public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization) throws ExtractionException, IOException { - for(Map.Entry e : kioskList.entrySet()) { + for (Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); - if(ke.handlerFactory.acceptUrl(url)) { + if (ke.handlerFactory.acceptUrl(url)) { return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java index 52510513..b7439c39 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.linkhandler; -import java.io.Serializable; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; +import java.io.Serializable; + public class LinkHandler implements Serializable { protected final String originalUrl; protected final String url; @@ -31,8 +31,8 @@ public class LinkHandler implements Serializable { public String getId() { return id; } - + public String getBaseUrl() throws ParsingException { - return Utils.getBaseUrl(url); + return Utils.getBaseUrl(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 3890c2b2..6bba7b4e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -35,9 +35,9 @@ public abstract class LinkHandlerFactory { public abstract boolean onAcceptUrl(final String url) throws ParsingException; public String getUrl(String id, String baseUrl) throws ParsingException { - return getUrl(id); + return getUrl(id); } - + /////////////////////////////////// // Logic /////////////////////////////////// @@ -47,7 +47,7 @@ public abstract class LinkHandlerFactory { final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } - + public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if (url == null) throw new IllegalArgumentException("url can not be null"); if (!acceptUrl(url)) { @@ -55,9 +55,9 @@ public abstract class LinkHandlerFactory { } final String id = getId(url); - return new LinkHandler(url, getUrl(id,baseUrl), id); + return new LinkHandler(url, getUrl(id, baseUrl), id); } - + public LinkHandler fromId(String id) throws ParsingException { if (id == null) throw new IllegalArgumentException("id can not be null"); final String url = getUrl(id); @@ -82,5 +82,5 @@ public abstract class LinkHandlerFactory { throw fe; } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 83955265..60e1db48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -1,29 +1,35 @@ package org.schabi.newpipe.extractor.linkhandler; -import java.util.ArrayList; -import java.util.List; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; +import java.util.ArrayList; +import java.util.List; + public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /////////////////////////////////// // To Override /////////////////////////////////// - public List getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);} - public String getSortFilter(String url) throws ParsingException {return ""; } + public List getContentFilter(String url) throws ParsingException { + return new ArrayList<>(0); + } + + public String getSortFilter(String url) throws ParsingException { + return ""; + } + public abstract String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException; - + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return getUrl(id, contentFilter, sortFilter); + return getUrl(id, contentFilter, sortFilter); } /////////////////////////////////// // Logic /////////////////////////////////// - + @Override public ListLinkHandler fromUrl(String url) throws ParsingException { String baseUrl = Utils.getBaseUrl(url); @@ -32,7 +38,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { @Override public ListLinkHandler fromUrl(String url, String baseUrl) throws ParsingException { - if(url == null) throw new IllegalArgumentException("url may not be null"); + if (url == null) throw new IllegalArgumentException("url may not be null"); return new ListLinkHandler(super.fromUrl(url, baseUrl), getContentFilter(url), getSortFilter(url)); } @@ -41,7 +47,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public ListLinkHandler fromId(String id) throws ParsingException { return new ListLinkHandler(super.fromId(id), new ArrayList(0), ""); } - + @Override public ListLinkHandler fromId(String id, String baseUrl) throws ParsingException { return new ListLinkHandler(super.fromId(id, baseUrl), new ArrayList(0), ""); @@ -53,7 +59,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { final String url = getUrl(id, contentFilters, sortFilter); return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } - + public ListLinkHandler fromQuery(String id, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { @@ -61,10 +67,11 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } - + /** * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, * however it should not be overridden by the actual implementation. + * * @param id * @return the url coresponding to id without any filters applied */ @@ -76,7 +83,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public String getUrl(String id, String baseUrl) throws ParsingException { return getUrl(id, new ArrayList(0), "", baseUrl); } - + /** * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java index 6c1893c7..67dd7e49 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java @@ -24,6 +24,7 @@ public class SearchQueryHandler extends ListLinkHandler { /** * Returns the search string. Since ListQIHandler is based on ListLinkHandler * getSearchString() is equivalent to calling getId(). + * * @return the search string */ public String getSearchString() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java index 997e2d09..55c76b78 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java @@ -13,14 +13,19 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { @Override public abstract String getUrl(String querry, List contentFilter, String sortFilter) throws ParsingException; - public String getSearchString(String url) { return "";} + + public String getSearchString(String url) { + return ""; + } /////////////////////////////////// // Logic /////////////////////////////////// @Override - public String getId(String url) { return getSearchString(url); } + public String getId(String url) { + return getSearchString(url); + } @Override public SearchQueryHandler fromQuery(String querry, @@ -35,9 +40,12 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { /** * It's not mandatorry for NewPipe to handle the Url + * * @param url * @return */ @Override - public boolean onAcceptUrl(String url) { return false; } + public boolean onAcceptUrl(String url) { + return false; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java index 7461462a..02b89087 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java @@ -3,7 +3,10 @@ package org.schabi.newpipe.extractor.localization; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; public class Localization implements Serializable { public static final Localization DEFAULT = new Localization("en", "GB"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index aae9b14c..fe20135f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -21,8 +21,9 @@ public class TimeAgoParser { /** * Creates a helper to parse upload dates in the format '2 days ago'. *

- * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items. + * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items. *

+ * * @param patternsHolder An object that holds the "time ago" patterns, special cases, and the language word separator. */ public TimeAgoParser(PatternsHolder patternsHolder) { @@ -164,6 +165,7 @@ public class TimeAgoParser { /** * Marks the time as approximated by setting minutes, seconds and milliseconds to 0. + * * @param calendarTime Time to be marked as approximated */ private void markApproximatedTime(Calendar calendarTime) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java index bad31ea2..206fb68f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; @@ -34,7 +33,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; /** * Collector for search results - * + *

* This collector can handle the following extractor types: *

    *
  • {@link StreamInfoItemExtractor}
    • @@ -59,11 +58,11 @@ public class InfoItemsSearchCollector extends InfoItemsCollector getInitialPage() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray events = conferenceData.getArray("events"); - for(int i = 0; i < events.size(); i++) { + for (int i = 0; i < events.size(); i++) { collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i))); } return new InfoItemsPage<>(collector, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java index 580f79b0..f9800b46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java @@ -32,7 +32,7 @@ public class MediaCCCConferenceKiosk extends KioskExtractor { public InfoItemsPage getInitialPage() throws IOException, ExtractionException { JsonArray conferences = doc.getArray("conferences"); ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId()); - for(int i = 0; i < conferences.size(); i++) { + for (int i = 0; i < conferences.size(); i++) { collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i))); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 429445f2..6d740337 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -50,7 +50,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); searchItems.reset(); - if(getLinkHandler().getContentFilters().contains(CONFERENCES) + if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { searchConferences(getSearchString(), @@ -58,7 +58,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { searchItems); } - if(getLinkHandler().getContentFilters().contains(EVENTS) + if (getLinkHandler().getContentFilters().contains(EVENTS) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { JsonArray events = doc.getArray("events"); @@ -82,8 +82,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - if(getLinkHandler().getContentFilters().contains(EVENTS) - || getLinkHandler().getContentFilters().contains(ALL) + if (getLinkHandler().getContentFilters().contains(EVENTS) + || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { final String site; final String url = getUrl(); @@ -94,17 +94,17 @@ public class MediaCCCSearchExtractor extends SearchExtractor { throw new ExtractionException("Could not parse json.", jpe); } } - if(getLinkHandler().getContentFilters().contains(CONFERENCES) + if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) - conferenceKiosk.fetchPage(); + conferenceKiosk.fetchPage(); } private void searchConferences(String searchString, - List channelItems, - InfoItemsSearchCollector collector) { - for(final ChannelInfoItem item : channelItems) { - if(item.getName().toUpperCase().contains( + List channelItems, + InfoItemsSearchCollector collector) { + for (final ChannelInfoItem item : channelItems) { + if (item.getName().toUpperCase().contains( searchString.toUpperCase())) { collector.commit(new ChannelInfoItemExtractor() { @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 08f21ef7..c499758d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -117,17 +117,17 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getAudioStreams() throws IOException, ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List audioStreams = new ArrayList<>(); - for(int i = 0; i < recordings.size(); i++) { + for (int i = 0; i < recordings.size(); i++) { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); - if(mimeType.startsWith("audio")) { + if (mimeType.startsWith("audio")) { //first we need to resolve the actual video data from CDN final MediaFormat mediaFormat; - if(mimeType.endsWith("opus")) { + if (mimeType.endsWith("opus")) { mediaFormat = MediaFormat.OPUS; - } else if(mimeType.endsWith("mpeg")) { + } else if (mimeType.endsWith("mpeg")) { mediaFormat = MediaFormat.MP3; - } else if(mimeType.endsWith("ogg")){ + } else if (mimeType.endsWith("ogg")) { mediaFormat = MediaFormat.OGG; } else { throw new ExtractionException("Unknown media format: " + mimeType); @@ -143,16 +143,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getVideoStreams() throws IOException, ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List videoStreams = new ArrayList<>(); - for(int i = 0; i < recordings.size(); i++) { + for (int i = 0; i < recordings.size(); i++) { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); - if(mimeType.startsWith("video")) { + if (mimeType.startsWith("video")) { //first we need to resolve the actual video data from CDN final MediaFormat mediaFormat; - if(mimeType.endsWith("webm")) { + if (mimeType.endsWith("webm")) { mediaFormat = MediaFormat.WEBM; - } else if(mimeType.endsWith("mp4")) { + } else if (mimeType.endsWith("mp4")) { mediaFormat = MediaFormat.MPEG_4; } else { throw new ExtractionException("Unknown media format: " + mimeType); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index b1207a9a..3c181767 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -15,9 +15,9 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory @Override public String getId(String url) throws ParsingException { - if(url.startsWith("https://api.media.ccc.de/public/conferences/")) { + if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { return url.replace("https://api.media.ccc.de/public/conferences/", ""); - } else if(url.startsWith("https://media.ccc.de/c/")) { + } else if (url.startsWith("https://media.ccc.de/c/")) { return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); } else { throw new ParsingException("Could not get id from url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java index 603f1ea2..d44dc538 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import java.util.List; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index fb6c705a..dcc0b29b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -15,7 +15,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, CONFERENCES, EVENTS diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index fc6bf39e..bf291a60 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -7,8 +7,8 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getId(String url) throws ParsingException { - if(url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q=")) { + if (url.startsWith("https://api.media.ccc.de/public/events/") && + !url.contains("?q=")) { return url.replace("https://api.media.ccc.de/public/events/", ""); } throw new ParsingException("Could not get id from url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 992b8eb6..17d08580 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube; -import java.io.IOException; - +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -10,22 +11,20 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import java.io.IOException; public class PeertubeInstance { - + private final String url; private String name; public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); - + public PeertubeInstance(String url) { this.url = url; this.name = "PeerTube"; } - - public PeertubeInstance(String url , String name) { + + public PeertubeInstance(String url, String name) { this.url = url; this.name = name; } @@ -33,22 +32,22 @@ public class PeertubeInstance { public String getUrl() { return url; } - + public void fetchInstanceMetaData() throws Exception { Downloader downloader = NewPipe.getDownloader(); Response response = null; - + try { response = downloader.get(url + "/api/v1/config"); } catch (ReCaptchaException | IOException e) { throw new Exception("unable to configure instance " + url, e); } - - if(null == response || StringUtil.isBlank(response.responseBody())) { + + if (null == response || StringUtil.isBlank(response.responseBody())) { throw new Exception("unable to configure instance " + url); } - - try { + + try { JsonObject json = JsonParser.object().from(response.responseBody()); this.name = JsonUtils.getString(json, "instance.name"); } catch (JsonParserException | ParsingException e) { @@ -59,5 +58,5 @@ public class PeertubeInstance { public String getName() { return name; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index c84e4e5b..7821aa29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -1,17 +1,16 @@ package org.schabi.newpipe.extractor.services.peertube; +import com.grack.nanojson.JsonObject; +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.TimeZone; -import org.jsoup.helper.StringUtil; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import com.grack.nanojson.JsonObject; - public class PeertubeParsingHelper { private PeertubeParsingHelper() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 47271732..29cc2b1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -1,49 +1,35 @@ package org.schabi.newpipe.extractor.services.peertube; -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.extractors.*; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + public class PeertubeService extends StreamingService { - + private PeertubeInstance instance; - + public PeertubeService(int id) { this(id, PeertubeInstance.defaultInstance); } - + public PeertubeService(int id, PeertubeInstance instance) { super(id, "PeerTube", asList(VIDEO, COMMENTS)); - this.instance = instance; + this.instance = instance; } @Override @@ -117,15 +103,15 @@ public class PeertubeService extends StreamingService { public String getBaseUrl() { return instance.getUrl(); } - + public PeertubeInstance getInstance() { return this.instance; } - + public void setInstance(PeertubeInstance instance) { this.instance = instance; } - + @Override public KioskList getKioskList() throws ExtractionException { KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() { @@ -155,6 +141,6 @@ public class PeertubeService extends StreamingService { return list; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 8b214670..4d1d00f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -1,7 +1,9 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -17,21 +19,18 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import java.io.IOException; public class PeertubeChannelExtractor extends ChannelExtractor { - + private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + private JsonObject json; private final String baseUrl; @@ -45,7 +44,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { String value; try { value = JsonUtils.getString(json, "avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -71,7 +70,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public String getDescription() throws ParsingException { try { return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { + } catch (ParsingException e) { return "No description"; } } @@ -86,18 +85,18 @@ public class PeertubeChannelExtractor extends ChannelExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract channel streams", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + } @Override @@ -110,26 +109,26 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for kiosk info", e); } } - + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { PeertubeParsingHelper.validate(json); Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube kiosk info"); } return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; @@ -138,30 +137,30 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } - + @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { + } else { throw new ExtractionException("Unable to extract peertube channel data"); } - + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } @@ -172,14 +171,14 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube channel data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube channel data"); + if (null == json) throw new ExtractionException("Unable to extract peertube channel data"); } @Override public String getName() throws ParsingException { return JsonUtils.getString(json, "displayName"); } - + @Override public String getOriginalUrl() throws ParsingException { return baseUrl + "/accounts/" + getId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index b7aba4fa..34477459 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -16,9 +17,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeCommentsExtractor extends CommentsExtractor { @@ -26,10 +25,10 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { super(service, uiHandler); } @@ -38,7 +37,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public String getName() throws ParsingException { return "Comments"; } - + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); @@ -49,18 +48,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract comments info", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); collector.commit(extractor); } } - + } @Override @@ -73,18 +72,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for comments info", e); } } - + CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube comments info"); @@ -97,7 +96,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -105,17 +104,17 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 0bcce9fc..b7cbcec7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.ServiceList; @@ -9,8 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; - public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @@ -34,7 +33,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -49,13 +48,13 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getTextualPublishedTime() throws ParsingException { return JsonUtils.getString(item, "createdAt"); } - + @Override public DateWrapper getPublishedTime() throws ParsingException { String textualUploadDate = getTextualPublishedTime(); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public int getLikeCount() throws ParsingException { return -1; @@ -67,7 +66,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac try { Document doc = Jsoup.parse(htmlText); return doc.body().text(); - } catch(Exception e) { + } catch (Exception e) { return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); } } @@ -83,7 +82,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - } catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -100,5 +99,5 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String host = JsonUtils.getString(item, "account.host"); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 98d20b53..81cd165d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -1,7 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -10,7 +8,9 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -public class PeertubePlaylistExtractor extends PlaylistExtractor{ +import java.io.IOException; + +public class PeertubePlaylistExtractor extends PlaylistExtractor { public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -74,7 +74,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor{ @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { // TODO Auto-generated method stub - + } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index ff021f73..3840bf76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemExtractor; @@ -18,9 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeSearchExtractor extends SearchExtractor { @@ -28,10 +27,10 @@ public class PeertubeSearchExtractor extends SearchExtractor { private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); } @@ -40,7 +39,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { public String getSearchSuggestion() throws ParsingException { return null; } - + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); @@ -48,27 +47,27 @@ public class PeertubeSearchExtractor extends SearchExtractor { } private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { - + final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - + JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract search info", e); } - + String baseUrl = getBaseUrl(); - for(Object c: contents) { - if(c instanceof JsonObject) { + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + return collector; - + } @Override @@ -81,17 +80,17 @@ public class PeertubeSearchExtractor extends SearchExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for search info", e); } } - - if(json != null) { + + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl)); } else { throw new ExtractionException("Unable to get peertube search info"); @@ -103,7 +102,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -111,20 +110,20 @@ public class PeertubeSearchExtractor extends SearchExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index a4dafbfc..76ead5d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,13 +1,9 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; @@ -24,12 +20,14 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearch import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; - import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; public class PeertubeStreamExtractor extends StreamExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index ca85270a..c4ed7730 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -8,58 +9,56 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; - public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - + protected final JsonObject item; private final String baseUrl; - + public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) { this.item = item; this.baseUrl = baseUrl; } - + @Override public String getUrl() throws ParsingException { String uuid = JsonUtils.getString(item, "uuid"); return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl(); } - + @Override public String getThumbnailUrl() throws ParsingException { String value = JsonUtils.getString(item, "thumbnailPath"); return baseUrl + value; } - + @Override public String getName() throws ParsingException { return JsonUtils.getString(item, "name"); } - + @Override public boolean isAd() throws ParsingException { return false; } - + @Override public long getViewCount() throws ParsingException { Number value = JsonUtils.getNumber(item, "views"); return value.longValue(); } - + @Override public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } - + @Override public String getUploaderName() throws ParsingException { return JsonUtils.getString(item, "account.displayName"); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(item, "publishedAt"); @@ -75,12 +74,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public StreamType getStreamType() throws ParsingException { return StreamType.VIDEO_STREAM; } - + @Override public long getDuration() throws ParsingException { Number value = JsonUtils.getNumber(item, "duration"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java index a2b24614..17062900 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import java.util.List; + public class PeertubeSubscriptionExtractor extends SubscriptionExtractor { public PeertubeSubscriptionExtractor(StreamingService service, List supportedSources) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java index 7e2ed4d9..a028e77a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -public class PeertubeSuggestionExtractor extends SuggestionExtractor{ +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class PeertubeSuggestionExtractor extends SuggestionExtractor { public PeertubeSuggestionExtractor(StreamingService service) { super(service); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index 1cb42046..e6c1a1b5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -16,17 +17,15 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeTrendingExtractor extends KioskExtractor { - + private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; @@ -49,19 +48,19 @@ public class PeertubeTrendingExtractor extends KioskExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract kiosk info", e); } - + String baseUrl = getBaseUrl(); - for(Object c: contents) { - if(c instanceof JsonObject) { + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + } @Override @@ -74,18 +73,18 @@ public class PeertubeTrendingExtractor extends KioskExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for kiosk info", e); } } - + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube kiosk info"); @@ -98,7 +97,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -106,17 +105,17 @@ public class PeertubeTrendingExtractor extends KioskExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index 640f99e5..c1e3f570 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); @@ -27,7 +27,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, searchFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java index 9faa3492..9c833750 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory(); @@ -33,10 +33,10 @@ public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilter, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { return baseUrl + String.format(COMMENTS_ENDPOINT, id); } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java index 64e291a5..dd1bd77f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory(); @@ -23,12 +23,12 @@ public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; } - + @Override public String getId(String url) throws ParsingException { return Parser.matchGroup1(ID_PATTERN, url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 69591167..8acf3e80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String CHARSET_UTF_8 = "UTF-8"; @@ -21,9 +21,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { String baseUrl = ServiceList.PeerTube.getBaseUrl(); - return getUrl(searchString, contentFilters, sortFilter, baseUrl); + return getUrl(searchString, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String searchString, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { try { @@ -38,6 +38,6 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { VIDEOS }; + return new String[]{VIDEOS}; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index 3ee7744c..222dc129 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -24,7 +24,7 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, baseUrl); } - + @Override public String getUrl(String id, String baseUrl) { return baseUrl + VIDEO_ENDPOINT + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index c0773c20..1d5609c1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -1,26 +1,26 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.schabi.newpipe.extractor.ServiceList; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; - public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { - - + + private static final PeertubeTrendingLinkHandlerFactory instance = new PeertubeTrendingLinkHandlerFactory(); - + public static final Map KIOSK_MAP; public static final Map REVERSE_KIOSK_MAP; public static final String KIOSK_TRENDING = "Trending"; public static final String KIOSK_MOST_LIKED = "Most liked"; public static final String KIOSK_RECENT = "Recently added"; public static final String KIOSK_LOCAL = "Local"; - + static { Map map = new HashMap<>(); map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending"); @@ -28,24 +28,24 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); KIOSK_MAP = Collections.unmodifiableMap(map); - + Map reverseMap = new HashMap<>(); - for(Map.Entry entry : KIOSK_MAP.entrySet()){ + for (Map.Entry entry : KIOSK_MAP.entrySet()) { reverseMap.put(entry.getValue(), entry.getKey()); } REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap); } - + public static PeertubeTrendingLinkHandlerFactory getInstance() { return instance; } - + @Override public String getUrl(String id, List contentFilters, String sortFilter) { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return String.format(KIOSK_MAP.get(id), baseUrl); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 54541217..853b6fc1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -86,7 +86,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - if(streamInfoItemsCollector == null) { + if (streamInfoItemsCollector == null) { computeNextPageAndGetStreams(); } return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); @@ -94,7 +94,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getNextPageUrl() throws ExtractionException { - if(nextPageUrl == null) { + if (nextPageUrl == null) { computeNextPageAndGetStreams(); } return nextPageUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java index ed5d9a98..be1b9fcd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index b773a935..b9f4894d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -12,8 +12,8 @@ import javax.annotation.Nonnull; import java.io.IOException; public class SoundcloudChartsExtractor extends KioskExtractor { - private StreamInfoItemsCollector collector = null; - private String nextPageUrl = null; + private StreamInfoItemsCollector collector = null; + private String nextPageUrl = null; public SoundcloudChartsExtractor(StreamingService service, ListLinkHandler linkHandler, @@ -68,7 +68,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public String getNextPageUrl() throws IOException, ExtractionException { - if(nextPageUrl == null) { + if (nextPageUrl == null) { computNextPageAndStreams(); } return nextPageUrl; @@ -77,7 +77,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - if(collector == null) { + if (collector == null) { computNextPageAndStreams(); } return new InfoItemsPage<>(collector, getNextPageUrl()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 82216075..ed50ea54 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -33,7 +33,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 private static String clientId; - + private SoundcloudParsingHelper() { } @@ -103,7 +103,7 @@ public class SoundcloudParsingHelper { /** * Call the endpoint "/resolve" of the api.

    - * + *

    * See https://developers.soundcloud.com/docs/api/reference#resolve */ public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 5bc30972..ca08f66c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java index 17e4e7f2..c7c882a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 9ed340fa..42cd8ae0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -4,7 +4,10 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java index 90057120..dffcc91f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java @@ -25,7 +25,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto try { String url = "https://api-v2.soundcloud.com/search"; - if(contentFilter.size() > 0) { + if (contentFilter.size() > 0) { switch (contentFilter.get(0)) { case TRACKS: url += "/tracks"; @@ -58,7 +58,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, TRACKS, USERS, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index def9c386..30f2fc87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,31 +1,26 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import static java.util.Collections.singletonList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; + public class SoundcloudService extends StreamingService { public SoundcloudService(int id) { super(id, "SoundCloud", singletonList(AUDIO)); } - + @Override public String getBaseUrl() { return "https://soundcloud.com"; @@ -110,15 +105,15 @@ public class SoundcloudService extends StreamingService { return new SoundcloudSubscriptionExtractor(this); } - @Override - public ListLinkHandlerFactory getCommentsLHFactory() { - return null; - } + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return null; + } - @Override + @Override public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { return null; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 18fffa41..2919c89b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -4,7 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -183,7 +185,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { try { JsonObject mp3UrlObject = JsonParser.object().from(res); // Links in this file are also only valid for a short period. - audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), + audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), MediaFormat.MP3, 128)); } catch (JsonParserException e) { throw new ParsingException("Could not parse streamable url", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java index 3476eab1..fc578411 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java index b12f0467..924c4698 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java @@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 6137f029..eae3bcb9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,13 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; - -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -15,28 +7,22 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import javax.annotation.Nonnull; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; /* * Created by Christian Schabesberger on 23.08.15. @@ -125,7 +111,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d675cb25..cc37cbda 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -196,7 +196,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) - throws ParsingException { + throws ParsingException { String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); if (!loadMoreHtmlDataRaw.isEmpty()) { return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 1c377cc5..bcacf7cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -22,7 +21,9 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import static java.util.Collections.singletonList; @@ -58,14 +59,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException { - + JsonArray arr; try { arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); } catch (Exception e) { return ""; } - if(arr.isEmpty()) { + if (arr.isEmpty()) { return ""; } String continuation; @@ -107,11 +108,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException { - + JsonArray contents; try { contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items"); - }catch(Exception e) { + } catch (Exception e) { //no comments return; } @@ -119,12 +120,12 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { List comments; try { comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to get parse youtube comments", e); } - - for(Object c: comments) { - if(c instanceof JsonObject) { + + for (Object c : comments) { + if (c instanceof JsonObject) { CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl(), getTimeAgoParser()); collector.commit(extractor); } @@ -132,7 +133,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void fetchTitle(JsonArray contents) { - if(null == title) { + if (null == title) { try { title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); } catch (Exception e) { @@ -190,7 +191,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { int endIndex = doc.indexOf(end, beginIndex); return doc.substring(beginIndex, endIndex); } - + public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException { try { return JsonUtils.getString(object, "simpleText"); @@ -198,7 +199,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { try { JsonArray arr = JsonUtils.getArray(object, "runs"); String result = ""; - for(int i=0; i getFrames() throws ExtractionException { - try { - final String script = doc.select("#player-api").first().siblingElements().select("script").html(); - int p = script.indexOf("ytplayer.config"); - if (p == -1) { - return Collections.emptyList(); - } - p = script.indexOf('{', p); - int e = script.indexOf("ytplayer.load", p); - if (e == -1) { - return Collections.emptyList(); - } - JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); - final String resp = jo.getObject("args").getString("player_response"); - jo = JsonParser.object().from(resp); - final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); - final String url = spec[0]; - final ArrayList result = new ArrayList<>(spec.length - 1); - for (int i = 1; i < spec.length; ++i) { - final String[] parts = spec[i].split("#"); - if (parts.length != 8) { - continue; - } - final int frameWidth = Integer.parseInt(parts[0]); - final int frameHeight = Integer.parseInt(parts[1]); - final int totalCount = Integer.parseInt(parts[2]); - final int framesPerPageX = Integer.parseInt(parts[3]); - final int framesPerPageY = Integer.parseInt(parts[4]); - final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; - final List urls; - if (baseUrl.contains("$M")) { - final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); - urls = new ArrayList<>(totalPages); - for (int j = 0; j < totalPages; j++) { - urls.add(baseUrl.replace("$M", String.valueOf(j))); - } - } else { - urls = Collections.singletonList(baseUrl); - } - result.add(new Frameset( - urls, - frameWidth, - frameHeight, - totalCount, - framesPerPageX, - framesPerPageY - )); - } - result.trimToSize(); - return result; - } catch (Exception e) { - throw new ExtractionException(e); - } - } + @Nonnull + @Override + public List getFrames() throws ExtractionException { + try { + final String script = doc.select("#player-api").first().siblingElements().select("script").html(); + int p = script.indexOf("ytplayer.config"); + if (p == -1) { + return Collections.emptyList(); + } + p = script.indexOf('{', p); + int e = script.indexOf("ytplayer.load", p); + if (e == -1) { + return Collections.emptyList(); + } + JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); + final String resp = jo.getObject("args").getString("player_response"); + jo = JsonParser.object().from(resp); + final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); + final String url = spec[0]; + final ArrayList result = new ArrayList<>(spec.length - 1); + for (int i = 1; i < spec.length; ++i) { + final String[] parts = spec[i].split("#"); + if (parts.length != 8) { + continue; + } + final int frameWidth = Integer.parseInt(parts[0]); + final int frameHeight = Integer.parseInt(parts[1]); + final int totalCount = Integer.parseInt(parts[2]); + final int framesPerPageX = Integer.parseInt(parts[3]); + final int framesPerPageY = Integer.parseInt(parts[4]); + final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; + final List urls; + if (baseUrl.contains("$M")) { + final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); + urls = new ArrayList<>(totalPages); + for (int j = 0; j < totalPages; j++) { + urls.add(baseUrl.replace("$M", String.valueOf(j))); + } + } else { + urls = Collections.singletonList(baseUrl); + } + result.add(new Frameset( + urls, + frameWidth, + frameHeight, + totalCount, + framesPerPageX, + framesPerPageY + )); + } + result.trimToSize(); + return result; + } catch (Exception e) { + throw new ExtractionException(e); + } + } @Override public String getHost() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 514b4fb0..eda36d21 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -42,6 +42,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { /** * Creates an extractor of StreamInfoItems from a YouTube page. + * * @param item The page element * @param timeAgoParser A parser of the textual dates or {@code null}. */ @@ -68,10 +69,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private boolean isPremiumVideo() { Element premiumSpan = item.select("span[class=\"standalone-collection-badge-renderer-red-text\"]").first(); - if(premiumSpan == null) return false; + if (premiumSpan == null) return false; // if this span has text it most likely says ("Free Video") so we can play this - if(premiumSpan.hasText()) return false; + if (premiumSpan.hasText()) return false; return true; } @@ -233,7 +234,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return Long.parseLong(Utils.removeNonDigitCharacters(input)); } catch (NumberFormatException e) { // if this happens the video probably has no views - if (!input.isEmpty()){ + if (!input.isEmpty()) { return 0; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index d41d90c5..8ed35c19 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -3,9 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; @@ -57,12 +57,12 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { String response = dl.get(url, getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" - response = response.substring(3, response.length()-1); + response = response.substring(3, response.length() - 1); try { JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray()); for (Object suggestion : collection) { if (!(suggestion instanceof JsonArray)) continue; - String suggestionStr = ((JsonArray)suggestion).getString(0); + String suggestionStr = ((JsonArray) suggestion).getString(0); if (suggestionStr == null) continue; suggestions.add(suggestionStr); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index a4014191..be5820de 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -88,8 +88,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for(Element ul : uls) { - for(final Element li : ul.children()) { + for (Element ul : uls) { + for (final Element li : ul.children()) { final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 62a0e737..29bbdfdf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -31,7 +31,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { } String path = urlObj.getPath(); - if (!path.equals("/watch" ) && !path.equals("/playlist")) { + if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index ba5ee152..c1760074 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -26,7 +26,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory final String url = "https://www.youtube.com/results" + "?q=" + URLEncoder.encode(searchString, CHARSET_UTF_8); - if(contentFilters.size() > 0) { + if (contentFilters.size() > 0) { switch (contentFilters.get(0)) { case VIDEOS: return url + "&sp=EgIQAVAU"; case CHANNELS: return url + "&sp=EgIQAlAU"; @@ -44,7 +44,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, VIDEOS, CHANNELS, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java index 2d4010dd..cc92cd7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java @@ -1,66 +1,64 @@ package org.schabi.newpipe.extractor.stream; -import javax.annotation.Nullable; -import java.util.Collection; import java.util.List; public final class Frameset { - private List urls; - private int frameWidth; - private int frameHeight; - private int totalCount; - private int framesPerPageX; - private int framesPerPageY; + private List urls; + private int frameWidth; + private int frameHeight; + private int totalCount; + private int framesPerPageX; + private int framesPerPageY; - public Frameset(List urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) { - this.urls = urls; - this.totalCount = totalCount; - this.frameWidth = frameWidth; - this.frameHeight = frameHeight; - this.framesPerPageX = framesPerPageX; - this.framesPerPageY = framesPerPageY; - } + public Frameset(List urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) { + this.urls = urls; + this.totalCount = totalCount; + this.frameWidth = frameWidth; + this.frameHeight = frameHeight; + this.framesPerPageX = framesPerPageX; + this.framesPerPageY = framesPerPageY; + } - /** - * @return list of urls to images with frames - */ - public List getUrls() { - return urls; - } + /** + * @return list of urls to images with frames + */ + public List getUrls() { + return urls; + } - /** - * @return total count of frames - */ - public int getTotalCount() { - return totalCount; - } + /** + * @return total count of frames + */ + public int getTotalCount() { + return totalCount; + } - /** - * @return maximum frames count by x - */ - public int getFramesPerPageX() { - return framesPerPageX; - } + /** + * @return maximum frames count by x + */ + public int getFramesPerPageX() { + return framesPerPageX; + } - /** - * @return maximum frames count by y - */ - public int getFramesPerPageY() { - return framesPerPageY; - } + /** + * @return maximum frames count by y + */ + public int getFramesPerPageY() { + return framesPerPageY; + } - /** - * @return width of a one frame, in pixels - */ - public int getFrameWidth() { - return frameWidth; - } + /** + * @return width of a one frame, in pixels + */ + public int getFrameWidth() { + return frameWidth; + } - /** - * @return height of a one frame, in pixels - */ - public int getFrameHeight() { - return frameHeight; - } + /** + * @return height of a one frame, in pixels + */ + public int getFrameHeight() { + return frameHeight; + } } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 1d49e253..b48eeab2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.MediaFormat; + import java.io.Serializable; import java.util.List; -import org.schabi.newpipe.extractor.MediaFormat; - /** * Creates a stream object from url, format and optional torrent url */ @@ -22,19 +22,19 @@ public abstract class Stream implements Serializable { /** * Instantiates a new stream object. * - * @param url the url + * @param url the url * @param format the format */ public Stream(String url, MediaFormat format) { this(url, null, format); } - + /** * Instantiates a new stream object. * - * @param url the url + * @param url the url * @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent - * @param format the format + * @param format the format */ public Stream(String url, String torrentUrl, MediaFormat format) { this.url = url; @@ -76,7 +76,7 @@ public abstract class Stream implements Serializable { public String getUrl() { return url; } - + /** * Gets the torrent url. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index fdc56972..587101cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -27,7 +27,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; @@ -77,6 +76,7 @@ public abstract class StreamExtractor extends Extractor { /** * This will return the url to the thumbnail of the stream. Try to return the medium resolution here. + * * @return The url of the thumbnail. * @throws ParsingException */ @@ -85,6 +85,7 @@ public abstract class StreamExtractor extends Extractor { /** * This is the stream description. + * * @return The description of the stream/video or Description.emptyDescription if the description is empty. * @throws ParsingException */ @@ -93,6 +94,7 @@ public abstract class StreamExtractor extends Extractor { /** * Get the age limit. + * * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit * @throws ParsingException if an error occurs while parsing */ @@ -100,6 +102,7 @@ public abstract class StreamExtractor extends Extractor { /** * This should return the length of a video in seconds. + * * @return The length of the stream in seconds. * @throws ParsingException */ @@ -109,6 +112,7 @@ public abstract class StreamExtractor extends Extractor { * If the url you are currently handling contains a time stamp/seek, you can return the * position it represents here. * If the url has no time stamp simply return zero. + * * @return the timestamp in seconds * @throws ParsingException */ @@ -117,6 +121,7 @@ public abstract class StreamExtractor extends Extractor { /** * The count of how many people have watched the video/listened to the audio stream. * If the current stream has no view count or its not available simply return -1 + * * @return amount of views. * @throws ParsingException */ @@ -125,6 +130,7 @@ public abstract class StreamExtractor extends Extractor { /** * The Amount of likes a video/audio stream got. * If the current stream has no likes or its not available simply return -1 + * * @return the amount of likes the stream got * @throws ParsingException */ @@ -133,6 +139,7 @@ public abstract class StreamExtractor extends Extractor { /** * The Amount of dislikes a video/audio stream got. * If the current stream has no dislikes or its not available simply return -1 + * * @return the amount of likes the stream got * @throws ParsingException */ @@ -144,6 +151,7 @@ public abstract class StreamExtractor extends Extractor { * ChannelExtractor, * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects * this url. + * * @return the url to the page of the creator/uploader of the stream or an empty String * @throws ParsingException */ @@ -153,6 +161,7 @@ public abstract class StreamExtractor extends Extractor { /** * The name of the creator/uploader of the stream. * If the name is not available you can simply return an empty string. + * * @return the name of the creator/uploader of the stream or an empty String * @throws ParsingException */ @@ -162,6 +171,7 @@ public abstract class StreamExtractor extends Extractor { /** * The url to the image file/profile picture/avatar of the creator/uploader of the stream. * If the url is not available you can return an empty String. + * * @return The url of the image file of the uploader or an empty String * @throws ParsingException */ @@ -171,20 +181,24 @@ public abstract class StreamExtractor extends Extractor { /** * Get the dash mpd url. If you don't know what a dash MPD is you can read about it * here. + * * @return the url as a string or an empty string * @throws ParsingException if an error occurs while reading */ - @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + @Nonnull + public abstract String getDashMpdUrl() throws ParsingException; /** * I am not sure if this is in use, and how this is used. However the frontend is missing support * for HLS streams. Prove me if I am wrong. Please open an * issue, * or fix this description if you know whats up with this. + * * @return The Url to the hls stream. * @throws ParsingException */ - @Nonnull public abstract String getHlsUrl() throws ParsingException; + @Nonnull + public abstract String getHlsUrl() throws ParsingException; /** * This should return a list of available @@ -192,6 +206,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of audio only streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -205,6 +220,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of combined video and streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -218,6 +234,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of video and streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -228,6 +245,7 @@ public abstract class StreamExtractor extends Extractor { * This will return a list of available * Subtitless. * If no subtitles are available an empty list can returned. + * * @return a list of available subtitles or an empty list * @throws IOException * @throws ExtractionException @@ -240,6 +258,7 @@ public abstract class StreamExtractor extends Extractor { * Subtitless. * given by a specific type. * If no subtitles in that specific format are available an empty list can returned. + * * @param format the media format by which the subtitles should be filtered * @return a list of available subtitles or an empty list * @throws IOException @@ -250,6 +269,7 @@ public abstract class StreamExtractor extends Extractor { /** * Get the StreamType. + * * @return the type of the stream * @throws ParsingException */ @@ -259,6 +279,7 @@ public abstract class StreamExtractor extends Extractor { * should return the url of the next stream. NewPipe will automatically play * the next stream if the user wants that. * If the next stream is is not available simply return null + * * @return the InfoItem of the next stream * @throws IOException * @throws ExtractionException @@ -271,6 +292,7 @@ public abstract class StreamExtractor extends Extractor { * be disabled by the user later in the frontend. * This list MUST NOT contain the next available video as this should be return through getNextStream() * If is is not available simply return null + * * @return a list of InfoItems showing the related videos/streams * @throws IOException * @throws ExtractionException @@ -279,6 +301,7 @@ public abstract class StreamExtractor extends Extractor { /** * Should return a list of Frameset object that contains preview of stream frames + * * @return list of preview frames or empty list if frames preview is not supported or not found for specified stream * @throws IOException * @throws ExtractionException @@ -302,6 +325,7 @@ public abstract class StreamExtractor extends Extractor { /** * Override this function if the format of time stamp in the url is not the same format as that form youtube. * Honestly I don't even know the time stamp fromat of youtube. + * * @param regexPattern * @return the sime stamp/seek for the video in seconds * @throws ParsingException @@ -357,6 +381,7 @@ public abstract class StreamExtractor extends Extractor { * If the host is not available, or if the service doesn't use * a federated system, but a centralised system, * you can simply return an empty string. + * * @return the host of the stream or an empty String. * @throws ParsingException */ @@ -366,6 +391,7 @@ public abstract class StreamExtractor extends Extractor { /** * The privacy of the stream (Eg. Public, Private, Unlisted…). * If the privacy is not available you can simply return an empty string. + * * @return the privacy of the stream or an empty String. * @throws ParsingException */ @@ -375,6 +401,7 @@ public abstract class StreamExtractor extends Extractor { /** * The name of the category of the stream. * If the category is not available you can simply return an empty string. + * * @return the category of the stream or an empty String. * @throws ParsingException */ @@ -384,6 +411,7 @@ public abstract class StreamExtractor extends Extractor { /** * The name of the licence of the stream. * If the licence is not available you can simply return an empty string. + * * @return the licence of the stream or an empty String. * @throws ParsingException */ @@ -395,6 +423,7 @@ public abstract class StreamExtractor extends Extractor { * If the language is not available you can simply return null. * If the language is provided by a language code, you can return * new Locale(language_code); + * * @return the locale language of the stream or null. * @throws ParsingException */ @@ -404,6 +433,7 @@ public abstract class StreamExtractor extends Extractor { /** * The list of tags of the stream. * If the tag list is not available you can simply return an empty list. + * * @return the list of tags of the stream or an empty list. * @throws ParsingException */ @@ -416,6 +446,7 @@ public abstract class StreamExtractor extends Extractor { * (support button). * If the support information are not available, * you can simply return an empty String. + * * @return the support information of the stream or an empty String. * @throws ParsingException */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 113f3a87..828d4520 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -31,6 +31,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream type + * * @return the stream type * @throws ParsingException thrown if there is an error in the extraction */ @@ -38,6 +39,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Check if the stream is an ad. + * * @return {@code true} if the stream is an ad. * @throws ParsingException thrown if there is an error in the extraction */ @@ -45,6 +47,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream duration in seconds + * * @return the stream duration in seconds * @throws ParsingException thrown if there is an error in the extraction */ @@ -52,6 +55,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Parses the number of views + * * @return the number of views or -1 for live streams * @throws ParsingException thrown if there is an error in the extraction */ @@ -59,6 +63,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the uploader name + * * @return the uploader name * @throws ParsingException if parsing fails */ @@ -80,9 +85,9 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Extracts the upload date and time of this item and parses it. *

    - * If the service doesn't provide an exact time, an approximation can be returned. - *
    - * If the service doesn't provide any date at all, then {@code null} should be returned. + * If the service doesn't provide an exact time, an approximation can be returned. + *
    + * If the service doesn't provide any date at all, then {@code null} should be returned. *

    * * @return The date and time (can be approximated) this item was uploaded or {@code null}. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 85dce8f0..b81052b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -101,8 +101,8 @@ public class StreamInfoItemsCollector extends InfoItemsCollector getStreamInfoItemList() { List siiList = new Vector<>(); - for(InfoItem ii : super.getItems()) { - if(ii instanceof StreamInfoItem) { + for (InfoItem ii : super.getItems()) { + if (ii instanceof StreamInfoItem) { siiList.add((StreamInfoItem) ii); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java index d0e09ac1..8f16942e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -52,7 +52,7 @@ public class SubtitlesStream extends Stream implements Serializable { @Override public boolean equalStats(Stream cmp) { - return super.equalStats(cmp)&& + return super.equalStats(cmp) && cmp instanceof SubtitlesStream && code.equals(((SubtitlesStream) cmp).code) && autoGenerated == ((SubtitlesStream) cmp).autoGenerated; @@ -67,7 +67,7 @@ public class SubtitlesStream extends Stream implements Serializable { } public Locale getLocale() { - return locale; + return locale; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java index f4531d36..67219240 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java @@ -36,7 +36,7 @@ public class VideoStream extends Stream { this.resolution = resolution; this.isVideoOnly = isVideoOnly; } - + public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) { this(url, torrentUrl, format, resolution, false); } @@ -56,6 +56,7 @@ public class VideoStream extends Stream { /** * Get the video resolution + * * @return the video resolution */ public String getResolution() { @@ -64,8 +65,9 @@ public class VideoStream extends Stream { /** * Check if the video is video only. - * + *

    * Video only streams have no audio + * * @return {@code true} if this stream is vid */ public boolean isVideoOnly() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 7b954a4c..b8b191ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor.utils; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.services.youtube.ItagItem; @@ -109,11 +109,11 @@ public class DashMpdParser { *

    * It has video, video only and audio streams and will only add to the list if it don't * find a similar stream in the respective lists (calling {@link Stream#equalStats}). - * + *

    * Info about dash MPD can be found here - * @see www.brendanlog.com * * @param streamInfo where the parsed streams will be added + * @see www.brendanlog.com */ public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { @@ -160,7 +160,7 @@ public class DashMpdParser { final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - if(segmentationList == null) { + if (segmentationList == null) { final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { audioStreams.add(audioStream); @@ -172,7 +172,7 @@ public class DashMpdParser { } else { boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - if(segmentationList == null) { + if (segmentationList == null) { final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, @@ -191,7 +191,7 @@ public class DashMpdParser { itag.resolutionString, isVideoOnly); - if(isVideoOnly) { + if (isVideoOnly) { segmentedVideoOnlyStreams.add(videoStream); } else { segmentedVideoStreams.add(videoStream); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index 28300649..c6ef73ca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; -import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; @@ -31,7 +30,7 @@ public class ExtractorHelper { public static List getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { try { InfoItemsCollector collector = extractor.getRelatedStreams(); - if(collector == null) return Collections.emptyList(); + if (collector == null) return Collections.emptyList(); info.addAllErrors(collector.getErrors()); //noinspection unchecked @@ -41,5 +40,5 @@ public class ExtractorHelper { return Collections.emptyList(); } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index 80e0dd58..ab916379 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -1,16 +1,14 @@ package org.schabi.newpipe.extractor.utils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; public class JsonUtils { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index b03e304c..30786420 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -1,5 +1,10 @@ package org.schabi.newpipe.extractor.utils; +import org.nibor.autolink.LinkExtractor; +import org.nibor.autolink.LinkSpan; +import org.nibor.autolink.LinkType; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; @@ -9,11 +14,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.nibor.autolink.LinkExtractor; -import org.nibor.autolink.LinkSpan; -import org.nibor.autolink.LinkType; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - /* * Created by Christian Schabesberger on 02.02.16. * @@ -51,7 +51,7 @@ public class Parser { public static String matchGroup1(String pattern, String input) throws RegexException { return matchGroup(pattern, input, 1); } - + public static String matchGroup1(Pattern pattern, String input) throws RegexException { return matchGroup(pattern, input, 1); } @@ -60,7 +60,7 @@ public class Parser { Pattern pat = Pattern.compile(pattern); return matchGroup(pat, input, group); } - + public static String matchGroup(Pattern pat, String input, int group) throws RegexException { Matcher mat = pat.matcher(input); boolean foundMatch = mat.find(); @@ -102,7 +102,7 @@ public class Parser { .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW)) .build(); Iterable linkss = linkExtractor.extractLinks(txt); - for(LinkSpan ls : linkss) { + for (LinkSpan ls : linkss) { links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex())); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 3489b6d6..ebd0ba16 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.utils; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.List; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - public class Utils { private Utils() { @@ -35,6 +35,7 @@ public class Utils { *

  • 1.23K -> 1230
    • *
  • 1.23M -> 1230000
    • * + * * @param numberWord string to be converted to a long * @return a long * @throws NumberFormatException @@ -165,17 +166,17 @@ public class Utils { return setsNoPort || usesDefaultPort; } - + public static String removeUTF8BOM(String s) { if (s.startsWith("\uFEFF")) { s = s.substring(1); } if (s.endsWith("\uFEFF")) { - s = s.substring(0, s.length()-1); + s = s.substring(0, s.length() - 1); } return s; } - + public static String getBaseUrl(String url) throws ParsingException { URL uri; try { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index b9c67c4e..15a68a73 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -45,6 +45,6 @@ public class MediaCCCConferenceExtractorTest { @Test public void testGetInitalPage() throws Exception { - assertEquals(97,extractor.getInitialPage().getItems().size()); + assertEquals(97, extractor.getInitialPage().getItems().size()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java index 252e4dbb..90ebf028 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java @@ -24,7 +24,7 @@ public class MediaCCCConferenceListExtractorTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); + extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); extractor.fetchPage(); } @@ -49,8 +49,8 @@ public class MediaCCCConferenceListExtractorTest { } private boolean contains(List itemList, String name) { - for(InfoItem item : itemList) { - if(item.getName().equals(name)) + for (InfoItem item : itemList) { + if (item.getName().equals(name)) return true; } return false; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java index 781f9138..25116af3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java @@ -22,7 +22,7 @@ public class MediaCCCOggTest { public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); + extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); extractor.fetchPage(); } @@ -33,7 +33,7 @@ public class MediaCCCOggTest { @Test public void getAudioStreamsContainOgg() throws Exception { - for(AudioStream stream : extractor.getAudioStreams()) { + for (AudioStream stream : extractor.getAudioStreams()) { assertEquals("OGG", stream.getFormat().toString()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java index dc38338e..cd33c562 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java @@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorAllTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("c3", Arrays.asList(new String[0]), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -37,8 +37,8 @@ public class MediaCCCSearchExtractorAllTest { @Test public void testIfChannelInfoItemsAvailable() { boolean isAvialable = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof ChannelInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof ChannelInfoItem) { isAvialable = true; } } @@ -48,8 +48,8 @@ public class MediaCCCSearchExtractorAllTest { @Test public void testIfStreamInfoitemsAvailable() { boolean isAvialable = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { isAvialable = true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java index 18fbc234..f9ce2334 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java @@ -27,7 +27,7 @@ public class MediaCCCSearchExtractorConferencesTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("c3", Arrays.asList(new String[]{"conferences"}), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -35,7 +35,7 @@ public class MediaCCCSearchExtractorConferencesTest { @Test public void testReturnTypeChannel() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { assertTrue("Item is not of type channel", item instanceof ChannelInfoItem); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java index 1ed65a8d..29f90caf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java @@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorEventsTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("linux", Arrays.asList(new String[]{"events"}), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -65,7 +65,7 @@ public class MediaCCCSearchExtractorEventsTest { @Test public void testReturnTypeStream() throws Exception { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 71b1e779..ed9dcee7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -28,7 +28,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest { public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); + extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 57b6360b..a1d1f383 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -1,15 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; - import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -20,6 +10,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + /** * Test for {@link PeertubeChannelExtractor} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index 633307a2..f9b0c69d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeChannelLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index e0a1d189..1241b0a8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -1,12 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.io.IOException; -import java.util.List; - import org.jsoup.helper.StringUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -18,6 +11,13 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + public class PeertubeCommentsExtractorTest { private static PeertubeCommentsExtractor extractor; @@ -59,11 +59,11 @@ public class PeertubeCommentsExtractorTest { assertTrue(result); } - + @Test public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); - for(CommentsInfoItem c: comments.getItems()) { + for (CommentsInfoItem c : comments.getItems()) { assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); assertFalse(StringUtil.isBlank(c.getAuthorName())); assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); @@ -82,8 +82,8 @@ public class PeertubeCommentsExtractorTest { } private boolean findInComments(List comments, String comment) { - for(CommentsInfoItem c: comments) { - if(c.getCommentText().contains(comment)) { + for (CommentsInfoItem c : comments) { + if (c.getCommentText().contains(comment)) { return true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java index fcc27c54..f6fb91f7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeCommentsLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java index 9ad29102..ac04a01e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubePlaylistLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index a81bf34c..23adf353 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -1,19 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Locale; -import java.util.TimeZone; - import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -26,6 +12,18 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; + +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link StreamExtractor} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java index 893b6eb8..af44f644 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeStreamLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index bb4e1529..5d2d8478 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -1,13 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.util.List; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -17,6 +9,11 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.util.List; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeTrendingExtractor} */ @@ -53,10 +50,10 @@ public class PeertubeTrendingExtractorTest { @Test public void testGetStreams() throws Exception { ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if(!page.getErrors().isEmpty()) { + if (!page.getErrors().isEmpty()) { System.err.println("----------"); List errors = page.getErrors(); - for(Throwable e: errors) { + for (Throwable e : errors) { e.printStackTrace(); System.err.println("----------"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java index d192e8be..033be650 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -1,9 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeTrendingLinkHandlerFactory} */ @@ -48,13 +48,13 @@ public class PeertubeTrendingLinkHandlerFactoryTest { public void acceptUrl() throws ParsingException { assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe")); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java index 08dd7a0a..030bd6c5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertTrue; - import org.junit.Test; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeSearchExtractor} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java index dd9c6deb..eeebf526 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -15,6 +10,9 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeSearchExtractor} */ @@ -38,15 +36,15 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB @Test public void testResultList_FirstElement() { InfoItem firstInfoItem = itemsPage.getItems().get(0); - + assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde")); } @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -67,7 +65,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } @@ -75,7 +73,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB assertFalse("First and second page are equal", equals); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", - secondPage.getNextPageUrl()); + secondPage.getNextPageUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java index 23e3100c..d7bd92aa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertEquals; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + public class PeertubeSearchQHTest { - + @BeforeClass public static void setUpClass() throws Exception { // setting instance might break test when running in parallel @@ -18,7 +18,7 @@ public class PeertubeSearchQHTest { @Test public void testRegularValues() throws Exception { assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans",PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans", PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index d96b4c6e..dea1d170 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; @@ -48,10 +47,10 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetStreams() throws Exception { ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if(!page.getErrors().isEmpty()) { + if (!page.getErrors().isEmpty()) { System.err.println("----------"); List errors = page.getErrors(); - for(Throwable e: errors) { + for (Throwable e : errors) { e.printStackTrace(); System.err.println("----------"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index 1c4f4159..16089aab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -1,10 +1,12 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.junit.*; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; public class SoundcloudParsingHelperTest { @BeforeClass diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java index abd26951..04e7e0ab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java @@ -38,7 +38,7 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } @@ -57,8 +57,8 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx @Test public void testOnlyContainChannels() { - for(InfoItem item : itemsPage.getItems()) { - if(!(item instanceof ChannelInfoItem)) { + for (InfoItem item : itemsPage.getItems()) { + if (!(item instanceof ChannelInfoItem)) { fail("The following item is no channel item: " + item.toString()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java index 685c52d7..ca0d7d59 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java @@ -60,8 +60,8 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -80,7 +80,7 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 251ae246..317bd4fa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,14 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -16,10 +7,14 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + /** * Test for {@link ChannelExtractor} */ @@ -489,7 +484,6 @@ public class YoutubeChannelExtractorTest { } - public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index 25f097a0..fc409ffa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -38,10 +38,10 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); - + assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh")); assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos")); - + assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); } @@ -60,10 +60,10 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); - + assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId()); assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId()); - + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 796238f9..aebb6bf1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -123,7 +123,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId()); } - + @Test public void testAcceptInvidioUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU")); @@ -133,7 +133,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE")); } - + @Test public void testGetInvidioIdfromUrl() throws ParsingException { assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java index c8e23c06..c85d6b52 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java @@ -1,24 +1,24 @@ package org.schabi.newpipe.extractor.services.youtube; - /* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2017 - * YoutubeTrendingKioskInfoTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingKioskInfoTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ import org.junit.BeforeClass; import org.junit.Test; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java index 8b19b11e..729254ad 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java @@ -3,10 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.Test; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 91cbebd9..63fc0375 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -57,7 +57,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto @Ignore @Test public void testOnlyContainChannels() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { if (!(item instanceof ChannelInfoItem)) { fail("The following item is no channel item: " + item.toString()); } @@ -66,7 +66,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto @Test public void testChannelUrl() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { if (item instanceof ChannelInfoItem) { ChannelInfoItem channel = (ChannelInfoItem) item; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 49e3520d..3078d873 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -6,7 +6,6 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -76,8 +75,8 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -96,7 +95,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java index 7df7b0cb..656e197a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java @@ -1,15 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.search; -import static java.util.Collections.singletonList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -19,6 +9,14 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Collections.singletonList; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + public class YoutubeSearchPagingTest { private static ListExtractor.InfoItemsPage page1; private static ListExtractor.InfoItemsPage page2; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index c3d3f93c..60bbf2ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -1,21 +1,18 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.Test; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; public class YoutubeSearchQHTest { @Test public void testRegularValues() throws Exception { assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans",YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 6afc7212..3710b4ee 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -117,7 +117,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { streams.addAll(extractor.getVideoStreams()); streams.addAll(extractor.getVideoOnlyStreams()); - assertTrue(Integer.toString(streams.size()),streams.size() > 0); + assertTrue(Integer.toString(streams.size()), streams.size() > 0); for (VideoStream s : streams) { assertTrue(s.getUrl(), s.getUrl().contains(HTTPS)); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index e2c0c7bb..2798e8db 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -19,7 +19,7 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; -import static java.util.Objects.*; +import static java.util.Objects.requireNonNull; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -126,7 +126,7 @@ public class YoutubeStreamExtractorDefaultTest { public void testGetUploaderUrl() throws ParsingException { String url = extractor.getUploaderUrl(); if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") && - !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { + !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { fail("Uploader url is neither the music channel one nor the Vevo one"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java index dc8f2b04..925a647c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java @@ -1,47 +1,46 @@ package org.schabi.newpipe.extractor.utils; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.junit.Test; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.junit.Test; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.util.List; + +import static org.junit.Assert.assertTrue; public class JsonUtilsTest { - + @Test public void testGetValueFlat() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); - assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); + assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); } - + @Test public void testGetValueNested() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2"))); } - + @Test public void testGetArray() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "batters.batter"); assertTrue(!arr.isEmpty()); } - + @Test public void testGetValues() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping"); List types = JsonUtils.getValues(arr, "type"); assertTrue(types.contains("Chocolate with Sprinkles")); - + } } From 6822574588cde36580cfd564363dc632e0f7446a Mon Sep 17 00:00:00 2001 From: Tobias Groza Date: Mon, 10 Feb 2020 09:50:19 +0100 Subject: [PATCH 38/40] Fix typos Co-Authored-By: B0pol --- .../org/schabi/newpipe/extractor/MediaFormat.java | 2 +- .../newpipe/extractor/comments/CommentsInfo.java | 1 - .../linkhandler/ListLinkHandlerFactory.java | 2 +- .../linkhandler/SearchQueryHandlerFactory.java | 2 +- .../extractors/PeertubeChannelExtractor.java | 6 +++--- .../extractors/PeertubePlaylistExtractor.java | 1 - .../extractors/PeertubeTrendingExtractor.java | 2 +- .../youtube/extractors/YoutubePlaylistExtractor.java | 2 +- .../newpipe/extractor/stream/StreamExtractor.java | 12 ++++++------ 9 files changed, 14 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index 0b6052f0..6936568a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -115,7 +115,7 @@ public enum MediaFormat { } /** - * Get the media format by it's id. + * Get the media format by its id. * * @param id the id * @return the id of the media format or null. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java index d5f4c434..e07c6746 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java @@ -14,7 +14,6 @@ public class CommentsInfo extends ListInfo { private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { super(serviceId, listUrlIdHandler, name); - // TODO Auto-generated constructor stub } public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 60e1db48..9ea478b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -69,7 +69,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /** - * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, + * For making ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, * however it should not be overridden by the actual implementation. * * @param id diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java index 55c76b78..50977e20 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java @@ -39,7 +39,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { } /** - * It's not mandatorry for NewPipe to handle the Url + * It's not mandatory for NewPipe to handle the Url * * @param url * @return diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 4d1d00f5..6476be8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -124,7 +124,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { - throw new ExtractionException("Unable to get peertube kiosk info"); + throw new ExtractionException("Unable to get PeerTube kiosk info"); } return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); } @@ -158,7 +158,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); } else { - throw new ExtractionException("Unable to extract peertube channel data"); + throw new ExtractionException("Unable to extract PeerTube channel data"); } String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; @@ -171,7 +171,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube channel data", e); } - if (null == json) throw new ExtractionException("Unable to extract peertube channel data"); + if (null == json) throw new ExtractionException("Unable to extract PeerTube channel data"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 81cd165d..1a96111a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -73,7 +73,6 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { - // TODO Auto-generated method stub } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index e6c1a1b5..e4bc1dff 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -49,7 +49,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { try { contents = (JsonArray) JsonUtils.getValue(json, "data"); } catch (Exception e) { - throw new ParsingException("unable to extract kiosk info", e); + throw new ParsingException("Unable to extract kiosk info", e); } String baseUrl = getBaseUrl(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 0f3f56e6..a03b1d5a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -68,7 +68,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getBannerUrl() { return ""; // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement this in the fornt end + // Whoever is willing to implement this should also implement this in the front end. } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 587101cf..4a3cdff4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -128,7 +128,7 @@ public abstract class StreamExtractor extends Extractor { public abstract long getViewCount() throws ParsingException; /** - * The Amount of likes a video/audio stream got. + * The amount of likes a video/audio stream got. * If the current stream has no likes or its not available simply return -1 * * @return the amount of likes the stream got @@ -137,7 +137,7 @@ public abstract class StreamExtractor extends Extractor { public abstract long getLikeCount() throws ParsingException; /** - * The Amount of dislikes a video/audio stream got. + * The amount of dislikes a video/audio stream got. * If the current stream has no dislikes or its not available simply return -1 * * @return the amount of likes the stream got @@ -276,7 +276,7 @@ public abstract class StreamExtractor extends Extractor { public abstract StreamType getStreamType() throws ParsingException; /** - * should return the url of the next stream. NewPipe will automatically play + * Should return the url of the next stream. NewPipe will automatically play * the next stream if the user wants that. * If the next stream is is not available simply return null * @@ -291,7 +291,7 @@ public abstract class StreamExtractor extends Extractor { * streams. If you don't like suggested streams you should implement them anyway since they can * be disabled by the user later in the frontend. * This list MUST NOT contain the next available video as this should be return through getNextStream() - * If is is not available simply return null + * If it is not available simply return null * * @return a list of InfoItems showing the related videos/streams * @throws IOException @@ -324,10 +324,10 @@ public abstract class StreamExtractor extends Extractor { /** * Override this function if the format of time stamp in the url is not the same format as that form youtube. - * Honestly I don't even know the time stamp fromat of youtube. + * Honestly I don't even know the time stamp format of YouTube. * * @param regexPattern - * @return the sime stamp/seek for the video in seconds + * @return the time stamp/seek for the video in seconds * @throws ParsingException */ protected long getTimestampSeconds(String regexPattern) throws ParsingException { From 0a4115286b46356b34bbcb2308261f86acd380c8 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 15 Feb 2020 12:52:29 +0100 Subject: [PATCH 39/40] Apply one suggestion --- .../org/schabi/newpipe/extractor/stream/StreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 4a3cdff4..900707ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -337,7 +337,7 @@ public abstract class StreamExtractor extends Extractor { } catch (Parser.RegexException e) { // catch this instantly since an url does not necessarily have to have a time stamp - // -2 because well the testing system will then know its the regex that failed :/ + // -2 because the testing system will consequently know that the regex failed // not good i know return -2; } From a129c65c63ee95462239b9681bf14c1562abfe07 Mon Sep 17 00:00:00 2001 From: Tobias Groza Date: Sun, 16 Feb 2020 23:59:31 +0100 Subject: [PATCH 40/40] more improvements of comments Co-Authored-By: Stypox --- .../java/org/schabi/newpipe/extractor/StreamingService.java | 2 +- .../newpipe/extractor/services/peertube/PeertubeInstance.java | 2 +- .../peertube/extractors/PeertubeChannelExtractor.java | 2 +- .../services/youtube/extractors/YoutubeCommentsExtractor.java | 2 +- .../services/youtube/extractors/YoutubePlaylistExtractor.java | 2 +- .../org/schabi/newpipe/extractor/stream/StreamExtractor.java | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 110efe04..e21b17f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -263,7 +263,7 @@ public abstract class StreamingService { public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException { ListLinkHandlerFactory llhf = getCommentsLHFactory(); - if (null == llhf) { + if (llhf == null) { return null; } return getCommentsExtractor(llhf.fromUrl(url)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 17d08580..82b67539 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -43,7 +43,7 @@ public class PeertubeInstance { throw new Exception("unable to configure instance " + url, e); } - if (null == response || StringUtil.isBlank(response.responseBody())) { + if (response == null || StringUtil.isBlank(response.responseBody())) { throw new Exception("unable to configure instance " + url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 6476be8b..d53fb7ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -171,7 +171,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube channel data", e); } - if (null == json) throw new ExtractionException("Unable to extract PeerTube channel data"); + if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index bcacf7cf..78eb59f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -133,7 +133,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void fetchTitle(JsonArray contents) { - if (null == title) { + if (title == null) { try { title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index a03b1d5a..520bda80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -68,7 +68,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getBannerUrl() { return ""; // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement this in the front end. + // Whoever is willing to implement this should also implement it in the frontend. } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 900707ab..78a5fbf3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -335,10 +335,10 @@ public abstract class StreamExtractor extends Extractor { try { timeStamp = Parser.matchGroup1(regexPattern, getOriginalUrl()); } catch (Parser.RegexException e) { - // catch this instantly since an url does not necessarily have to have a time stamp + // catch this instantly since a url does not necessarily have a timestamp // -2 because the testing system will consequently know that the regex failed - // not good i know + // not good, I know return -2; }