From a281519e771d139e7d6cd1b7f64dece92cbb5124 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 19 Jan 2020 12:45:52 +0100 Subject: [PATCH 001/149] added metadata, fix descriptions, fix thumbnail, update tests thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails) test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted) metadata: got host, privacy (public, private, unlisted), licence, language, tags --- README.md | 2 +- .../extractors/MediaCCCStreamExtractor.java | 32 +++++++ .../extractors/PeertubeStreamExtractor.java | 50 +++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 31 +++++++ .../extractors/YoutubeStreamExtractor.java | 31 +++++++ .../extractor/stream/StreamExtractor.java | 57 ++++++++++++ .../newpipe/extractor/stream/StreamInfo.java | 88 +++++++++++++++++++ .../schabi/newpipe/extractor/utils/Utils.java | 1 - .../PeertubeCommentsExtractorTest.java | 4 +- .../PeertubeStreamExtractorDefaultTest.java | 21 ++--- 10 files changed, 295 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f8c9c8cf..66515055 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`) 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`) - + > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project. ## Supported sites diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index a1dfd245..84fb7685 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import java.io.IOException; @@ -225,4 +226,35 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public String getOriginalUrl() throws ParsingException { return data.getString("frontend_link"); } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index d8d0de00..0bead848 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -9,7 +9,7 @@ import java.util.List; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -66,16 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(json, "thumbnailPath"); + return baseUrl + JsonUtils.getString(json, "previewPath"); } @Override public String getDescription() throws ParsingException { + String description = ""; + Downloader dl = NewPipe.getDownloader(); try { - return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { - return "No description"; + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + description = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); } + if (description.equals("")) { + //if the request above failed + description = JsonUtils.getString(json, "description"); + } + return description; } @Override @@ -224,8 +232,9 @@ public class PeertubeStreamExtractor extends StreamExtractor { if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } - - private List getTags(){ + + @Override + public List getTags(){ try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -339,4 +348,29 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + "/videos/watch/" + getId(); } + //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it + @Override + public String getHost() throws ParsingException { + return JsonUtils.getString(json, "account.host"); + } + + @Override + public String getPrivacy() throws ParsingException { + return JsonUtils.getString(json, "privacy.label"); + } + + @Override + public String getCategory() throws ParsingException { + return JsonUtils.getString(json, "category.label"); + } + + @Override + public String getLicence() throws ParsingException { + return JsonUtils.getString(json, "licence.label"); + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return JsonUtils.getString(json, "language.label"); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 14f7023f..19ab987c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -254,4 +254,35 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public String getErrorMessage() { return null; } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c05004ed..0e380fbc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1113,4 +1113,35 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ExtractionException(e); } } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index d697dff2..ce18dfa3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; @@ -349,4 +350,60 @@ public abstract class StreamExtractor extends Extractor { return 0; } } + + /** + * The host of the stream (Eg. peertube.cpy.re). + * If the privacy is not available, or if the service doesn't use + * a federated system, but a centralised system, + * you can simply return an empty string. + * @return the host of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getHost() throws ParsingException; + + /** + * The privacy of the stream (Eg. Public, Private, Unlisted…). + * If the privacy is not available you can simply return an empty string. + * @return the privacy of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getPrivacy() throws ParsingException; + + /** + * The name of the category of the stream. + * If the category is not available you can simply return an empty string. + * @return the category of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getCategory() throws ParsingException; + + /** + * The name of the licence of the stream. + * If the licence is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getLicence() throws ParsingException; + + /** + * The language of the stream. + * If the language is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getStreamInfoLanguage() throws ParsingException; + + /** + * The list of tags of the stream. + * If the tag list is not available you can simply return an empty list. + * @return the list of tags of the stream or an empty list. + * @throws ParsingException + */ + @Nonnull + public abstract List getTags() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 34ea703c..111f387d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -270,6 +270,38 @@ public class StreamInfo extends Info { streamInfo.addError(e); } + //additional info + try { + streamInfo.setHost(extractor.getHost()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setPrivacy(extractor.getPrivacy()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setCategory(extractor.getCategory()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLicence(extractor.getLicence()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setTags(extractor.getTags()); + } catch (Exception e) { + streamInfo.addError(e); + } + streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -308,6 +340,13 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles = new ArrayList<>(); + private String host = ""; + private String privacy = ""; + private String category = ""; + private String licence = ""; + private String language = ""; + private List tags = new ArrayList<>(); + /** * Get the stream type * @@ -533,4 +572,53 @@ public class StreamInfo extends Info { this.subtitles = subtitles; } + public String getHost() { + return this.host; + } + + public void setHost(String str) { + this.host = str; + } + + public String getPrivacy() { + return this.privacy; + } + + public void setPrivacy(String str) { + this.privacy = str; + } + + public String getCategory() { + return this.category; + } + + public void setCategory(String cat) { + this.category = cat; + } + + public String getLicence() { + return this.licence; + } + + public void setLicence(String str) { + this.licence = str; + } + + public String getLanguage() { + return this.language; + } + + public void setLanguage(String lang) { + this.language = lang; + } + + public List getTags() { + return this.tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index fd06d40f..3489b6d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -185,5 +185,4 @@ public class Utils { } return uri.getProtocol() + "://" + uri.getAuthority(); } - } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index fef1c3ee..e0a1d189 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (PeertubeCommentsExtractor) PeerTube - .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); } @Test @@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); assertTrue("Comments".equals(commentsInfo.getName())); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 75a692d4..b2175e85 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,13 +29,14 @@ import org.schabi.newpipe.extractor.stream.StreamType; */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; + private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); + PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d"); extractor.fetchPage(); } @@ -47,22 +48,22 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "Power Corrupts the Best"); + assertEquals("What is PeerTube?", extractor.getName()); } @Test public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); + assertEquals(expectedDescription, extractor.getDescription()); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "Rowsedower"); + assertEquals("Framasoft", extractor.getUploaderName()); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 269); + assertEquals(113, extractor.getLength()); } @Test @@ -74,7 +75,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } @@ -82,7 +83,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); + assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); } @Test @@ -115,11 +116,11 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } @Test public void testGetSubtitlesList() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } } From b382416372add1c9420cf1657f67c9cae825751f Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 14:36:12 +0100 Subject: [PATCH 002/149] changed the way to extract peertube description if the description length is above 254, and ends with ..., it means the description we got from the first request is shortened. why above 254: because in fact, shortened descriptions length are between 255 : https://framatube.org/videos/watch/24efbe1b-73c0-4d72-a3ff-77c8b32d3fcd https://framatube.org/videos/watch/1ca99f40-fb5b-4fa4-abe5-4d54325df7fc and 269: https://framatube.org/videos/watch/4d661d5f-a884-4510-bca8-15cb19aa3fe5 also fixed a typo in StreamExtractor.java --- .../extractors/PeertubeStreamExtractor.java | 25 +++++++++++-------- .../extractor/stream/StreamExtractor.java | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0bead848..ac3d6ecb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -70,20 +70,23 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public String getDescription() throws ParsingException { - String description = ""; - Downloader dl = NewPipe.getDownloader(); + String desc; try { - Response response = dl.get(getUrl() + "/description"); - JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - description = JsonUtils.getString(jsonObject, "description"); - } catch (ReCaptchaException | IOException | JsonParserException e) { - e.printStackTrace(); + desc = JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return "No description"; } - if (description.equals("")) { - //if the request above failed - description = JsonUtils.getString(json, "description"); + if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + Downloader dl = NewPipe.getDownloader(); + try { + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + desc = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); + } } - return description; + return desc; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index ce18dfa3..a638c7b6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -353,7 +353,7 @@ public abstract class StreamExtractor extends Extractor { /** * The host of the stream (Eg. peertube.cpy.re). - * If the privacy is not available, or if the service doesn't use + * If the host is not available, or if the service doesn't use * a federated system, but a centralised system, * you can simply return an empty string. * @return the host of the stream or an empty String. From ad7f97ae836f89a0f555e1062e5cc3b5211b1dc3 Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 15:14:23 +0100 Subject: [PATCH 003/149] fix PeerTube description and add more description tests actually, the max description length is 250 after request with our extractor. during my tests, I made API requests with Firefox, copy/pasted into echo "insert description" | wc, and it was giving a wrong length, maybe due to the escapers, I have no idea anyway, it's now fixed --- .../extractors/PeertubeStreamExtractor.java | 3 ++- .../PeertubeStreamExtractorDefaultTest.java | 22 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index ac3d6ecb..c922b2fe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (ParsingException e) { return "No description"; } - if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { Response response = dl.get(getUrl() + "/description"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index b2175e85..2a4378a2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,7 +29,8 @@ import org.schabi.newpipe.extractor.stream.StreamType; */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; - private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary"; @BeforeClass public static void setUp() throws Exception { @@ -52,8 +53,23 @@ public class PeertubeStreamExtractorDefaultTest { } @Test - public void testGetDescription() throws ParsingException { - assertEquals(expectedDescription, extractor.getDescription()); + public void testGetLargeDescription() throws ParsingException { + assertEquals(expectedLargeDescription, extractor.getDescription()); + } + + @Test + public void testGetEmptyDescription() throws Exception { + PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); + extractorEmpty.fetchPage(); + assertEquals("No description", extractorEmpty.getDescription()); + } + + @Test + public void testGetSmallDescription() throws Exception { + PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); + PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); + extractorSmall.fetchPage(); + assertEquals(expectedSmallDescription, extractorSmall.getDescription()); } @Test From f403490bd9ada2b19176ffc694d5b5ad405975b6 Mon Sep 17 00:00:00 2001 From: B0pol Date: Mon, 20 Jan 2020 21:28:26 +0100 Subject: [PATCH 004/149] Refactoring Co-Authored-By: Tobias Groza --- .../services/peertube/extractors/PeertubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index c922b2fe..0de72a94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (ParsingException e) { return "No description"; } - if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { From 1a15c0e750dc95fa0a92cc1c988e781e66631fa7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Thu, 23 Jan 2020 04:42:54 +0100 Subject: [PATCH 005/149] agelimit now returns 18 if the video is marked as nsfw, 0 otherwise + created getBoolean method in JsonUtils.java --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../org/schabi/newpipe/extractor/utils/JsonUtils.java | 10 ++++++++++ .../peertube/PeertubeStreamExtractorDefaultTest.java | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0de72a94..b841d9fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -92,7 +92,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() throws ParsingException { - return NO_AGE_LIMIT; + boolean isNSFW = JsonUtils.getBoolean(json, "nsfw"); + if (isNSFW) { + return 18; + } else { + return NO_AGE_LIMIT; + } } @Override @@ -352,7 +357,6 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + "/videos/watch/" + getId(); } - //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it @Override public String getHost() throws ParsingException { return JsonUtils.getString(json, "account.host"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index e5d7bb62..ebd251d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -37,6 +37,16 @@ public class JsonUtils { throw new ParsingException("Unable to get " + path); } } + + @Nonnull + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof Boolean) { + return (Boolean) value; + }else { + throw new ParsingException("Unable to get " + path); + } + } @Nonnull public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 2a4378a2..4166b192 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -139,4 +139,12 @@ public class PeertubeStreamExtractorDefaultTest { public void testGetSubtitlesList() throws IOException, ExtractionException { assertFalse(extractor.getSubtitlesDefault().isEmpty()); } + + @Test + public void testGetAgeLimit() throws ExtractionException, IOException { + assertEquals(0, extractor.getAgeLimit()); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7"); + ageLimit.fetchPage(); + assertEquals(18, ageLimit.getAgeLimit()); + } } From 74439f692a234d89b5556e1949192f3f9681f840 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:19:22 +0100 Subject: [PATCH 006/149] add extraction for support info and rename getLanguageInfo function --- .../extractors/MediaCCCStreamExtractor.java | 8 +++++++- .../extractors/PeertubeStreamExtractor.java | 12 ++++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 8 +++++++- .../extractors/YoutubeStreamExtractor.java | 8 +++++++- .../extractor/stream/StreamExtractor.java | 14 +++++++++++++- .../newpipe/extractor/stream/StreamInfo.java | 18 +++++++++++++++--- .../PeertubeStreamExtractorDefaultTest.java | 7 +++++++ 7 files changed, 66 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 84fb7685..9168d6e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -248,7 +248,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -257,4 +257,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index b841d9fd..4dadae77 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -35,6 +35,8 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import javax.annotation.Nonnull; + public class PeertubeStreamExtractor extends StreamExtractor { @@ -250,7 +252,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { return Collections.emptyList(); } } - + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return JsonUtils.getString(json, "support"); + } + private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); @@ -378,7 +386,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return JsonUtils.getString(json, "language.label"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 19ab987c..c5d34bd0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -276,7 +276,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -285,4 +285,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ea9a84d4..74a44a66 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1156,7 +1156,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -1165,4 +1165,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index a638c7b6..31a5a583 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -396,7 +396,7 @@ public abstract class StreamExtractor extends Extractor { * @throws ParsingException */ @Nonnull - public abstract String getStreamInfoLanguage() throws ParsingException; + public abstract String getLanguageInfo() throws ParsingException; /** * The list of tags of the stream. @@ -406,4 +406,16 @@ public abstract class StreamExtractor extends Extractor { */ @Nonnull public abstract List getTags() throws ParsingException; + + /** + * The support information of the stream. + * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37 + * (support button). + * If the support information are not available, + * you can simply return an empty String. + * @return the support information of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getSupportInfo() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 111f387d..e32a133d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -292,7 +292,7 @@ public class StreamInfo extends Info { streamInfo.addError(e); } try { - streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + streamInfo.setLanguageInfo(extractor.getLanguageInfo()); } catch (Exception e) { streamInfo.addError(e); } @@ -301,6 +301,11 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setSupportInfo(extractor.getSupportInfo()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); @@ -345,6 +350,7 @@ public class StreamInfo extends Info { private String category = ""; private String licence = ""; private String language = ""; + private String support = ""; private List tags = new ArrayList<>(); /** @@ -604,11 +610,11 @@ public class StreamInfo extends Info { this.licence = str; } - public String getLanguage() { + public String getLanguageInfo() { return this.language; } - public void setLanguage(String lang) { + public void setLanguageInfo(String lang) { this.language = lang; } @@ -620,5 +626,11 @@ public class StreamInfo extends Info { this.tags = tags; } + public void setSupportInfo(String support) { + this.support = support; + } + public String getSupportInfo() { + return this.support; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 4166b192..82967852 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -147,4 +147,11 @@ public class PeertubeStreamExtractorDefaultTest { ageLimit.fetchPage(); assertEquals(18, ageLimit.getAgeLimit()); } + + @Test + public void testGetSupportInformation() throws ExtractionException, IOException { + PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37"); + supportInfoExtractor.fetchPage(); + assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); + } } From 20da4750f8f779e7fd8efaf9471b890cd47c75a7 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:37:14 +0100 Subject: [PATCH 007/149] empty support returns "", same for empty description --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 4dadae77..a57720e3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { try { desc = JsonUtils.getString(json, "description"); } catch (ParsingException e) { - return "No description"; + return ""; } if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description @@ -256,7 +256,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getSupportInfo() throws ParsingException { - return JsonUtils.getString(json, "support"); + try { + return JsonUtils.getString(json, "support"); + } catch (ParsingException e) { + return ""; + } } private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 82967852..96273506 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -61,7 +61,7 @@ public class PeertubeStreamExtractorDefaultTest { public void testGetEmptyDescription() throws Exception { PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); extractorEmpty.fetchPage(); - assertEquals("No description", extractorEmpty.getDescription()); + assertEquals("", extractorEmpty.getDescription()); } @Test From a691d6d0fc4949ab1392082865416e83a04af341 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:02:40 +0100 Subject: [PATCH 008/149] fix upload date: there was a one hour delay on peertube.co.uk: https://i.imgur.com/8Pai1rb.png on newpipe, before this commit: https://i.imgur.com/NIRbs4Z.png on newpipe, with this commit: https://i.ibb.co/mhKKtRD/Screenshot-20200123-185422-New-Pipe-Debug.png --- .../extractor/services/peertube/PeertubeParsingHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index f518faf9..2483dd94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,6 +4,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.concurrent.TimeUnit; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -27,6 +28,7 @@ public class PeertubeParsingHelper { Date date; try { date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); + date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1)); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } From c790261eed4ba148b603325075bcbbfbd3e9b4c9 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:08:41 +0100 Subject: [PATCH 009/149] update test --- .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 96273506..c6a164f7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -91,7 +91,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T11:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } From 812c8e0ad25de2f1d3f5da16975cae1b145072f1 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 00:22:05 +0100 Subject: [PATCH 010/149] authorName in comments now follow PeerTube website MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea anonyme zirbeldrüse is his displayName, what was displayed in NewPipe. but on the website, it's shown 777@mastodon.xyz --- .../extractors/PeertubeCommentsInfoItemExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index edb59e1f..0bcce9fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -67,7 +67,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac try { Document doc = Jsoup.parse(htmlText); return doc.body().text(); - }catch(Exception e) { + } catch(Exception e) { return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); } } @@ -83,7 +83,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -91,7 +91,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac @Override public String getAuthorName() throws ParsingException { - return JsonUtils.getString(item, "account.displayName"); + return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); } @Override From 341372c0d0dadcce1b88333d9c8afc58ec50fff7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 20:16:24 +0100 Subject: [PATCH 011/149] reindenting (ctrl alt l) on JsonUtils and PeertubeStreamExtractor --- .../extractors/PeertubeStreamExtractor.java | 75 ++++++++++--------- .../newpipe/extractor/utils/JsonUtils.java | 46 ++++++------ 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index a57720e3..e6e32d44 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -39,16 +39,16 @@ import javax.annotation.Nonnull; public class PeertubeStreamExtractor extends StreamExtractor { - + + private final String baseUrl; private JsonObject json; private List subtitles = new ArrayList<>(); - private final String baseUrl; - + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); this.baseUrl = getBaseUrl(); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(json, "publishedAt"); @@ -64,7 +64,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public String getThumbnailUrl() throws ParsingException { return baseUrl + JsonUtils.getString(json, "previewPath"); @@ -149,7 +149,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { String value; try { value = JsonUtils.getString(json, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -176,8 +176,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { List videoStreams = new ArrayList<>(); try { JsonArray streams = json.getArray("files", new JsonArray()); - for(Object s: streams) { - if(!(s instanceof JsonObject)) continue; + for (Object s : streams) { + if (!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; String url = JsonUtils.getString(stream, "fileUrl"); String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); @@ -211,8 +211,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { List filteredSubs = new ArrayList<>(); - for(SubtitlesStream sub: subtitles) { - if(sub.getFormat() == format) { + for (SubtitlesStream sub : subtitles) { + if (sub.getFormat() == format) { filteredSubs.add(sub); } } @@ -234,18 +234,18 @@ public class PeertubeStreamExtractor extends StreamExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); List tags = getTags(); String apiUrl = null; - if(!tags.isEmpty()) { + if (!tags.isEmpty()) { apiUrl = getRelatedStreamsUrl(tags); - - }else { + + } else { apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; } - if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); + if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } @Override - public List getTags(){ + public List getTags() { try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -267,7 +267,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); - for(String tag : tags) { + for (String tag : tags) { params.append("&tagsOneOf="); params.append(URLEncoder.encode(tag, "UTF-8")); } @@ -277,38 +277,38 @@ public class PeertubeStreamExtractor extends StreamExtractor { private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { Response response = getDownloader().get(apiUrl); JsonObject relatedVideosJson = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { relatedVideosJson = JsonParser.object().from(response.responseBody()); } catch (JsonParserException e) { throw new ParsingException("Could not parse json data for related videos", e); } } - - if(relatedVideosJson != null) { + + if (relatedVideosJson != null) { collectStreamsFrom(collector, relatedVideosJson); } } - + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract related videos", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); //do not add the same stream in related streams - if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); + if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor); } } - + } - + @Override public String getErrorMessage() { @@ -318,12 +318,12 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { + } else { throw new ExtractionException("Unable to extract peertube channel data"); } - + loadSubtitles(); } @@ -333,24 +333,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube stream data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + if (null == json) throw new ExtractionException("Unable to extract peertube stream data"); PeertubeParsingHelper.validate(json); } - + private void loadSubtitles() { if (subtitles.isEmpty()) { try { - Response response = getDownloader().get(getUrl() + "/captions"); + Response response = getDownloader().get(getUrl() + "/captions"); JsonObject captionsJson = JsonParser.object().from(response.responseBody()); JsonArray captions = JsonUtils.getArray(captionsJson, "data"); - for(Object c: captions) { - if(c instanceof JsonObject) { - JsonObject caption = (JsonObject)c; + for (Object c : captions) { + if (c instanceof JsonObject) { + JsonObject caption = (JsonObject) c; String url = baseUrl + JsonUtils.getString(caption, "captionPath"); String languageCode = JsonUtils.getString(caption, "language.id"); String ext = url.substring(url.lastIndexOf(".") + 1); MediaFormat fmt = MediaFormat.getFromSuffix(ext); - if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + if (fmt != null && languageCode != null) + subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); } } } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ebd251d7..80e0dd58 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -16,71 +16,71 @@ public class JsonUtils { private JsonUtils() { } - + @Nonnull - public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { List keys = Arrays.asList(path.split("\\.")); object = getObject(object, keys.subList(0, keys.size() - 1)); if (null == object) throw new ParsingException("Unable to get " + path); Object result = object.get(keys.get(keys.size() - 1)); - if(null == result) throw new ParsingException("Unable to get " + path); + if (null == result) throw new ParsingException("Unable to get " + path); return result; } - + @Nonnull - public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof String) { + if (value instanceof String) { return (String) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull - public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Boolean) { + if (value instanceof Boolean) { return (Boolean) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Number) { + if (value instanceof Number) { return (Number) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonObject) { + if (value instanceof JsonObject) { return (JsonObject) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonArray) { + if (value instanceof JsonArray) { return (JsonArray) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { - + List result = new ArrayList<>(); for (int i = 0; i < array.size(); i++) { JsonObject obj = array.getObject(i); From e392b6c68f247e954a2e4eac5c097a4d809d3e1f Mon Sep 17 00:00:00 2001 From: B0pol Date: Sat, 25 Jan 2020 13:16:42 +0100 Subject: [PATCH 012/149] getLanguageInfo returns Locale instead of String so that java can automatically translate with Locale.getDisplayLanguage(), instead of always having English name of the language --- .../extractors/MediaCCCStreamExtractor.java | 5 +++-- .../extractors/PeertubeStreamExtractor.java | 9 +++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 5 +++-- .../youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- .../newpipe/extractor/stream/StreamExtractor.java | 13 ++++++++----- .../schabi/newpipe/extractor/stream/StreamInfo.java | 7 ++++--- .../PeertubeStreamExtractorDefaultTest.java | 6 ++++++ 7 files changed, 33 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 9168d6e6..07c36d12 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -18,6 +18,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; public class MediaCCCStreamExtractor extends StreamExtractor { @@ -248,8 +249,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index e6e32d44..80cae88a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -6,6 +6,7 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; @@ -391,7 +392,11 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return JsonUtils.getString(json, "language.label"); + public Locale getLanguageInfo() throws ParsingException { + try { + return new Locale(JsonUtils.getString(json, "language.id")); + } catch (ParsingException e) { + return null; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index c5d34bd0..5b1153d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -20,6 +20,7 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -276,8 +277,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index dcf302cb..10ffb143 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1157,8 +1157,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public String getLanguageInfo() throws ParsingException { - return ""; + public Locale getLanguageInfo() throws ParsingException { + return null; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 31a5a583..652e0116 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -35,6 +35,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Locale; /** * Scrapes information from a video/audio streaming service (eg, YouTube). @@ -390,13 +391,15 @@ public abstract class StreamExtractor extends Extractor { public abstract String getLicence() throws ParsingException; /** - * The language of the stream. - * If the language is not available you can simply return an empty string. - * @return the licence of the stream or an empty String. + * The locale language of the stream. + * If the language is not available you can simply return null. + * If the language is provided by a language code, you can return + * new Locale(language_code); + * @return the locale language of the stream or null. * @throws ParsingException */ - @Nonnull - public abstract String getLanguageInfo() throws ParsingException; + @Nullable + public abstract Locale getLanguageInfo() throws ParsingException; /** * The list of tags of the stream. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index e32a133d..64999ce7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; /* * Created by Christian Schabesberger on 26.08.15. @@ -349,8 +350,8 @@ public class StreamInfo extends Info { private String privacy = ""; private String category = ""; private String licence = ""; - private String language = ""; private String support = ""; + private Locale language = null; private List tags = new ArrayList<>(); /** @@ -610,11 +611,11 @@ public class StreamInfo extends Info { this.licence = str; } - public String getLanguageInfo() { + public Locale getLanguageInfo() { return this.language; } - public void setLanguageInfo(String lang) { + public void setLanguageInfo(Locale lang) { this.language = lang; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index c6a164f7..06a8352c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Locale; import org.junit.BeforeClass; import org.junit.Ignore; @@ -154,4 +155,9 @@ public class PeertubeStreamExtractorDefaultTest { supportInfoExtractor.fetchPage(); assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); } + + @Test + public void testGetLanguageInformation() throws ParsingException { + assertEquals(new Locale("en"), extractor.getLanguageInfo()); + } } From 5756df8dc7e89b7383d1d1e07a91c30bdab6f868 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 6 Feb 2020 22:24:28 +0100 Subject: [PATCH 013/149] Use GMT as base time (actually fix upload date) --- .../services/peertube/PeertubeParsingHelper.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 2483dd94..c84e4e5b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,7 +4,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import java.util.concurrent.TimeUnit; +import java.util.TimeZone; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -13,22 +13,23 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import com.grack.nanojson.JsonObject; public class PeertubeParsingHelper { - + private PeertubeParsingHelper() { } public static void validate(JsonObject json) throws ContentNotAvailableException { String error = json.getString("error"); - if(!StringUtil.isBlank(error)) { + if (!StringUtil.isBlank(error)) { throw new ContentNotAvailableException(error); } } - + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); - date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1)); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } From 26c65b2948dcc58be4a004938f415bbd46380203 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 6 Feb 2020 23:35:46 +0100 Subject: [PATCH 014/149] Create class Description --- .../extractors/MediaCCCStreamExtractor.java | 5 +- .../extractors/PeertubeStreamExtractor.java | 23 ++++------ .../soundcloud/SoundcloudStreamExtractor.java | 5 +- .../extractors/YoutubeStreamExtractor.java | 6 +-- .../newpipe/extractor/stream/Description.java | 46 +++++++++++++++++++ .../extractor/stream/StreamExtractor.java | 6 +-- .../newpipe/extractor/stream/StreamInfo.java | 6 +-- .../PeertubeStreamExtractorDefaultTest.java | 7 +-- ...utubeStreamExtractorAgeRestrictedTest.java | 2 +- ...utubeStreamExtractorControversialTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 38 +++++++-------- .../YoutubeStreamExtractorLivestreamTest.java | 6 +-- 12 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 07c36d12..56036c2e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,7 +12,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; -import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import java.io.IOException; @@ -49,8 +48,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() throws ParsingException { - return data.getString("description"); + public Description getDescription() throws ParsingException { + return new Description(getServiceId(), data.getString("description")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 80cae88a..ac3267f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -21,14 +21,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; import com.grack.nanojson.JsonArray; @@ -72,25 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public String getDescription() throws ParsingException { - String desc; + public Description getDescription() throws ParsingException { + String text; try { - desc = JsonUtils.getString(json, "description"); + text = JsonUtils.getString(json, "description"); } catch (ParsingException e) { - return ""; + return Description.emptyDescription; } - if (desc.length() == 250 && desc.substring(247).equals("...")) { + if (text.length() == 250 && text.substring(247).equals("...")) { //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { Response response = dl.get(getUrl() + "/description"); JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - desc = JsonUtils.getString(jsonObject, "description"); + text = JsonUtils.getString(jsonObject, "description"); } catch (ReCaptchaException | IOException | JsonParserException e) { e.printStackTrace(); } } - return desc; + return new Description(getServiceId(), text); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 5b1153d5..c2a537a2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -74,10 +74,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return artworkUrlBetterResolution; } - @Nonnull @Override - public String getDescription() { - return track.getString("description"); + public Description getDescription() { + return new Description(getServiceId(), track.getString("description")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f8523ea5..d8ded292 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -180,15 +180,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getDescription() throws ParsingException { + public Description getDescription() throws ParsingException { assertPageFetched(); try { // first try to get html-formatted description - return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); + return new Description(getServiceId(), parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html())); } catch (Exception e) { try { // fallback to raw non-html description - return playerResponse.getObject("videoDetails").getString("shortDescription"); + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); } catch (Exception ignored) { throw new ParsingException("Could not get the description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java new file mode 100644 index 00000000..1ce5e948 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.stream; + +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class Description { + private String content; + private int type; + + public static final int HTML = 1; + public static final int MARKDOWN = 2; + public static final int PLAIN_TEXT = 3; + public static final Description emptyDescription = new Description(PLAIN_TEXT, ""); + + public Description(int serviceID, String content) { + if (serviceID == PeerTube.getServiceId()) { + this.type = MARKDOWN; + } else if (serviceID == YouTube.getServiceId()) { + this.type = HTML; + } else { + this.type = PLAIN_TEXT; + } + setContent(content); + } + + private void setContent(String content) { + if (content == null) { + this.content = ""; + } else { + this.content = content; + } + } + + public Description(String content, int type) { + this.type = type; + setContent(content); + } + + public String getContent() { + return content; + } + + public int getType() { + return type; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 652e0116..fdc56972 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -84,12 +84,12 @@ public abstract class StreamExtractor extends Extractor { public abstract String getThumbnailUrl() throws ParsingException; /** - * This is the stream description. On YouTube this is the video description. You can return simple HTML here. - * @return The description of the stream/video. + * This is the stream description. + * @return The description of the stream/video or Description.emptyDescription if the description is empty. * @throws ParsingException */ @Nonnull - public abstract String getDescription() throws ParsingException; + public abstract Description getDescription() throws ParsingException; /** * Get the age limit. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 64999ce7..580f9c21 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -319,7 +319,7 @@ public class StreamInfo extends Info { private DateWrapper uploadDate; private long duration = -1; private int ageLimit = -1; - private String description; + private Description description; private long viewCount = -1; private long likeCount = -1; @@ -417,11 +417,11 @@ public class StreamInfo extends Info { this.ageLimit = ageLimit; } - public String getDescription() { + public Description getDescription() { return description; } - public void setDescription(String description) { + public void setDescription(Description description) { this.description = description; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 06a8352c..794a14a3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -55,14 +55,14 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetLargeDescription() throws ParsingException { - assertEquals(expectedLargeDescription, extractor.getDescription()); + assertEquals(expectedLargeDescription, extractor.getDescription().getContent()); } @Test public void testGetEmptyDescription() throws Exception { PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); extractorEmpty.fetchPage(); - assertEquals("", extractorEmpty.getDescription()); + assertEquals("", extractorEmpty.getDescription().getContent()); } @Test @@ -70,7 +70,7 @@ public class PeertubeStreamExtractorDefaultTest { PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); extractorSmall.fetchPage(); - assertEquals(expectedSmallDescription, extractorSmall.getDescription()); + assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent()); } @Test @@ -89,6 +89,7 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } + @Ignore //fixme @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index b9da6d17..6afc7212 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -64,7 +64,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index f0bd7715..915f42ae 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 5f21df55..e2c0c7bb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -83,13 +83,13 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("http://adele.com")); - assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); + assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); + assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test @@ -215,18 +215,18 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); - assertTrue(extractor.getDescription().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); + assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); + assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); - assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub...")); - assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8...")); - assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/...")); + assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub...")); + assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8...")); + assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/...")); } } @@ -244,20 +244,20 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertFalse(extractor.getDescription().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); - assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); + assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index eb2541ed..b1d6f53a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -44,13 +44,13 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); + assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/")); - assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba...")); + assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/")); + assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba...")); } @Test From 70a40e7388f5bed7ca2019e0d4f82c18631f8902 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 13:28:27 +0100 Subject: [PATCH 015/149] Description: rm constructor by serviceId --- .../extractors/MediaCCCStreamExtractor.java | 2 +- .../extractors/PeertubeStreamExtractor.java | 2 +- .../soundcloud/SoundcloudStreamExtractor.java | 2 +- .../extractors/YoutubeStreamExtractor.java | 2 +- .../newpipe/extractor/stream/Description.java | 27 ++++--------------- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 56036c2e..08f21ef7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -49,7 +49,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public Description getDescription() throws ParsingException { - return new Description(getServiceId(), data.getString("description")); + return new Description(data.getString("description"), Description.PLAIN_TEXT); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index ac3267f1..a4dafbfc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -83,7 +83,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { e.printStackTrace(); } } - return new Description(getServiceId(), text); + return new Description(text, Description.MARKDOWN); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index c2a537a2..18fffa41 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -76,7 +76,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public Description getDescription() { - return new Description(getServiceId(), track.getString("description")); + return new Description(track.getString("description"), Description.PLAIN_TEXT); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index d8ded292..ea870d71 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -184,7 +184,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { // first try to get html-formatted description - return new Description(getServiceId(), parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html())); + return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML); } catch (Exception e) { try { // fallback to raw non-html description diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java index 1ce5e948..cde422bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -1,29 +1,17 @@ package org.schabi.newpipe.extractor.stream; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - public class Description { - private String content; - private int type; public static final int HTML = 1; public static final int MARKDOWN = 2; public static final int PLAIN_TEXT = 3; - public static final Description emptyDescription = new Description(PLAIN_TEXT, ""); + public static final Description emptyDescription = new Description("", PLAIN_TEXT); - public Description(int serviceID, String content) { - if (serviceID == PeerTube.getServiceId()) { - this.type = MARKDOWN; - } else if (serviceID == YouTube.getServiceId()) { - this.type = HTML; - } else { - this.type = PLAIN_TEXT; - } - setContent(content); - } + private String content; + private int type; - private void setContent(String content) { + public Description(String content, int type) { + this.type = type; if (content == null) { this.content = ""; } else { @@ -31,11 +19,6 @@ public class Description { } } - public Description(String content, int type) { - this.type = type; - setContent(content); - } - public String getContent() { return content; } From 0f8a7f9df2138d0e5697cc723655d8ff082699be Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 13:33:50 +0100 Subject: [PATCH 016/149] fix testGetUploadDate for PeerTubeStreamExtractor --- .../PeertubeStreamExtractorDefaultTest.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 794a14a3..a81bf34c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -12,6 +12,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Locale; +import java.util.TimeZone; import org.junit.BeforeClass; import org.junit.Ignore; @@ -42,6 +43,16 @@ public class PeertubeStreamExtractorDefaultTest { extractor.fetchPage(); } + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + + } + @Test public void testGetInvalidTimeStamp() throws ParsingException { assertTrue(extractor.getTimeStamp() + "", @@ -89,15 +100,6 @@ public class PeertubeStreamExtractorDefaultTest { extractor.getViewCount() > 10); } - @Ignore //fixme - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T11:52:46.396Z")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - - } - @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); From 11bcc78d9c8eb39e8d61a6f4bc4112025937f087 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 7 Feb 2020 14:10:54 +0100 Subject: [PATCH 017/149] Description implements Serializable. fix NotSerializableException --- .../java/org/schabi/newpipe/extractor/stream/Description.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java index cde422bb..b19b8de3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.stream; -public class Description { +import java.io.Serializable; + +public class Description implements Serializable { public static final int HTML = 1; public static final int MARKDOWN = 2; From 0e33249c85d2f23be9d8aa7fccd4bdc409a8f8ac Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 22:20:20 +0100 Subject: [PATCH 018/149] Fix SoundCloud description test --- .../soundcloud/SoundcloudStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 8a3883fd..0ffe0622 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { - assertEquals("The Perfect LUV Tape®️", extractor.getDescription()); + assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); } @Test From 008b6f505c94a657e9a645e87b84852b432e2462 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 23:08:37 +0100 Subject: [PATCH 019/149] [PeerTube] Fix subscriber count in ChannelExtractorTest --- .../services/peertube/PeertubeChannelExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 3d8bdff3..57b6360b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -204,7 +204,7 @@ public class PeertubeChannelExtractorTest { @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1); } } } From 9112a106eab9c05ca5c9e574623d4908ecc15656 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 23:09:03 +0100 Subject: [PATCH 020/149] [SoundCloud] Update client_id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 4617f916..82216075 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "r5ELVSy3RkcjX7ilaL7n2v1Z8irA9SL8"; // Updated on 31/12/19 + private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 private static String clientId; private SoundcloudParsingHelper() { From aae3476ee4356fc73f38c66d278aa23c00355192 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 23:09:03 +0100 Subject: [PATCH 021/149] [SoundCloud] Update client_id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 4617f916..82216075 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "r5ELVSy3RkcjX7ilaL7n2v1Z8irA9SL8"; // Updated on 31/12/19 + private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 private static String clientId; private SoundcloudParsingHelper() { From 33961b209206778fb07dd52252c9bfd2a4fef968 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 10 Feb 2020 21:12:39 +0100 Subject: [PATCH 022/149] Version 0.18.3 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 3d8ced48..a40ee853 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.0' + version 'v0.18.3' group 'com.github.TeamNewPipe' repositories { From 030465b5d4d8c738a42cfc52bd1d78f54b0f6bd4 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 8 Feb 2020 23:58:46 +0100 Subject: [PATCH 023/149] Improve code formatting and optimise imports --- .../schabi/newpipe/extractor/Extractor.java | 36 ++--- .../newpipe/extractor/InfoItemsCollector.java | 2 +- .../schabi/newpipe/extractor/MediaFormat.java | 12 +- .../schabi/newpipe/extractor/ServiceList.java | 8 +- .../newpipe/extractor/StreamingService.java | 14 +- .../channel/ChannelInfoItemsCollector.java | 2 +- .../extractor/comments/CommentsInfo.java | 32 ++--- .../extractor/kiosk/KioskExtractor.java | 1 - .../newpipe/extractor/kiosk/KioskList.java | 27 ++-- .../extractor/linkhandler/LinkHandler.java | 8 +- .../linkhandler/LinkHandlerFactory.java | 12 +- .../linkhandler/ListLinkHandlerFactory.java | 33 +++-- .../linkhandler/SearchQueryHandler.java | 1 + .../SearchQueryHandlerFactory.java | 14 +- .../extractor/localization/Localization.java | 5 +- .../extractor/localization/TimeAgoParser.java | 4 +- .../search/InfoItemsSearchCollector.java | 9 +- .../MediaCCCConferenceExtractor.java | 2 +- .../extractors/MediaCCCConferenceKiosk.java | 2 +- .../extractors/MediaCCCSearchExtractor.java | 20 +-- .../extractors/MediaCCCStreamExtractor.java | 18 +-- .../MediaCCCConferenceLinkHandlerFactory.java | 4 +- ...aCCCConferencesListLinkHandlerFactory.java | 1 - .../MediaCCCSearchQueryHandlerFactory.java | 2 +- .../MediaCCCStreamLinkHandlerFactory.java | 4 +- .../services/peertube/PeertubeInstance.java | 31 ++--- .../peertube/PeertubeParsingHelper.java | 11 +- .../services/peertube/PeertubeService.java | 44 ++---- .../extractors/PeertubeChannelExtractor.java | 61 ++++----- .../extractors/PeertubeCommentsExtractor.java | 43 +++--- .../PeertubeCommentsInfoItemExtractor.java | 15 +- .../extractors/PeertubePlaylistExtractor.java | 8 +- .../extractors/PeertubeSearchExtractor.java | 51 ++++--- .../extractors/PeertubeStreamExtractor.java | 24 ++-- .../PeertubeStreamInfoItemExtractor.java | 27 ++-- .../PeertubeSubscriptionExtractor.java | 4 +- .../PeertubeSuggestionExtractor.java | 10 +- .../extractors/PeertubeTrendingExtractor.java | 41 +++--- .../PeertubeChannelLinkHandlerFactory.java | 6 +- .../PeertubeCommentsLinkHandlerFactory.java | 8 +- .../PeertubePlaylistLinkHandlerFactory.java | 8 +- .../PeertubeSearchQueryHandlerFactory.java | 14 +- .../PeertubeStreamLinkHandlerFactory.java | 2 +- .../PeertubeTrendingLinkHandlerFactory.java | 26 ++-- .../SoundcloudChannelExtractor.java | 6 +- .../SoundcloudChannelLinkHandlerFactory.java | 2 +- .../soundcloud/SoundcloudChartsExtractor.java | 10 +- .../soundcloud/SoundcloudParsingHelper.java | 4 +- .../SoundcloudPlaylistExtractor.java | 2 +- .../SoundcloudPlaylistLinkHandlerFactory.java | 2 +- .../soundcloud/SoundcloudSearchExtractor.java | 5 +- .../SoundcloudSearchQueryHandlerFactory.java | 4 +- .../soundcloud/SoundcloudService.java | 27 ++-- .../soundcloud/SoundcloudStreamExtractor.java | 6 +- .../SoundcloudStreamLinkHandlerFactory.java | 2 +- .../SoundcloudSuggestionExtractor.java | 2 +- .../services/youtube/YoutubeService.java | 28 +--- .../extractors/YoutubeChannelExtractor.java | 2 +- .../extractors/YoutubeCommentsExtractor.java | 29 ++-- .../YoutubeCommentsInfoItemExtractor.java | 2 +- .../extractors/YoutubePlaylistExtractor.java | 9 +- .../YoutubePlaylistInfoItemExtractor.java | 2 +- .../extractors/YoutubeSearchExtractor.java | 2 +- .../extractors/YoutubeStreamExtractor.java | 128 +++++++++--------- .../YoutubeStreamInfoItemExtractor.java | 9 +- .../YoutubeSuggestionExtractor.java | 6 +- .../extractors/YoutubeTrendingExtractor.java | 4 +- .../YoutubePlaylistLinkHandlerFactory.java | 2 +- .../YoutubeSearchQueryHandlerFactory.java | 4 +- .../newpipe/extractor/stream/Frameset.java | 102 +++++++------- .../newpipe/extractor/stream/Stream.java | 14 +- .../extractor/stream/StreamExtractor.java | 37 ++++- .../stream/StreamInfoItemExtractor.java | 11 +- .../stream/StreamInfoItemsCollector.java | 4 +- .../extractor/stream/SubtitlesStream.java | 4 +- .../newpipe/extractor/stream/VideoStream.java | 6 +- .../extractor/utils/DashMpdParser.java | 12 +- .../extractor/utils/ExtractorHelper.java | 5 +- .../newpipe/extractor/utils/JsonUtils.java | 16 +-- .../newpipe/extractor/utils/Parser.java | 16 +-- .../schabi/newpipe/extractor/utils/Utils.java | 11 +- .../MediaCCCConferenceExtractorTest.java | 2 +- .../MediaCCCConferenceListExtractorTest.java | 6 +- .../services/media_ccc/MediaCCCOggTest.java | 4 +- .../MediaCCCSearchExtractorAllTest.java | 10 +- ...ediaCCCSearchExtractorConferencesTest.java | 4 +- .../MediaCCCSearchExtractorEventsTest.java | 4 +- .../MediaCCCStreamExtractorTest.java | 2 +- .../PeertubeChannelExtractorTest.java | 16 +-- ...PeertubeChannelLinkHandlerFactoryTest.java | 6 +- .../PeertubeCommentsExtractorTest.java | 22 +-- ...eertubeCommentsLinkHandlerFactoryTest.java | 6 +- ...eertubePlaylistLinkHandlerFactoryTest.java | 6 +- .../PeertubeStreamExtractorDefaultTest.java | 26 ++-- .../PeertubeStreamLinkHandlerFactoryTest.java | 6 +- .../PeertubeTrendingExtractorTest.java | 17 +-- ...eertubeTrendingLinkHandlerFactoryTest.java | 14 +- .../PeertubeSearchExtractorBaseTest.java | 4 +- .../PeertubeSearchExtractorDefaultTest.java | 18 ++- .../peertube/search/PeertubeSearchQHTest.java | 10 +- .../SoundcloudChartsExtractorTest.java | 5 +- .../SoundcloudParsingHelperTest.java | 6 +- ...ndcloudSearchExtractorChannelOnlyTest.java | 6 +- .../SoundcloudSearchExtractorDefaultTest.java | 6 +- .../youtube/YoutubeChannelExtractorTest.java | 16 +-- .../YoutubeChannelLinkHandlerFactoryTest.java | 8 +- .../YoutubeStreamLinkHandlerFactoryTest.java | 4 +- .../youtube/YoutubeTrendingKioskInfoTest.java | 38 +++--- .../YoutubeSearchExtractorBaseTest.java | 2 - ...YoutubeSearchExtractorChannelOnlyTest.java | 4 +- .../YoutubeSearchExtractorDefaultTest.java | 7 +- .../search/YoutubeSearchPagingTest.java | 18 ++- .../youtube/search/YoutubeSearchQHTest.java | 7 +- ...utubeStreamExtractorAgeRestrictedTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 4 +- .../extractor/utils/JsonUtilsTest.java | 25 ++-- 116 files changed, 792 insertions(+), 793 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index b6e90d53..df1c2261 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor; -import java.io.IOException; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -13,7 +8,11 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -public abstract class Extractor{ +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; + +public abstract class Extractor { /** * {@link StreamingService} currently related to this extractor.
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls). @@ -21,19 +20,21 @@ public abstract class Extractor{ private final StreamingService service; private final LinkHandler linkHandler; - @Nullable private Localization forcedLocalization = null; - @Nullable private ContentCountry forcedContentCountry = null; + @Nullable + private Localization forcedLocalization = null; + @Nullable + private ContentCountry forcedContentCountry = null; private boolean pageFetched = false; private final Downloader downloader; public Extractor(final StreamingService service, final LinkHandler linkHandler) { - if(service == null) throw new NullPointerException("service is null"); - if(linkHandler == null) throw new NullPointerException("LinkHandler is null"); + if (service == null) throw new NullPointerException("service is null"); + if (linkHandler == null) throw new NullPointerException("LinkHandler is null"); this.service = service; this.linkHandler = linkHandler; this.downloader = NewPipe.getDownloader(); - if(downloader == null) throw new NullPointerException("downloader is null"); + if (downloader == null) throw new NullPointerException("downloader is null"); } /** @@ -46,11 +47,12 @@ public abstract class Extractor{ /** * Fetch the current page. - * @throws IOException if the page can not be loaded + * + * @throws IOException if the page can not be loaded * @throws ExtractionException if the pages content is not understood */ public void fetchPage() throws IOException, ExtractionException { - if(pageFetched) return; + if (pageFetched) return; onFetchPage(downloader); pageFetched = true; } @@ -65,8 +67,9 @@ public abstract class Extractor{ /** * Fetch the current page. + * * @param downloader the download to use - * @throws IOException if the page can not be loaded + * @throws IOException if the page can not be loaded * @throws ExtractionException if the pages content is not understood */ public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException; @@ -78,6 +81,7 @@ public abstract class Extractor{ /** * Get the name + * * @return the name * @throws ParsingException if the name cannot be extracted */ @@ -93,10 +97,10 @@ public abstract class Extractor{ public String getUrl() throws ParsingException { return linkHandler.getUrl(); } - + @Nonnull public String getBaseUrl() throws ParsingException { - return linkHandler.getBaseUrl(); + return linkHandler.getBaseUrl(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java index 2a3cb5f3..72eca16e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java @@ -27,7 +27,7 @@ import java.util.List; * along with NewPipe. If not, see . */ -public abstract class InfoItemsCollector implements Collector { +public abstract class InfoItemsCollector implements Collector { private final List itemList = new ArrayList<>(); private final List errors = new ArrayList<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index c65283a5..0b6052f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -116,25 +116,27 @@ public enum MediaFormat { /** * Get the media format by it's id. + * * @param id the id * @return the id of the media format or null. */ public static MediaFormat getFormatById(int id) { - for (MediaFormat vf: values()) { + for (MediaFormat vf : values()) { if (vf.id == id) return vf; } return null; } public static MediaFormat getFromSuffix(String suffix) { - for (MediaFormat vf: values()) { + for (MediaFormat vf : values()) { if (vf.suffix.equals(suffix)) return vf; } return null; } - + /** * Get the name of the format + * * @return the name of the format */ public String getName() { @@ -143,6 +145,7 @@ public enum MediaFormat { /** * Get the filename extension + * * @return the filename extension */ public String getSuffix() { @@ -151,10 +154,11 @@ public enum MediaFormat { /** * Get the mime type + * * @return the mime type */ public String getMimeType() { return mimeType; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 6be1cea4..83d8522f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /* * Copyright (C) Christian Schabesberger 2018 * ServiceList.java is part of NewPipe. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 68398506..110efe04 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -1,20 +1,12 @@ package org.schabi.newpipe.extractor; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -26,6 +18,8 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; /* * Copyright (C) Christian Schabesberger 2018 @@ -269,7 +263,7 @@ public abstract class StreamingService { public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException { ListLinkHandlerFactory llhf = getCommentsLHFactory(); - if(null == llhf) { + if (null == llhf) { return null; } return getCommentsExtractor(llhf.fromUrl(url)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java index a60636b2..19abb1c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java @@ -33,7 +33,7 @@ public class ChannelInfoItemsCollector extends InfoItemsCollector{ +import java.io.IOException; - private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { - super(serviceId, listUrlIdHandler, name); - // TODO Auto-generated constructor stub - } - - public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { +public class CommentsInfo extends ListInfo { + + private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { + super(serviceId, listUrlIdHandler, name); + // TODO Auto-generated constructor stub + } + + public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { return getInfo(NewPipe.getServiceByUrl(url), url); } - public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException { - return getInfo(serviceByUrl.getCommentsExtractor(url)); - } + public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException { + return getInfo(serviceByUrl.getCommentsExtractor(url)); + } private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException { // for services which do not have a comments extractor @@ -44,21 +44,21 @@ public class CommentsInfo extends ListInfo{ return commentsInfo; } - + public static InfoItemsPage getMoreItems(CommentsInfo commentsInfo, String pageUrl) throws ExtractionException, IOException { return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl); } - + public static InfoItemsPage getMoreItems(StreamingService service, CommentsInfo commentsInfo, - String pageUrl) throws IOException, ExtractionException { + String pageUrl) throws IOException, ExtractionException { if (null == commentsInfo.getCommentsExtractor()) { commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl())); commentsInfo.getCommentsExtractor().fetchPage(); } return commentsInfo.getCommentsExtractor().getPage(pageUrl); } - + private transient CommentsExtractor commentsExtractor; public CommentsExtractor getCommentsExtractor() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 7aef494d..44890524 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -25,7 +25,6 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; import javax.annotation.Nonnull; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 5a3be079..878fa8ce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -13,27 +13,30 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -public class KioskList { +public class KioskList { public interface KioskExtractorFactory { KioskExtractor createNewKiosk(final StreamingService streamingService, final String url, final String kioskId) - throws ExtractionException, IOException; + throws ExtractionException, IOException; } private final StreamingService service; private final HashMap kioskList = new HashMap<>(); private String defaultKiosk = null; - @Nullable private Localization forcedLocalization; - @Nullable private ContentCountry forcedContentCountry; + @Nullable + private Localization forcedLocalization; + @Nullable + private ContentCountry forcedContentCountry; private class KioskEntry { public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) { extractorFactory = ef; handlerFactory = h; } + final KioskExtractorFactory extractorFactory; final ListLinkHandlerFactory handlerFactory; } @@ -43,8 +46,8 @@ public class KioskList { } public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id) - throws Exception { - if(kioskList.get(id) != null) { + throws Exception { + if (kioskList.get(id) != null) { throw new Exception("Kiosk with type " + id + " already exists."); } kioskList.put(id, new KioskEntry(extractorFactory, handlerFactory)); @@ -66,10 +69,10 @@ public class KioskList { public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization) throws ExtractionException, IOException { - if(defaultKiosk != null && !defaultKiosk.equals("")) { + if (defaultKiosk != null && !defaultKiosk.equals("")) { return getExtractorById(defaultKiosk, nextPageUrl, localization); } else { - if(!kioskList.isEmpty()) { + if (!kioskList.isEmpty()) { // if not set get any entry Object[] keySet = kioskList.keySet().toArray(); return getExtractorById(keySet[0].toString(), nextPageUrl, localization); @@ -91,7 +94,7 @@ public class KioskList { public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization) throws ExtractionException, IOException { KioskEntry ke = kioskList.get(kioskId); - if(ke == null) { + if (ke == null) { throw new ExtractionException("No kiosk found with the type: " + kioskId); } else { final KioskExtractor kioskExtractor = ke.extractorFactory.createNewKiosk(service, @@ -109,15 +112,15 @@ public class KioskList { } public KioskExtractor getExtractorByUrl(String url, String nextPageUrl) - throws ExtractionException, IOException{ + throws ExtractionException, IOException { return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization()); } public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization) throws ExtractionException, IOException { - for(Map.Entry e : kioskList.entrySet()) { + for (Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); - if(ke.handlerFactory.acceptUrl(url)) { + if (ke.handlerFactory.acceptUrl(url)) { return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java index 52510513..b7439c39 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.linkhandler; -import java.io.Serializable; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; +import java.io.Serializable; + public class LinkHandler implements Serializable { protected final String originalUrl; protected final String url; @@ -31,8 +31,8 @@ public class LinkHandler implements Serializable { public String getId() { return id; } - + public String getBaseUrl() throws ParsingException { - return Utils.getBaseUrl(url); + return Utils.getBaseUrl(url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 3890c2b2..6bba7b4e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -35,9 +35,9 @@ public abstract class LinkHandlerFactory { public abstract boolean onAcceptUrl(final String url) throws ParsingException; public String getUrl(String id, String baseUrl) throws ParsingException { - return getUrl(id); + return getUrl(id); } - + /////////////////////////////////// // Logic /////////////////////////////////// @@ -47,7 +47,7 @@ public abstract class LinkHandlerFactory { final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } - + public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if (url == null) throw new IllegalArgumentException("url can not be null"); if (!acceptUrl(url)) { @@ -55,9 +55,9 @@ public abstract class LinkHandlerFactory { } final String id = getId(url); - return new LinkHandler(url, getUrl(id,baseUrl), id); + return new LinkHandler(url, getUrl(id, baseUrl), id); } - + public LinkHandler fromId(String id) throws ParsingException { if (id == null) throw new IllegalArgumentException("id can not be null"); final String url = getUrl(id); @@ -82,5 +82,5 @@ public abstract class LinkHandlerFactory { throw fe; } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 83955265..60e1db48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -1,29 +1,35 @@ package org.schabi.newpipe.extractor.linkhandler; -import java.util.ArrayList; -import java.util.List; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Utils; +import java.util.ArrayList; +import java.util.List; + public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /////////////////////////////////// // To Override /////////////////////////////////// - public List getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);} - public String getSortFilter(String url) throws ParsingException {return ""; } + public List getContentFilter(String url) throws ParsingException { + return new ArrayList<>(0); + } + + public String getSortFilter(String url) throws ParsingException { + return ""; + } + public abstract String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException; - + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - return getUrl(id, contentFilter, sortFilter); + return getUrl(id, contentFilter, sortFilter); } /////////////////////////////////// // Logic /////////////////////////////////// - + @Override public ListLinkHandler fromUrl(String url) throws ParsingException { String baseUrl = Utils.getBaseUrl(url); @@ -32,7 +38,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { @Override public ListLinkHandler fromUrl(String url, String baseUrl) throws ParsingException { - if(url == null) throw new IllegalArgumentException("url may not be null"); + if (url == null) throw new IllegalArgumentException("url may not be null"); return new ListLinkHandler(super.fromUrl(url, baseUrl), getContentFilter(url), getSortFilter(url)); } @@ -41,7 +47,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public ListLinkHandler fromId(String id) throws ParsingException { return new ListLinkHandler(super.fromId(id), new ArrayList(0), ""); } - + @Override public ListLinkHandler fromId(String id, String baseUrl) throws ParsingException { return new ListLinkHandler(super.fromId(id, baseUrl), new ArrayList(0), ""); @@ -53,7 +59,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { final String url = getUrl(id, contentFilters, sortFilter); return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } - + public ListLinkHandler fromQuery(String id, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { @@ -61,10 +67,11 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } - + /** * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, * however it should not be overridden by the actual implementation. + * * @param id * @return the url coresponding to id without any filters applied */ @@ -76,7 +83,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public String getUrl(String id, String baseUrl) throws ParsingException { return getUrl(id, new ArrayList(0), "", baseUrl); } - + /** * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java index 6c1893c7..67dd7e49 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java @@ -24,6 +24,7 @@ public class SearchQueryHandler extends ListLinkHandler { /** * Returns the search string. Since ListQIHandler is based on ListLinkHandler * getSearchString() is equivalent to calling getId(). + * * @return the search string */ public String getSearchString() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java index 997e2d09..55c76b78 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java @@ -13,14 +13,19 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { @Override public abstract String getUrl(String querry, List contentFilter, String sortFilter) throws ParsingException; - public String getSearchString(String url) { return "";} + + public String getSearchString(String url) { + return ""; + } /////////////////////////////////// // Logic /////////////////////////////////// @Override - public String getId(String url) { return getSearchString(url); } + public String getId(String url) { + return getSearchString(url); + } @Override public SearchQueryHandler fromQuery(String querry, @@ -35,9 +40,12 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { /** * It's not mandatorry for NewPipe to handle the Url + * * @param url * @return */ @Override - public boolean onAcceptUrl(String url) { return false; } + public boolean onAcceptUrl(String url) { + return false; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java index 7461462a..02b89087 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java @@ -3,7 +3,10 @@ package org.schabi.newpipe.extractor.localization; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; public class Localization implements Serializable { public static final Localization DEFAULT = new Localization("en", "GB"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index aae9b14c..fe20135f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -21,8 +21,9 @@ public class TimeAgoParser { /** * Creates a helper to parse upload dates in the format '2 days ago'. *

- * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items. + * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items. *

+ * * @param patternsHolder An object that holds the "time ago" patterns, special cases, and the language word separator. */ public TimeAgoParser(PatternsHolder patternsHolder) { @@ -164,6 +165,7 @@ public class TimeAgoParser { /** * Marks the time as approximated by setting minutes, seconds and milliseconds to 0. + * * @param calendarTime Time to be marked as approximated */ private void markApproximatedTime(Calendar calendarTime) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java index bad31ea2..206fb68f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; @@ -34,7 +33,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; /** * Collector for search results - * + *

* This collector can handle the following extractor types: *

    *
  • {@link StreamInfoItemExtractor}
    • @@ -59,11 +58,11 @@ public class InfoItemsSearchCollector extends InfoItemsCollector getInitialPage() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray events = conferenceData.getArray("events"); - for(int i = 0; i < events.size(); i++) { + for (int i = 0; i < events.size(); i++) { collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i))); } return new InfoItemsPage<>(collector, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java index 580f79b0..f9800b46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java @@ -32,7 +32,7 @@ public class MediaCCCConferenceKiosk extends KioskExtractor { public InfoItemsPage getInitialPage() throws IOException, ExtractionException { JsonArray conferences = doc.getArray("conferences"); ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId()); - for(int i = 0; i < conferences.size(); i++) { + for (int i = 0; i < conferences.size(); i++) { collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i))); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 429445f2..6d740337 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -50,7 +50,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); searchItems.reset(); - if(getLinkHandler().getContentFilters().contains(CONFERENCES) + if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { searchConferences(getSearchString(), @@ -58,7 +58,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { searchItems); } - if(getLinkHandler().getContentFilters().contains(EVENTS) + if (getLinkHandler().getContentFilters().contains(EVENTS) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { JsonArray events = doc.getArray("events"); @@ -82,8 +82,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - if(getLinkHandler().getContentFilters().contains(EVENTS) - || getLinkHandler().getContentFilters().contains(ALL) + if (getLinkHandler().getContentFilters().contains(EVENTS) + || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) { final String site; final String url = getUrl(); @@ -94,17 +94,17 @@ public class MediaCCCSearchExtractor extends SearchExtractor { throw new ExtractionException("Could not parse json.", jpe); } } - if(getLinkHandler().getContentFilters().contains(CONFERENCES) + if (getLinkHandler().getContentFilters().contains(CONFERENCES) || getLinkHandler().getContentFilters().contains(ALL) || getLinkHandler().getContentFilters().isEmpty()) - conferenceKiosk.fetchPage(); + conferenceKiosk.fetchPage(); } private void searchConferences(String searchString, - List channelItems, - InfoItemsSearchCollector collector) { - for(final ChannelInfoItem item : channelItems) { - if(item.getName().toUpperCase().contains( + List channelItems, + InfoItemsSearchCollector collector) { + for (final ChannelInfoItem item : channelItems) { + if (item.getName().toUpperCase().contains( searchString.toUpperCase())) { collector.commit(new ChannelInfoItemExtractor() { @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 08f21ef7..c499758d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -117,17 +117,17 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getAudioStreams() throws IOException, ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List audioStreams = new ArrayList<>(); - for(int i = 0; i < recordings.size(); i++) { + for (int i = 0; i < recordings.size(); i++) { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); - if(mimeType.startsWith("audio")) { + if (mimeType.startsWith("audio")) { //first we need to resolve the actual video data from CDN final MediaFormat mediaFormat; - if(mimeType.endsWith("opus")) { + if (mimeType.endsWith("opus")) { mediaFormat = MediaFormat.OPUS; - } else if(mimeType.endsWith("mpeg")) { + } else if (mimeType.endsWith("mpeg")) { mediaFormat = MediaFormat.MP3; - } else if(mimeType.endsWith("ogg")){ + } else if (mimeType.endsWith("ogg")) { mediaFormat = MediaFormat.OGG; } else { throw new ExtractionException("Unknown media format: " + mimeType); @@ -143,16 +143,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor { public List getVideoStreams() throws IOException, ExtractionException { final JsonArray recordings = data.getArray("recordings"); final List videoStreams = new ArrayList<>(); - for(int i = 0; i < recordings.size(); i++) { + for (int i = 0; i < recordings.size(); i++) { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); - if(mimeType.startsWith("video")) { + if (mimeType.startsWith("video")) { //first we need to resolve the actual video data from CDN final MediaFormat mediaFormat; - if(mimeType.endsWith("webm")) { + if (mimeType.endsWith("webm")) { mediaFormat = MediaFormat.WEBM; - } else if(mimeType.endsWith("mp4")) { + } else if (mimeType.endsWith("mp4")) { mediaFormat = MediaFormat.MPEG_4; } else { throw new ExtractionException("Unknown media format: " + mimeType); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index b1207a9a..3c181767 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -15,9 +15,9 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory @Override public String getId(String url) throws ParsingException { - if(url.startsWith("https://api.media.ccc.de/public/conferences/")) { + if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { return url.replace("https://api.media.ccc.de/public/conferences/", ""); - } else if(url.startsWith("https://media.ccc.de/c/")) { + } else if (url.startsWith("https://media.ccc.de/c/")) { return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); } else { throw new ParsingException("Could not get id from url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java index 603f1ea2..d44dc538 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import java.util.List; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index fb6c705a..dcc0b29b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -15,7 +15,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, CONFERENCES, EVENTS diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index fc6bf39e..bf291a60 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -7,8 +7,8 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getId(String url) throws ParsingException { - if(url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q=")) { + if (url.startsWith("https://api.media.ccc.de/public/events/") && + !url.contains("?q=")) { return url.replace("https://api.media.ccc.de/public/events/", ""); } throw new ParsingException("Could not get id from url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 992b8eb6..17d08580 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube; -import java.io.IOException; - +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -10,22 +11,20 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import java.io.IOException; public class PeertubeInstance { - + private final String url; private String name; public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); - + public PeertubeInstance(String url) { this.url = url; this.name = "PeerTube"; } - - public PeertubeInstance(String url , String name) { + + public PeertubeInstance(String url, String name) { this.url = url; this.name = name; } @@ -33,22 +32,22 @@ public class PeertubeInstance { public String getUrl() { return url; } - + public void fetchInstanceMetaData() throws Exception { Downloader downloader = NewPipe.getDownloader(); Response response = null; - + try { response = downloader.get(url + "/api/v1/config"); } catch (ReCaptchaException | IOException e) { throw new Exception("unable to configure instance " + url, e); } - - if(null == response || StringUtil.isBlank(response.responseBody())) { + + if (null == response || StringUtil.isBlank(response.responseBody())) { throw new Exception("unable to configure instance " + url); } - - try { + + try { JsonObject json = JsonParser.object().from(response.responseBody()); this.name = JsonUtils.getString(json, "instance.name"); } catch (JsonParserException | ParsingException e) { @@ -59,5 +58,5 @@ public class PeertubeInstance { public String getName() { return name; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index c84e4e5b..7821aa29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -1,17 +1,16 @@ package org.schabi.newpipe.extractor.services.peertube; +import com.grack.nanojson.JsonObject; +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.TimeZone; -import org.jsoup.helper.StringUtil; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import com.grack.nanojson.JsonObject; - public class PeertubeParsingHelper { private PeertubeParsingHelper() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 47271732..29cc2b1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -1,49 +1,35 @@ package org.schabi.newpipe.extractor.services.peertube; -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.extractors.*; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + public class PeertubeService extends StreamingService { - + private PeertubeInstance instance; - + public PeertubeService(int id) { this(id, PeertubeInstance.defaultInstance); } - + public PeertubeService(int id, PeertubeInstance instance) { super(id, "PeerTube", asList(VIDEO, COMMENTS)); - this.instance = instance; + this.instance = instance; } @Override @@ -117,15 +103,15 @@ public class PeertubeService extends StreamingService { public String getBaseUrl() { return instance.getUrl(); } - + public PeertubeInstance getInstance() { return this.instance; } - + public void setInstance(PeertubeInstance instance) { this.instance = instance; } - + @Override public KioskList getKioskList() throws ExtractionException { KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() { @@ -155,6 +141,6 @@ public class PeertubeService extends StreamingService { return list; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 8b214670..4d1d00f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -1,7 +1,9 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -17,21 +19,18 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; +import java.io.IOException; public class PeertubeChannelExtractor extends ChannelExtractor { - + private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + private JsonObject json; private final String baseUrl; @@ -45,7 +44,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { String value; try { value = JsonUtils.getString(json, "avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -71,7 +70,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public String getDescription() throws ParsingException { try { return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { + } catch (ParsingException e) { return "No description"; } } @@ -86,18 +85,18 @@ public class PeertubeChannelExtractor extends ChannelExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract channel streams", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + } @Override @@ -110,26 +109,26 @@ public class PeertubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for kiosk info", e); } } - + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { PeertubeParsingHelper.validate(json); Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube kiosk info"); } return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; @@ -138,30 +137,30 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } - + @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { + } else { throw new ExtractionException("Unable to extract peertube channel data"); } - + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } @@ -172,14 +171,14 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube channel data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube channel data"); + if (null == json) throw new ExtractionException("Unable to extract peertube channel data"); } @Override public String getName() throws ParsingException { return JsonUtils.getString(json, "displayName"); } - + @Override public String getOriginalUrl() throws ParsingException { return baseUrl + "/accounts/" + getId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index b7aba4fa..34477459 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -16,9 +17,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeCommentsExtractor extends CommentsExtractor { @@ -26,10 +25,10 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { super(service, uiHandler); } @@ -38,7 +37,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public String getName() throws ParsingException { return "Comments"; } - + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); @@ -49,18 +48,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract comments info", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); collector.commit(extractor); } } - + } @Override @@ -73,18 +72,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for comments info", e); } } - + CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube comments info"); @@ -97,7 +96,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -105,17 +104,17 @@ public class PeertubeCommentsExtractor extends CommentsExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 0bcce9fc..b7cbcec7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.ServiceList; @@ -9,8 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; - public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @@ -34,7 +33,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -49,13 +48,13 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac public String getTextualPublishedTime() throws ParsingException { return JsonUtils.getString(item, "createdAt"); } - + @Override public DateWrapper getPublishedTime() throws ParsingException { String textualUploadDate = getTextualPublishedTime(); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public int getLikeCount() throws ParsingException { return -1; @@ -67,7 +66,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac try { Document doc = Jsoup.parse(htmlText); return doc.body().text(); - } catch(Exception e) { + } catch (Exception e) { return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); } } @@ -83,7 +82,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - } catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -100,5 +99,5 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac String host = JsonUtils.getString(item, "account.host"); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 98d20b53..81cd165d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -1,7 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -10,7 +8,9 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -public class PeertubePlaylistExtractor extends PlaylistExtractor{ +import java.io.IOException; + +public class PeertubePlaylistExtractor extends PlaylistExtractor { public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -74,7 +74,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor{ @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { // TODO Auto-generated method stub - + } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index ff021f73..3840bf76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemExtractor; @@ -18,9 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeSearchExtractor extends SearchExtractor { @@ -28,10 +27,10 @@ public class PeertubeSearchExtractor extends SearchExtractor { private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; - + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); } @@ -40,7 +39,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { public String getSearchSuggestion() throws ParsingException { return null; } - + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); @@ -48,27 +47,27 @@ public class PeertubeSearchExtractor extends SearchExtractor { } private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { - + final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - + JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract search info", e); } - + String baseUrl = getBaseUrl(); - for(Object c: contents) { - if(c instanceof JsonObject) { + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + return collector; - + } @Override @@ -81,17 +80,17 @@ public class PeertubeSearchExtractor extends SearchExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for search info", e); } } - - if(json != null) { + + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl)); } else { throw new ExtractionException("Unable to get peertube search info"); @@ -103,7 +102,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -111,20 +110,20 @@ public class PeertubeSearchExtractor extends SearchExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index a4dafbfc..76ead5d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,13 +1,9 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; @@ -24,12 +20,14 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearch import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; - import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; public class PeertubeStreamExtractor extends StreamExtractor { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index ca85270a..c4ed7730 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -8,58 +9,56 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; -import com.grack.nanojson.JsonObject; - public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - + protected final JsonObject item; private final String baseUrl; - + public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) { this.item = item; this.baseUrl = baseUrl; } - + @Override public String getUrl() throws ParsingException { String uuid = JsonUtils.getString(item, "uuid"); return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl(); } - + @Override public String getThumbnailUrl() throws ParsingException { String value = JsonUtils.getString(item, "thumbnailPath"); return baseUrl + value; } - + @Override public String getName() throws ParsingException { return JsonUtils.getString(item, "name"); } - + @Override public boolean isAd() throws ParsingException { return false; } - + @Override public long getViewCount() throws ParsingException { Number value = JsonUtils.getNumber(item, "views"); return value.longValue(); } - + @Override public String getUploaderUrl() throws ParsingException { String name = JsonUtils.getString(item, "account.name"); String host = JsonUtils.getString(item, "account.host"); return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); } - + @Override public String getUploaderName() throws ParsingException { return JsonUtils.getString(item, "account.displayName"); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(item, "publishedAt"); @@ -75,12 +74,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public StreamType getStreamType() throws ParsingException { return StreamType.VIDEO_STREAM; } - + @Override public long getDuration() throws ParsingException { Number value = JsonUtils.getNumber(item, "duration"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java index a2b24614..17062900 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import java.util.List; + public class PeertubeSubscriptionExtractor extends SubscriptionExtractor { public PeertubeSubscriptionExtractor(StreamingService service, List supportedSources) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java index 7e2ed4d9..a028e77a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; -import java.util.Collections; -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -public class PeertubeSuggestionExtractor extends SuggestionExtractor{ +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class PeertubeSuggestionExtractor extends SuggestionExtractor { public PeertubeSuggestionExtractor(StreamingService service) { super(service); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index 1cb42046..e6c1a1b5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -1,7 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import java.io.IOException; - +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -16,17 +17,15 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; +import java.io.IOException; public class PeertubeTrendingExtractor extends KioskExtractor { - + private static final String START_KEY = "start"; private static final String COUNT_KEY = "count"; private static final int ITEMS_PER_PAGE = 12; private static final String START_PATTERN = "start=(\\d*)"; - + private InfoItemsPage initPage; private long total; @@ -49,19 +48,19 @@ public class PeertubeTrendingExtractor extends KioskExtractor { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract kiosk info", e); } - + String baseUrl = getBaseUrl(); - for(Object c: contents) { - if(c instanceof JsonObject) { + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); collector.commit(extractor); } } - + } @Override @@ -74,18 +73,18 @@ public class PeertubeTrendingExtractor extends KioskExtractor { public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { Response response = getDownloader().get(pageUrl); JsonObject json = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { json = JsonParser.object().from(response.responseBody()); } catch (Exception e) { throw new ParsingException("Could not parse json data for kiosk info", e); } } - + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - if(json != null) { + if (json != null) { Number number = JsonUtils.getNumber(json, "total"); - if(number != null) this.total = number.longValue(); + if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { throw new ExtractionException("Unable to get peertube kiosk info"); @@ -98,7 +97,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; this.initPage = getPage(pageUrl); } - + private String getNextPageUrl(String prevPageUrl) { String prevStart; try { @@ -106,17 +105,17 @@ public class PeertubeTrendingExtractor extends KioskExtractor { } catch (RegexException e) { return ""; } - if(StringUtil.isBlank(prevStart)) return ""; + if (StringUtil.isBlank(prevStart)) return ""; long nextStart = 0; try { nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; } catch (NumberFormatException e) { return ""; } - - if(nextStart >= total) { + + if (nextStart >= total) { return ""; - }else { + } else { return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index 640f99e5..c1e3f570 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); @@ -27,7 +27,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, searchFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java index 9faa3492..9c833750 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory(); @@ -33,10 +33,10 @@ public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilter, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { return baseUrl + String.format(COMMENTS_ENDPOINT, id); } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java index 64e291a5..dd1bd77f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import java.util.List; + public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory(); @@ -23,12 +23,12 @@ public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; } - + @Override public String getId(String url) throws ParsingException { return Parser.matchGroup1(ID_PATTERN, url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 69591167..8acf3e80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; - import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String CHARSET_UTF_8 = "UTF-8"; @@ -21,9 +21,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { String baseUrl = ServiceList.PeerTube.getBaseUrl(); - return getUrl(searchString, contentFilters, sortFilter, baseUrl); + return getUrl(searchString, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String searchString, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { try { @@ -38,6 +38,6 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { VIDEOS }; + return new String[]{VIDEOS}; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index 3ee7744c..222dc129 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -24,7 +24,7 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, baseUrl); } - + @Override public String getUrl(String id, String baseUrl) { return baseUrl + VIDEO_ENDPOINT + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java index c0773c20..1d5609c1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -1,26 +1,26 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.schabi.newpipe.extractor.ServiceList; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; - public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { - - + + private static final PeertubeTrendingLinkHandlerFactory instance = new PeertubeTrendingLinkHandlerFactory(); - + public static final Map KIOSK_MAP; public static final Map REVERSE_KIOSK_MAP; public static final String KIOSK_TRENDING = "Trending"; public static final String KIOSK_MOST_LIKED = "Most liked"; public static final String KIOSK_RECENT = "Recently added"; public static final String KIOSK_LOCAL = "Local"; - + static { Map map = new HashMap<>(); map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending"); @@ -28,24 +28,24 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); KIOSK_MAP = Collections.unmodifiableMap(map); - + Map reverseMap = new HashMap<>(); - for(Map.Entry entry : KIOSK_MAP.entrySet()){ + for (Map.Entry entry : KIOSK_MAP.entrySet()) { reverseMap.put(entry.getValue(), entry.getKey()); } REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap); } - + public static PeertubeTrendingLinkHandlerFactory getInstance() { return instance; } - + @Override public String getUrl(String id, List contentFilters, String sortFilter) { String baseUrl = ServiceList.PeerTube.getBaseUrl(); return getUrl(id, contentFilters, sortFilter, baseUrl); } - + @Override public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { return String.format(KIOSK_MAP.get(id), baseUrl); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 54541217..853b6fc1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -86,7 +86,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - if(streamInfoItemsCollector == null) { + if (streamInfoItemsCollector == null) { computeNextPageAndGetStreams(); } return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); @@ -94,7 +94,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getNextPageUrl() throws ExtractionException { - if(nextPageUrl == null) { + if (nextPageUrl == null) { computeNextPageAndGetStreams(); } return nextPageUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java index ed5d9a98..be1b9fcd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index b773a935..b9f4894d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -12,8 +12,8 @@ import javax.annotation.Nonnull; import java.io.IOException; public class SoundcloudChartsExtractor extends KioskExtractor { - private StreamInfoItemsCollector collector = null; - private String nextPageUrl = null; + private StreamInfoItemsCollector collector = null; + private String nextPageUrl = null; public SoundcloudChartsExtractor(StreamingService service, ListLinkHandler linkHandler, @@ -68,7 +68,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public String getNextPageUrl() throws IOException, ExtractionException { - if(nextPageUrl == null) { + if (nextPageUrl == null) { computNextPageAndStreams(); } return nextPageUrl; @@ -77,7 +77,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - if(collector == null) { + if (collector == null) { computNextPageAndStreams(); } return new InfoItemsPage<>(collector, getNextPageUrl()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 82216075..ed50ea54 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -33,7 +33,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 private static String clientId; - + private SoundcloudParsingHelper() { } @@ -103,7 +103,7 @@ public class SoundcloudParsingHelper { /** * Call the endpoint "/resolve" of the api.

    - * + *

    * See https://developers.soundcloud.com/docs/api/reference#resolve */ public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 5bc30972..ca08f66c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java index 17e4e7f2..c7c882a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 9ed340fa..42cd8ae0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -4,7 +4,10 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java index 90057120..dffcc91f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java @@ -25,7 +25,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto try { String url = "https://api-v2.soundcloud.com/search"; - if(contentFilter.size() > 0) { + if (contentFilter.size() > 0) { switch (contentFilter.get(0)) { case TRACKS: url += "/tracks"; @@ -58,7 +58,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, TRACKS, USERS, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index def9c386..30f2fc87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,31 +1,26 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import static java.util.Collections.singletonList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; + public class SoundcloudService extends StreamingService { public SoundcloudService(int id) { super(id, "SoundCloud", singletonList(AUDIO)); } - + @Override public String getBaseUrl() { return "https://soundcloud.com"; @@ -110,15 +105,15 @@ public class SoundcloudService extends StreamingService { return new SoundcloudSubscriptionExtractor(this); } - @Override - public ListLinkHandlerFactory getCommentsLHFactory() { - return null; - } + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return null; + } - @Override + @Override public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { return null; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 18fffa41..2919c89b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -4,7 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -183,7 +185,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { try { JsonObject mp3UrlObject = JsonParser.object().from(res); // Links in this file are also only valid for a short period. - audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), + audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), MediaFormat.MP3, 128)); } catch (JsonParserException e) { throw new ParsingException("Could not parse streamable url", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java index 3476eab1..fc578411 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java index b12f0467..924c4698 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java @@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 6137f029..eae3bcb9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,13 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; - -import java.util.List; - import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; @@ -15,28 +7,22 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import javax.annotation.Nonnull; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; /* * Created by Christian Schabesberger on 23.08.15. @@ -125,7 +111,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d675cb25..cc37cbda 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -196,7 +196,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) - throws ParsingException { + throws ParsingException { String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); if (!loadMoreHtmlDataRaw.isEmpty()) { return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 1c377cc5..bcacf7cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -22,7 +21,9 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import static java.util.Collections.singletonList; @@ -58,14 +59,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException { - + JsonArray arr; try { arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); } catch (Exception e) { return ""; } - if(arr.isEmpty()) { + if (arr.isEmpty()) { return ""; } String continuation; @@ -107,11 +108,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException { - + JsonArray contents; try { contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items"); - }catch(Exception e) { + } catch (Exception e) { //no comments return; } @@ -119,12 +120,12 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { List comments; try { comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to get parse youtube comments", e); } - - for(Object c: comments) { - if(c instanceof JsonObject) { + + for (Object c : comments) { + if (c instanceof JsonObject) { CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl(), getTimeAgoParser()); collector.commit(extractor); } @@ -132,7 +133,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void fetchTitle(JsonArray contents) { - if(null == title) { + if (null == title) { try { title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); } catch (Exception e) { @@ -190,7 +191,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { int endIndex = doc.indexOf(end, beginIndex); return doc.substring(beginIndex, endIndex); } - + public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException { try { return JsonUtils.getString(object, "simpleText"); @@ -198,7 +199,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { try { JsonArray arr = JsonUtils.getArray(object, "runs"); String result = ""; - for(int i=0; i getFrames() throws ExtractionException { - try { - final String script = doc.select("#player-api").first().siblingElements().select("script").html(); - int p = script.indexOf("ytplayer.config"); - if (p == -1) { - return Collections.emptyList(); - } - p = script.indexOf('{', p); - int e = script.indexOf("ytplayer.load", p); - if (e == -1) { - return Collections.emptyList(); - } - JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); - final String resp = jo.getObject("args").getString("player_response"); - jo = JsonParser.object().from(resp); - final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); - final String url = spec[0]; - final ArrayList result = new ArrayList<>(spec.length - 1); - for (int i = 1; i < spec.length; ++i) { - final String[] parts = spec[i].split("#"); - if (parts.length != 8) { - continue; - } - final int frameWidth = Integer.parseInt(parts[0]); - final int frameHeight = Integer.parseInt(parts[1]); - final int totalCount = Integer.parseInt(parts[2]); - final int framesPerPageX = Integer.parseInt(parts[3]); - final int framesPerPageY = Integer.parseInt(parts[4]); - final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; - final List urls; - if (baseUrl.contains("$M")) { - final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); - urls = new ArrayList<>(totalPages); - for (int j = 0; j < totalPages; j++) { - urls.add(baseUrl.replace("$M", String.valueOf(j))); - } - } else { - urls = Collections.singletonList(baseUrl); - } - result.add(new Frameset( - urls, - frameWidth, - frameHeight, - totalCount, - framesPerPageX, - framesPerPageY - )); - } - result.trimToSize(); - return result; - } catch (Exception e) { - throw new ExtractionException(e); - } - } + @Nonnull + @Override + public List getFrames() throws ExtractionException { + try { + final String script = doc.select("#player-api").first().siblingElements().select("script").html(); + int p = script.indexOf("ytplayer.config"); + if (p == -1) { + return Collections.emptyList(); + } + p = script.indexOf('{', p); + int e = script.indexOf("ytplayer.load", p); + if (e == -1) { + return Collections.emptyList(); + } + JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); + final String resp = jo.getObject("args").getString("player_response"); + jo = JsonParser.object().from(resp); + final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); + final String url = spec[0]; + final ArrayList result = new ArrayList<>(spec.length - 1); + for (int i = 1; i < spec.length; ++i) { + final String[] parts = spec[i].split("#"); + if (parts.length != 8) { + continue; + } + final int frameWidth = Integer.parseInt(parts[0]); + final int frameHeight = Integer.parseInt(parts[1]); + final int totalCount = Integer.parseInt(parts[2]); + final int framesPerPageX = Integer.parseInt(parts[3]); + final int framesPerPageY = Integer.parseInt(parts[4]); + final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; + final List urls; + if (baseUrl.contains("$M")) { + final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); + urls = new ArrayList<>(totalPages); + for (int j = 0; j < totalPages; j++) { + urls.add(baseUrl.replace("$M", String.valueOf(j))); + } + } else { + urls = Collections.singletonList(baseUrl); + } + result.add(new Frameset( + urls, + frameWidth, + frameHeight, + totalCount, + framesPerPageX, + framesPerPageY + )); + } + result.trimToSize(); + return result; + } catch (Exception e) { + throw new ExtractionException(e); + } + } @Override public String getHost() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 514b4fb0..eda36d21 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -42,6 +42,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { /** * Creates an extractor of StreamInfoItems from a YouTube page. + * * @param item The page element * @param timeAgoParser A parser of the textual dates or {@code null}. */ @@ -68,10 +69,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private boolean isPremiumVideo() { Element premiumSpan = item.select("span[class=\"standalone-collection-badge-renderer-red-text\"]").first(); - if(premiumSpan == null) return false; + if (premiumSpan == null) return false; // if this span has text it most likely says ("Free Video") so we can play this - if(premiumSpan.hasText()) return false; + if (premiumSpan.hasText()) return false; return true; } @@ -233,7 +234,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return Long.parseLong(Utils.removeNonDigitCharacters(input)); } catch (NumberFormatException e) { // if this happens the video probably has no views - if (!input.isEmpty()){ + if (!input.isEmpty()) { return 0; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index d41d90c5..8ed35c19 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -3,9 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; @@ -57,12 +57,12 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { String response = dl.get(url, getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" - response = response.substring(3, response.length()-1); + response = response.substring(3, response.length() - 1); try { JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray()); for (Object suggestion : collection) { if (!(suggestion instanceof JsonArray)) continue; - String suggestionStr = ((JsonArray)suggestion).getString(0); + String suggestionStr = ((JsonArray) suggestion).getString(0); if (suggestionStr == null) continue; suggestions.add(suggestionStr); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index a4014191..be5820de 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -88,8 +88,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for(Element ul : uls) { - for(final Element li : ul.children()) { + for (Element ul : uls) { + for (final Element li : ul.children()) { final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 62a0e737..29bbdfdf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -31,7 +31,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { } String path = urlObj.getPath(); - if (!path.equals("/watch" ) && !path.equals("/playlist")) { + if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index ba5ee152..c1760074 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -26,7 +26,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory final String url = "https://www.youtube.com/results" + "?q=" + URLEncoder.encode(searchString, CHARSET_UTF_8); - if(contentFilters.size() > 0) { + if (contentFilters.size() > 0) { switch (contentFilters.get(0)) { case VIDEOS: return url + "&sp=EgIQAVAU"; case CHANNELS: return url + "&sp=EgIQAlAU"; @@ -44,7 +44,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return new String[] { + return new String[]{ ALL, VIDEOS, CHANNELS, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java index 2d4010dd..cc92cd7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java @@ -1,66 +1,64 @@ package org.schabi.newpipe.extractor.stream; -import javax.annotation.Nullable; -import java.util.Collection; import java.util.List; public final class Frameset { - private List urls; - private int frameWidth; - private int frameHeight; - private int totalCount; - private int framesPerPageX; - private int framesPerPageY; + private List urls; + private int frameWidth; + private int frameHeight; + private int totalCount; + private int framesPerPageX; + private int framesPerPageY; - public Frameset(List urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) { - this.urls = urls; - this.totalCount = totalCount; - this.frameWidth = frameWidth; - this.frameHeight = frameHeight; - this.framesPerPageX = framesPerPageX; - this.framesPerPageY = framesPerPageY; - } + public Frameset(List urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) { + this.urls = urls; + this.totalCount = totalCount; + this.frameWidth = frameWidth; + this.frameHeight = frameHeight; + this.framesPerPageX = framesPerPageX; + this.framesPerPageY = framesPerPageY; + } - /** - * @return list of urls to images with frames - */ - public List getUrls() { - return urls; - } + /** + * @return list of urls to images with frames + */ + public List getUrls() { + return urls; + } - /** - * @return total count of frames - */ - public int getTotalCount() { - return totalCount; - } + /** + * @return total count of frames + */ + public int getTotalCount() { + return totalCount; + } - /** - * @return maximum frames count by x - */ - public int getFramesPerPageX() { - return framesPerPageX; - } + /** + * @return maximum frames count by x + */ + public int getFramesPerPageX() { + return framesPerPageX; + } - /** - * @return maximum frames count by y - */ - public int getFramesPerPageY() { - return framesPerPageY; - } + /** + * @return maximum frames count by y + */ + public int getFramesPerPageY() { + return framesPerPageY; + } - /** - * @return width of a one frame, in pixels - */ - public int getFrameWidth() { - return frameWidth; - } + /** + * @return width of a one frame, in pixels + */ + public int getFrameWidth() { + return frameWidth; + } - /** - * @return height of a one frame, in pixels - */ - public int getFrameHeight() { - return frameHeight; - } + /** + * @return height of a one frame, in pixels + */ + public int getFrameHeight() { + return frameHeight; + } } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 1d49e253..b48eeab2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.MediaFormat; + import java.io.Serializable; import java.util.List; -import org.schabi.newpipe.extractor.MediaFormat; - /** * Creates a stream object from url, format and optional torrent url */ @@ -22,19 +22,19 @@ public abstract class Stream implements Serializable { /** * Instantiates a new stream object. * - * @param url the url + * @param url the url * @param format the format */ public Stream(String url, MediaFormat format) { this(url, null, format); } - + /** * Instantiates a new stream object. * - * @param url the url + * @param url the url * @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent - * @param format the format + * @param format the format */ public Stream(String url, String torrentUrl, MediaFormat format) { this.url = url; @@ -76,7 +76,7 @@ public abstract class Stream implements Serializable { public String getUrl() { return url; } - + /** * Gets the torrent url. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index fdc56972..587101cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -27,7 +27,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; @@ -77,6 +76,7 @@ public abstract class StreamExtractor extends Extractor { /** * This will return the url to the thumbnail of the stream. Try to return the medium resolution here. + * * @return The url of the thumbnail. * @throws ParsingException */ @@ -85,6 +85,7 @@ public abstract class StreamExtractor extends Extractor { /** * This is the stream description. + * * @return The description of the stream/video or Description.emptyDescription if the description is empty. * @throws ParsingException */ @@ -93,6 +94,7 @@ public abstract class StreamExtractor extends Extractor { /** * Get the age limit. + * * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit * @throws ParsingException if an error occurs while parsing */ @@ -100,6 +102,7 @@ public abstract class StreamExtractor extends Extractor { /** * This should return the length of a video in seconds. + * * @return The length of the stream in seconds. * @throws ParsingException */ @@ -109,6 +112,7 @@ public abstract class StreamExtractor extends Extractor { * If the url you are currently handling contains a time stamp/seek, you can return the * position it represents here. * If the url has no time stamp simply return zero. + * * @return the timestamp in seconds * @throws ParsingException */ @@ -117,6 +121,7 @@ public abstract class StreamExtractor extends Extractor { /** * The count of how many people have watched the video/listened to the audio stream. * If the current stream has no view count or its not available simply return -1 + * * @return amount of views. * @throws ParsingException */ @@ -125,6 +130,7 @@ public abstract class StreamExtractor extends Extractor { /** * The Amount of likes a video/audio stream got. * If the current stream has no likes or its not available simply return -1 + * * @return the amount of likes the stream got * @throws ParsingException */ @@ -133,6 +139,7 @@ public abstract class StreamExtractor extends Extractor { /** * The Amount of dislikes a video/audio stream got. * If the current stream has no dislikes or its not available simply return -1 + * * @return the amount of likes the stream got * @throws ParsingException */ @@ -144,6 +151,7 @@ public abstract class StreamExtractor extends Extractor { * ChannelExtractor, * so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects * this url. + * * @return the url to the page of the creator/uploader of the stream or an empty String * @throws ParsingException */ @@ -153,6 +161,7 @@ public abstract class StreamExtractor extends Extractor { /** * The name of the creator/uploader of the stream. * If the name is not available you can simply return an empty string. + * * @return the name of the creator/uploader of the stream or an empty String * @throws ParsingException */ @@ -162,6 +171,7 @@ public abstract class StreamExtractor extends Extractor { /** * The url to the image file/profile picture/avatar of the creator/uploader of the stream. * If the url is not available you can return an empty String. + * * @return The url of the image file of the uploader or an empty String * @throws ParsingException */ @@ -171,20 +181,24 @@ public abstract class StreamExtractor extends Extractor { /** * Get the dash mpd url. If you don't know what a dash MPD is you can read about it * here. + * * @return the url as a string or an empty string * @throws ParsingException if an error occurs while reading */ - @Nonnull public abstract String getDashMpdUrl() throws ParsingException; + @Nonnull + public abstract String getDashMpdUrl() throws ParsingException; /** * I am not sure if this is in use, and how this is used. However the frontend is missing support * for HLS streams. Prove me if I am wrong. Please open an * issue, * or fix this description if you know whats up with this. + * * @return The Url to the hls stream. * @throws ParsingException */ - @Nonnull public abstract String getHlsUrl() throws ParsingException; + @Nonnull + public abstract String getHlsUrl() throws ParsingException; /** * This should return a list of available @@ -192,6 +206,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of audio only streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -205,6 +220,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of combined video and streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -218,6 +234,7 @@ public abstract class StreamExtractor extends Extractor { * You can also return null or an empty list, however be aware that if you don't return anything * in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as * a failed extraction procedure. + * * @return a list of video and streams in the format of AudioStream * @throws IOException * @throws ExtractionException @@ -228,6 +245,7 @@ public abstract class StreamExtractor extends Extractor { * This will return a list of available * Subtitless. * If no subtitles are available an empty list can returned. + * * @return a list of available subtitles or an empty list * @throws IOException * @throws ExtractionException @@ -240,6 +258,7 @@ public abstract class StreamExtractor extends Extractor { * Subtitless. * given by a specific type. * If no subtitles in that specific format are available an empty list can returned. + * * @param format the media format by which the subtitles should be filtered * @return a list of available subtitles or an empty list * @throws IOException @@ -250,6 +269,7 @@ public abstract class StreamExtractor extends Extractor { /** * Get the StreamType. + * * @return the type of the stream * @throws ParsingException */ @@ -259,6 +279,7 @@ public abstract class StreamExtractor extends Extractor { * should return the url of the next stream. NewPipe will automatically play * the next stream if the user wants that. * If the next stream is is not available simply return null + * * @return the InfoItem of the next stream * @throws IOException * @throws ExtractionException @@ -271,6 +292,7 @@ public abstract class StreamExtractor extends Extractor { * be disabled by the user later in the frontend. * This list MUST NOT contain the next available video as this should be return through getNextStream() * If is is not available simply return null + * * @return a list of InfoItems showing the related videos/streams * @throws IOException * @throws ExtractionException @@ -279,6 +301,7 @@ public abstract class StreamExtractor extends Extractor { /** * Should return a list of Frameset object that contains preview of stream frames + * * @return list of preview frames or empty list if frames preview is not supported or not found for specified stream * @throws IOException * @throws ExtractionException @@ -302,6 +325,7 @@ public abstract class StreamExtractor extends Extractor { /** * Override this function if the format of time stamp in the url is not the same format as that form youtube. * Honestly I don't even know the time stamp fromat of youtube. + * * @param regexPattern * @return the sime stamp/seek for the video in seconds * @throws ParsingException @@ -357,6 +381,7 @@ public abstract class StreamExtractor extends Extractor { * If the host is not available, or if the service doesn't use * a federated system, but a centralised system, * you can simply return an empty string. + * * @return the host of the stream or an empty String. * @throws ParsingException */ @@ -366,6 +391,7 @@ public abstract class StreamExtractor extends Extractor { /** * The privacy of the stream (Eg. Public, Private, Unlisted…). * If the privacy is not available you can simply return an empty string. + * * @return the privacy of the stream or an empty String. * @throws ParsingException */ @@ -375,6 +401,7 @@ public abstract class StreamExtractor extends Extractor { /** * The name of the category of the stream. * If the category is not available you can simply return an empty string. + * * @return the category of the stream or an empty String. * @throws ParsingException */ @@ -384,6 +411,7 @@ public abstract class StreamExtractor extends Extractor { /** * The name of the licence of the stream. * If the licence is not available you can simply return an empty string. + * * @return the licence of the stream or an empty String. * @throws ParsingException */ @@ -395,6 +423,7 @@ public abstract class StreamExtractor extends Extractor { * If the language is not available you can simply return null. * If the language is provided by a language code, you can return * new Locale(language_code); + * * @return the locale language of the stream or null. * @throws ParsingException */ @@ -404,6 +433,7 @@ public abstract class StreamExtractor extends Extractor { /** * The list of tags of the stream. * If the tag list is not available you can simply return an empty list. + * * @return the list of tags of the stream or an empty list. * @throws ParsingException */ @@ -416,6 +446,7 @@ public abstract class StreamExtractor extends Extractor { * (support button). * If the support information are not available, * you can simply return an empty String. + * * @return the support information of the stream or an empty String. * @throws ParsingException */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 113f3a87..828d4520 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -31,6 +31,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream type + * * @return the stream type * @throws ParsingException thrown if there is an error in the extraction */ @@ -38,6 +39,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Check if the stream is an ad. + * * @return {@code true} if the stream is an ad. * @throws ParsingException thrown if there is an error in the extraction */ @@ -45,6 +47,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the stream duration in seconds + * * @return the stream duration in seconds * @throws ParsingException thrown if there is an error in the extraction */ @@ -52,6 +55,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Parses the number of views + * * @return the number of views or -1 for live streams * @throws ParsingException thrown if there is an error in the extraction */ @@ -59,6 +63,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Get the uploader name + * * @return the uploader name * @throws ParsingException if parsing fails */ @@ -80,9 +85,9 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** * Extracts the upload date and time of this item and parses it. *

    - * If the service doesn't provide an exact time, an approximation can be returned. - *
    - * If the service doesn't provide any date at all, then {@code null} should be returned. + * If the service doesn't provide an exact time, an approximation can be returned. + *
    + * If the service doesn't provide any date at all, then {@code null} should be returned. *

    * * @return The date and time (can be approximated) this item was uploaded or {@code null}. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 85dce8f0..b81052b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -101,8 +101,8 @@ public class StreamInfoItemsCollector extends InfoItemsCollector getStreamInfoItemList() { List siiList = new Vector<>(); - for(InfoItem ii : super.getItems()) { - if(ii instanceof StreamInfoItem) { + for (InfoItem ii : super.getItems()) { + if (ii instanceof StreamInfoItem) { siiList.add((StreamInfoItem) ii); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java index d0e09ac1..8f16942e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -52,7 +52,7 @@ public class SubtitlesStream extends Stream implements Serializable { @Override public boolean equalStats(Stream cmp) { - return super.equalStats(cmp)&& + return super.equalStats(cmp) && cmp instanceof SubtitlesStream && code.equals(((SubtitlesStream) cmp).code) && autoGenerated == ((SubtitlesStream) cmp).autoGenerated; @@ -67,7 +67,7 @@ public class SubtitlesStream extends Stream implements Serializable { } public Locale getLocale() { - return locale; + return locale; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java index f4531d36..67219240 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java @@ -36,7 +36,7 @@ public class VideoStream extends Stream { this.resolution = resolution; this.isVideoOnly = isVideoOnly; } - + public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) { this(url, torrentUrl, format, resolution, false); } @@ -56,6 +56,7 @@ public class VideoStream extends Stream { /** * Get the video resolution + * * @return the video resolution */ public String getResolution() { @@ -64,8 +65,9 @@ public class VideoStream extends Stream { /** * Check if the video is video only. - * + *

    * Video only streams have no audio + * * @return {@code true} if this stream is vid */ public boolean isVideoOnly() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 7b954a4c..b8b191ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor.utils; -import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.services.youtube.ItagItem; @@ -109,11 +109,11 @@ public class DashMpdParser { *

    * It has video, video only and audio streams and will only add to the list if it don't * find a similar stream in the respective lists (calling {@link Stream#equalStats}). - * + *

    * Info about dash MPD can be found here - * @see www.brendanlog.com * * @param streamInfo where the parsed streams will be added + * @see www.brendanlog.com */ public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { @@ -160,7 +160,7 @@ public class DashMpdParser { final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - if(segmentationList == null) { + if (segmentationList == null) { final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { audioStreams.add(audioStream); @@ -172,7 +172,7 @@ public class DashMpdParser { } else { boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - if(segmentationList == null) { + if (segmentationList == null) { final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, @@ -191,7 +191,7 @@ public class DashMpdParser { itag.resolutionString, isVideoOnly); - if(isVideoOnly) { + if (isVideoOnly) { segmentedVideoOnlyStreams.add(videoStream); } else { segmentedVideoStreams.add(videoStream); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index 28300649..c6ef73ca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; -import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; @@ -31,7 +30,7 @@ public class ExtractorHelper { public static List getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { try { InfoItemsCollector collector = extractor.getRelatedStreams(); - if(collector == null) return Collections.emptyList(); + if (collector == null) return Collections.emptyList(); info.addAllErrors(collector.getErrors()); //noinspection unchecked @@ -41,5 +40,5 @@ public class ExtractorHelper { return Collections.emptyList(); } } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index 80e0dd58..ab916379 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -1,16 +1,14 @@ package org.schabi.newpipe.extractor.utils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; public class JsonUtils { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index b03e304c..30786420 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -1,5 +1,10 @@ package org.schabi.newpipe.extractor.utils; +import org.nibor.autolink.LinkExtractor; +import org.nibor.autolink.LinkSpan; +import org.nibor.autolink.LinkType; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; @@ -9,11 +14,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.nibor.autolink.LinkExtractor; -import org.nibor.autolink.LinkSpan; -import org.nibor.autolink.LinkType; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - /* * Created by Christian Schabesberger on 02.02.16. * @@ -51,7 +51,7 @@ public class Parser { public static String matchGroup1(String pattern, String input) throws RegexException { return matchGroup(pattern, input, 1); } - + public static String matchGroup1(Pattern pattern, String input) throws RegexException { return matchGroup(pattern, input, 1); } @@ -60,7 +60,7 @@ public class Parser { Pattern pat = Pattern.compile(pattern); return matchGroup(pat, input, group); } - + public static String matchGroup(Pattern pat, String input, int group) throws RegexException { Matcher mat = pat.matcher(input); boolean foundMatch = mat.find(); @@ -102,7 +102,7 @@ public class Parser { .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW)) .build(); Iterable linkss = linkExtractor.extractLinks(txt); - for(LinkSpan ls : linkss) { + for (LinkSpan ls : linkss) { links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex())); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 3489b6d6..ebd0ba16 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.utils; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.List; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - public class Utils { private Utils() { @@ -35,6 +35,7 @@ public class Utils { *

  • 1.23K -> 1230
    • *
  • 1.23M -> 1230000
    • * + * * @param numberWord string to be converted to a long * @return a long * @throws NumberFormatException @@ -165,17 +166,17 @@ public class Utils { return setsNoPort || usesDefaultPort; } - + public static String removeUTF8BOM(String s) { if (s.startsWith("\uFEFF")) { s = s.substring(1); } if (s.endsWith("\uFEFF")) { - s = s.substring(0, s.length()-1); + s = s.substring(0, s.length() - 1); } return s; } - + public static String getBaseUrl(String url) throws ParsingException { URL uri; try { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index b9c67c4e..15a68a73 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -45,6 +45,6 @@ public class MediaCCCConferenceExtractorTest { @Test public void testGetInitalPage() throws Exception { - assertEquals(97,extractor.getInitialPage().getItems().size()); + assertEquals(97, extractor.getInitialPage().getItems().size()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java index 252e4dbb..90ebf028 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java @@ -24,7 +24,7 @@ public class MediaCCCConferenceListExtractorTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); + extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); extractor.fetchPage(); } @@ -49,8 +49,8 @@ public class MediaCCCConferenceListExtractorTest { } private boolean contains(List itemList, String name) { - for(InfoItem item : itemList) { - if(item.getName().equals(name)) + for (InfoItem item : itemList) { + if (item.getName().equals(name)) return true; } return false; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java index 781f9138..25116af3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java @@ -22,7 +22,7 @@ public class MediaCCCOggTest { public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); + extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); extractor.fetchPage(); } @@ -33,7 +33,7 @@ public class MediaCCCOggTest { @Test public void getAudioStreamsContainOgg() throws Exception { - for(AudioStream stream : extractor.getAudioStreams()) { + for (AudioStream stream : extractor.getAudioStreams()) { assertEquals("OGG", stream.getFormat().toString()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java index dc38338e..cd33c562 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java @@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorAllTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("c3", Arrays.asList(new String[0]), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -37,8 +37,8 @@ public class MediaCCCSearchExtractorAllTest { @Test public void testIfChannelInfoItemsAvailable() { boolean isAvialable = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof ChannelInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof ChannelInfoItem) { isAvialable = true; } } @@ -48,8 +48,8 @@ public class MediaCCCSearchExtractorAllTest { @Test public void testIfStreamInfoitemsAvailable() { boolean isAvialable = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { isAvialable = true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java index 18fbc234..f9ce2334 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java @@ -27,7 +27,7 @@ public class MediaCCCSearchExtractorConferencesTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("c3", Arrays.asList(new String[]{"conferences"}), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -35,7 +35,7 @@ public class MediaCCCSearchExtractorConferencesTest { @Test public void testReturnTypeChannel() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { assertTrue("Item is not of type channel", item instanceof ChannelInfoItem); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java index 1ed65a8d..29f90caf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java @@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorEventsTest { @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory() .fromQuery("linux", Arrays.asList(new String[]{"events"}), "")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -65,7 +65,7 @@ public class MediaCCCSearchExtractorEventsTest { @Test public void testReturnTypeStream() throws Exception { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 71b1e779..ed9dcee7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -28,7 +28,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest { public static void setUpClass() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); + extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 57b6360b..a1d1f383 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -1,15 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; - import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -20,6 +10,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + /** * Test for {@link PeertubeChannelExtractor} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index 633307a2..f9b0c69d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeChannelLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index e0a1d189..1241b0a8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -1,12 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.io.IOException; -import java.util.List; - import org.jsoup.helper.StringUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -18,6 +11,13 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + public class PeertubeCommentsExtractorTest { private static PeertubeCommentsExtractor extractor; @@ -59,11 +59,11 @@ public class PeertubeCommentsExtractorTest { assertTrue(result); } - + @Test public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); - for(CommentsInfoItem c: comments.getItems()) { + for (CommentsInfoItem c : comments.getItems()) { assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); assertFalse(StringUtil.isBlank(c.getAuthorName())); assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); @@ -82,8 +82,8 @@ public class PeertubeCommentsExtractorTest { } private boolean findInComments(List comments, String comment) { - for(CommentsInfoItem c: comments) { - if(c.getCommentText().contains(comment)) { + for (CommentsInfoItem c : comments) { + if (c.getCommentText().contains(comment)) { return true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java index fcc27c54..f6fb91f7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeCommentsLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java index 9ad29102..ac04a01e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubePlaylistLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index a81bf34c..23adf353 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -1,19 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Locale; -import java.util.TimeZone; - import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -26,6 +12,18 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; + +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link StreamExtractor} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java index 893b6eb8..af44f644 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeStreamLinkHandlerFactory} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index bb4e1529..5d2d8478 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -1,13 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -import java.util.List; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -17,6 +9,11 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.util.List; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeTrendingExtractor} */ @@ -53,10 +50,10 @@ public class PeertubeTrendingExtractorTest { @Test public void testGetStreams() throws Exception { ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if(!page.getErrors().isEmpty()) { + if (!page.getErrors().isEmpty()) { System.err.println("----------"); List errors = page.getErrors(); - for(Throwable e: errors) { + for (Throwable e : errors) { e.printStackTrace(); System.err.println("----------"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java index d192e8be..033be650 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -1,9 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeTrendingLinkHandlerFactory} */ @@ -48,13 +48,13 @@ public class PeertubeTrendingLinkHandlerFactoryTest { public void acceptUrl() throws ParsingException { assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe")); - + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local")); assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe")); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java index 08dd7a0a..030bd6c5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertTrue; - import org.junit.Test; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import static org.junit.Assert.assertTrue; + /** * Test for {@link PeertubeSearchExtractor} */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java index dd9c6deb..eeebf526 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -1,10 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -15,6 +10,9 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + /** * Test for {@link PeertubeSearchExtractor} */ @@ -38,15 +36,15 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB @Test public void testResultList_FirstElement() { InfoItem firstInfoItem = itemsPage.getItems().get(0); - + assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde")); } @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -67,7 +65,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } @@ -75,7 +73,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB assertFalse("First and second page are equal", equals); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", - secondPage.getNextPageUrl()); + secondPage.getNextPageUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java index 23e3100c..d7bd92aa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -1,14 +1,14 @@ package org.schabi.newpipe.extractor.services.peertube.search; -import static org.junit.Assert.assertEquals; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + public class PeertubeSearchQHTest { - + @BeforeClass public static void setUpClass() throws Exception { // setting instance might break test when running in parallel @@ -18,7 +18,7 @@ public class PeertubeSearchQHTest { @Test public void testRegularValues() throws Exception { assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans",PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans", PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index d96b4c6e..dea1d170 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; @@ -48,10 +47,10 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetStreams() throws Exception { ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if(!page.getErrors().isEmpty()) { + if (!page.getErrors().isEmpty()) { System.err.println("----------"); List errors = page.getErrors(); - for(Throwable e: errors) { + for (Throwable e : errors) { e.printStackTrace(); System.err.println("----------"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index 1c4f4159..16089aab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -1,10 +1,12 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.junit.*; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; public class SoundcloudParsingHelperTest { @BeforeClass diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java index abd26951..04e7e0ab 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java @@ -38,7 +38,7 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } @@ -57,8 +57,8 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx @Test public void testOnlyContainChannels() { - for(InfoItem item : itemsPage.getItems()) { - if(!(item instanceof ChannelInfoItem)) { + for (InfoItem item : itemsPage.getItems()) { + if (!(item instanceof ChannelInfoItem)) { fail("The following item is no channel item: " + item.toString()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java index 685c52d7..ca0d7d59 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java @@ -60,8 +60,8 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -80,7 +80,7 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 251ae246..317bd4fa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,14 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -16,10 +7,14 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; + /** * Test for {@link ChannelExtractor} */ @@ -489,7 +484,6 @@ public class YoutubeChannelExtractorTest { } - public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index 25f097a0..fc409ffa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -38,10 +38,10 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); - + assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Gronkh")); assertTrue(linkHandler.acceptUrl("https://invidio.us/user/Netzkino/videos")); - + assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); } @@ -60,10 +60,10 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); - + assertEquals("user/Gronkh", linkHandler.fromUrl("https://invidio.us/user/Gronkh").getId()); assertEquals("user/Netzkino", linkHandler.fromUrl("https://invidio.us/user/Netzkino/videos").getId()); - + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA").getId()); assertEquals("channel/UClq42foiSgl7sSpLupnugGA", linkHandler.fromUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 796238f9..aebb6bf1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -123,7 +123,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId()); assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId()); } - + @Test public void testAcceptInvidioUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU")); @@ -133,7 +133,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE")); } - + @Test public void testGetInvidioIdfromUrl() throws ParsingException { assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java index c8e23c06..c85d6b52 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java @@ -1,24 +1,24 @@ package org.schabi.newpipe.extractor.services.youtube; - /* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2017 - * YoutubeTrendingKioskInfoTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingKioskInfoTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ import org.junit.BeforeClass; import org.junit.Test; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java index 8b19b11e..729254ad 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorBaseTest.java @@ -3,10 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.Test; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 91cbebd9..63fc0375 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -57,7 +57,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto @Ignore @Test public void testOnlyContainChannels() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { if (!(item instanceof ChannelInfoItem)) { fail("The following item is no channel item: " + item.toString()); } @@ -66,7 +66,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto @Test public void testChannelUrl() { - for(InfoItem item : itemsPage.getItems()) { + for (InfoItem item : itemsPage.getItems()) { if (item instanceof ChannelInfoItem) { ChannelInfoItem channel = (ChannelInfoItem) item; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 49e3520d..3078d873 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -6,7 +6,6 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -76,8 +75,8 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; - for(InfoItem item : itemsPage.getItems()) { - if(item instanceof StreamInfoItem) { + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof StreamInfoItem) { hasStreams = true; } } @@ -96,7 +95,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas boolean equals = true; for (int i = 0; i < secondPage.getItems().size() && i < itemsPage.getItems().size(); i++) { - if(!secondPage.getItems().get(i).getUrl().equals( + if (!secondPage.getItems().get(i).getUrl().equals( itemsPage.getItems().get(i).getUrl())) { equals = false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java index 7df7b0cb..656e197a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java @@ -1,15 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.search; -import static java.util.Collections.singletonList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -19,6 +9,14 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Collections.singletonList; +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + public class YoutubeSearchPagingTest { private static ListExtractor.InfoItemsPage page1; private static ListExtractor.InfoItemsPage page2; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index c3d3f93c..60bbf2ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -1,21 +1,18 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.Test; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*; public class YoutubeSearchQHTest { @Test public void testRegularValues() throws Exception { assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans",YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 6afc7212..3710b4ee 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -117,7 +117,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { streams.addAll(extractor.getVideoStreams()); streams.addAll(extractor.getVideoOnlyStreams()); - assertTrue(Integer.toString(streams.size()),streams.size() > 0); + assertTrue(Integer.toString(streams.size()), streams.size() > 0); for (VideoStream s : streams) { assertTrue(s.getUrl(), s.getUrl().contains(HTTPS)); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index e2c0c7bb..2798e8db 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -19,7 +19,7 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; -import static java.util.Objects.*; +import static java.util.Objects.requireNonNull; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -126,7 +126,7 @@ public class YoutubeStreamExtractorDefaultTest { public void testGetUploaderUrl() throws ParsingException { String url = extractor.getUploaderUrl(); if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") && - !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { + !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { fail("Uploader url is neither the music channel one nor the Vevo one"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java index dc8f2b04..925a647c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java @@ -1,47 +1,46 @@ package org.schabi.newpipe.extractor.utils; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.junit.Test; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import org.junit.Test; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.util.List; + +import static org.junit.Assert.assertTrue; public class JsonUtilsTest { - + @Test public void testGetValueFlat() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); - assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); + assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); } - + @Test public void testGetValueNested() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2"))); } - + @Test public void testGetArray() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "batters.batter"); assertTrue(!arr.isEmpty()); } - + @Test public void testGetValues() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping"); List types = JsonUtils.getValues(arr, "type"); assertTrue(types.contains("Chocolate with Sprinkles")); - + } } From 6822574588cde36580cfd564363dc632e0f7446a Mon Sep 17 00:00:00 2001 From: Tobias Groza Date: Mon, 10 Feb 2020 09:50:19 +0100 Subject: [PATCH 024/149] Fix typos Co-Authored-By: B0pol --- .../org/schabi/newpipe/extractor/MediaFormat.java | 2 +- .../newpipe/extractor/comments/CommentsInfo.java | 1 - .../linkhandler/ListLinkHandlerFactory.java | 2 +- .../linkhandler/SearchQueryHandlerFactory.java | 2 +- .../extractors/PeertubeChannelExtractor.java | 6 +++--- .../extractors/PeertubePlaylistExtractor.java | 1 - .../extractors/PeertubeTrendingExtractor.java | 2 +- .../youtube/extractors/YoutubePlaylistExtractor.java | 2 +- .../newpipe/extractor/stream/StreamExtractor.java | 12 ++++++------ 9 files changed, 14 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index 0b6052f0..6936568a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -115,7 +115,7 @@ public enum MediaFormat { } /** - * Get the media format by it's id. + * Get the media format by its id. * * @param id the id * @return the id of the media format or null. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java index d5f4c434..e07c6746 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java @@ -14,7 +14,6 @@ public class CommentsInfo extends ListInfo { private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) { super(serviceId, listUrlIdHandler, name); - // TODO Auto-generated constructor stub } public static CommentsInfo getInfo(String url) throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 60e1db48..9ea478b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -69,7 +69,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /** - * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, + * For making ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, * however it should not be overridden by the actual implementation. * * @param id diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java index 55c76b78..50977e20 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java @@ -39,7 +39,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { } /** - * It's not mandatorry for NewPipe to handle the Url + * It's not mandatory for NewPipe to handle the Url * * @param url * @return diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 4d1d00f5..6476be8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -124,7 +124,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { if (number != null) this.total = number.longValue(); collectStreamsFrom(collector, json, pageUrl); } else { - throw new ExtractionException("Unable to get peertube kiosk info"); + throw new ExtractionException("Unable to get PeerTube kiosk info"); } return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); } @@ -158,7 +158,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); } else { - throw new ExtractionException("Unable to extract peertube channel data"); + throw new ExtractionException("Unable to extract PeerTube channel data"); } String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; @@ -171,7 +171,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube channel data", e); } - if (null == json) throw new ExtractionException("Unable to extract peertube channel data"); + if (null == json) throw new ExtractionException("Unable to extract PeerTube channel data"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 81cd165d..1a96111a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -73,7 +73,6 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { - // TODO Auto-generated method stub } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index e6c1a1b5..e4bc1dff 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -49,7 +49,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { try { contents = (JsonArray) JsonUtils.getValue(json, "data"); } catch (Exception e) { - throw new ParsingException("unable to extract kiosk info", e); + throw new ParsingException("Unable to extract kiosk info", e); } String baseUrl = getBaseUrl(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 0f3f56e6..a03b1d5a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -68,7 +68,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getBannerUrl() { return ""; // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement this in the fornt end + // Whoever is willing to implement this should also implement this in the front end. } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 587101cf..4a3cdff4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -128,7 +128,7 @@ public abstract class StreamExtractor extends Extractor { public abstract long getViewCount() throws ParsingException; /** - * The Amount of likes a video/audio stream got. + * The amount of likes a video/audio stream got. * If the current stream has no likes or its not available simply return -1 * * @return the amount of likes the stream got @@ -137,7 +137,7 @@ public abstract class StreamExtractor extends Extractor { public abstract long getLikeCount() throws ParsingException; /** - * The Amount of dislikes a video/audio stream got. + * The amount of dislikes a video/audio stream got. * If the current stream has no dislikes or its not available simply return -1 * * @return the amount of likes the stream got @@ -276,7 +276,7 @@ public abstract class StreamExtractor extends Extractor { public abstract StreamType getStreamType() throws ParsingException; /** - * should return the url of the next stream. NewPipe will automatically play + * Should return the url of the next stream. NewPipe will automatically play * the next stream if the user wants that. * If the next stream is is not available simply return null * @@ -291,7 +291,7 @@ public abstract class StreamExtractor extends Extractor { * streams. If you don't like suggested streams you should implement them anyway since they can * be disabled by the user later in the frontend. * This list MUST NOT contain the next available video as this should be return through getNextStream() - * If is is not available simply return null + * If it is not available simply return null * * @return a list of InfoItems showing the related videos/streams * @throws IOException @@ -324,10 +324,10 @@ public abstract class StreamExtractor extends Extractor { /** * Override this function if the format of time stamp in the url is not the same format as that form youtube. - * Honestly I don't even know the time stamp fromat of youtube. + * Honestly I don't even know the time stamp format of YouTube. * * @param regexPattern - * @return the sime stamp/seek for the video in seconds + * @return the time stamp/seek for the video in seconds * @throws ParsingException */ protected long getTimestampSeconds(String regexPattern) throws ParsingException { From 0a4115286b46356b34bbcb2308261f86acd380c8 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 15 Feb 2020 12:52:29 +0100 Subject: [PATCH 025/149] Apply one suggestion --- .../org/schabi/newpipe/extractor/stream/StreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 4a3cdff4..900707ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -337,7 +337,7 @@ public abstract class StreamExtractor extends Extractor { } catch (Parser.RegexException e) { // catch this instantly since an url does not necessarily have to have a time stamp - // -2 because well the testing system will then know its the regex that failed :/ + // -2 because the testing system will consequently know that the regex failed // not good i know return -2; } From a129c65c63ee95462239b9681bf14c1562abfe07 Mon Sep 17 00:00:00 2001 From: Tobias Groza Date: Sun, 16 Feb 2020 23:59:31 +0100 Subject: [PATCH 026/149] more improvements of comments Co-Authored-By: Stypox --- .../java/org/schabi/newpipe/extractor/StreamingService.java | 2 +- .../newpipe/extractor/services/peertube/PeertubeInstance.java | 2 +- .../peertube/extractors/PeertubeChannelExtractor.java | 2 +- .../services/youtube/extractors/YoutubeCommentsExtractor.java | 2 +- .../services/youtube/extractors/YoutubePlaylistExtractor.java | 2 +- .../org/schabi/newpipe/extractor/stream/StreamExtractor.java | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 110efe04..e21b17f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -263,7 +263,7 @@ public abstract class StreamingService { public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException { ListLinkHandlerFactory llhf = getCommentsLHFactory(); - if (null == llhf) { + if (llhf == null) { return null; } return getCommentsExtractor(llhf.fromUrl(url)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index 17d08580..82b67539 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -43,7 +43,7 @@ public class PeertubeInstance { throw new Exception("unable to configure instance " + url, e); } - if (null == response || StringUtil.isBlank(response.responseBody())) { + if (response == null || StringUtil.isBlank(response.responseBody())) { throw new Exception("unable to configure instance " + url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 6476be8b..d53fb7ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -171,7 +171,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube channel data", e); } - if (null == json) throw new ExtractionException("Unable to extract PeerTube channel data"); + if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index bcacf7cf..78eb59f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -133,7 +133,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private void fetchTitle(JsonArray contents) { - if (null == title) { + if (title == null) { try { title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index a03b1d5a..520bda80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -68,7 +68,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getBannerUrl() { return ""; // Banner can't be handled by frontend right now. - // Whoever is willing to implement this should also implement this in the front end. + // Whoever is willing to implement this should also implement it in the frontend. } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 900707ab..78a5fbf3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -335,10 +335,10 @@ public abstract class StreamExtractor extends Extractor { try { timeStamp = Parser.matchGroup1(regexPattern, getOriginalUrl()); } catch (Parser.RegexException e) { - // catch this instantly since an url does not necessarily have to have a time stamp + // catch this instantly since a url does not necessarily have a timestamp // -2 because the testing system will consequently know that the regex failed - // not good i know + // not good, I know return -2; } From a63371a2c5b0b19a9daf8ff3d902a27844c4045f Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 21 Feb 2020 22:29:20 +0100 Subject: [PATCH 027/149] Version 0.18.4 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a40ee853..fcfa4fa9 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.3' + version 'v0.18.4' group 'com.github.TeamNewPipe' repositories { From 39bf1ff1eba51a059944d9f54872f9596823f8d1 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 17 Feb 2020 09:55:06 +0100 Subject: [PATCH 028/149] Add ytInitialData regex --- .../youtube/extractors/YoutubeChannelExtractor.java | 11 +++++++++++ .../youtube/extractors/YoutubeStreamExtractor.java | 11 +++++++++++ .../java/org/schabi/newpipe/DownloaderTestImpl.java | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index cc37cbda..24c7f4d1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -49,6 +49,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; private Document doc; + private JsonObject ytInitialData; public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -59,6 +60,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor { String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; final Response response = downloader.get(channelUrl, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); + ytInitialData = getInitialData(); + } + + private JsonObject getInitialData() throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 9568c7ff..65946436 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -88,6 +88,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private final Map videoInfoPage = new HashMap<>(); private JsonObject playerResponse; + private JsonObject ytInitialData; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -736,6 +737,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { isAgeRestricted = false; } playerResponse = getPlayerResponse(); + ytInitialData = getInitialData(); if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); @@ -809,6 +811,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + private JsonObject getInitialData() throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } + } + @Nonnull private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { try { diff --git a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java index 948975a0..4fdab77e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java +++ b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java @@ -20,7 +20,7 @@ import java.util.Map; public class DownloaderTestImpl extends Downloader { - private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; + private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/73.0"; private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en"; private static DownloaderTestImpl instance = null; From be3e20e263a88ed63c901164e6875ddf82279210 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 17 Feb 2020 11:02:43 +0100 Subject: [PATCH 029/149] Adapt most channel changes --- .../extractors/YoutubeChannelExtractor.java | 37 +++++++++++++------ .../schabi/newpipe/extractor/utils/Utils.java | 6 +-- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 24c7f4d1..103a59c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -23,6 +23,9 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; + /* * Created by Christian Schabesberger on 25.07.16. * @@ -91,7 +94,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getId() throws ParsingException { try { - return doc.select("meta[itemprop=\"channelId\"]").first().attr("content"); + return doc.select("meta[property=\"og:url\"]").first().attr("content").replace(CHANNEL_URL_BASE, ""); + } catch (Exception ignored) {} + try { + return ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); } catch (Exception ignored) {} // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) @@ -118,7 +124,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { try { - return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src"); + return ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -127,11 +133,21 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); - String cssContent = el.html(); - String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent); + String url = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); + if (url.contains("s.ytimg.com") || url.contains("default_banner")) { + return null; + } + // the first characters of the banner URLs are different for each channel and some are not even valid URLs + if (url.startsWith("//")) { + url = url.substring(2); + } + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } - return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url; + return url; } catch (Exception e) { throw new ParsingException("Could not get Banner", e); } @@ -149,11 +165,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); - if (el != null) { - String elTitle = el.attr("title"); + final String simpleText = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText").getArray("runs").getObject(0).getString("text"); + if (simpleText != null) { try { - return Utils.mixedNumberWordToLong(elTitle); + return Utils.mixedNumberWordToLong(simpleText); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } @@ -166,7 +181,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getDescription() throws ParsingException { try { - return doc.select("meta[name=\"description\"]").first().attr("content"); + return ytInitialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); } catch (Exception e) { throw new ParsingException("Could not get channel description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index ebd0ba16..76aa2944 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -10,6 +10,9 @@ import java.util.List; public class Utils { + public static final String HTTP = "http://"; + public static final String HTTPS = "https://"; + private Utils() { //no instance } @@ -83,9 +86,6 @@ public class Utils { } } - private static final String HTTP = "http://"; - private static final String HTTPS = "https://"; - public static String replaceHttpWithHttps(final String url) { if (url == null) return null; From 7dcc9f159bb7280fd97f48af42cf5c2b3c182e27 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 17 Feb 2020 11:41:11 +0100 Subject: [PATCH 030/149] Stream: fix uploader avatar url --- .../youtube/extractors/YoutubeStreamExtractor.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 65946436..b60fa875 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -476,9 +476,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { String uploaderAvatarUrl = null; try { - uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first() - .select("img").first() - .attr("abs:data-thumb"); + uploaderAvatarUrl = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") + .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") + .getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail") + .getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception e) {//todo: add fallback method throw new ParsingException("Could not get uploader avatar url", e); } From 655c9997956147d2f16ef577e2c271d5ecb5d522 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 17 Feb 2020 17:30:22 +0100 Subject: [PATCH 031/149] Detect disabled subriber count correctly Fix parsing of kiosk name --- .../extractors/YoutubeChannelExtractor.java | 7 ++--- .../extractors/YoutubeTrendingExtractor.java | 26 ++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 103a59c4..544a9e46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -165,10 +165,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final String simpleText = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText").getArray("runs").getObject(0).getString("text"); - if (simpleText != null) { + final JsonObject subscriberInfo = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); + if (subscriberInfo != null) { try { - return Utils.mixedNumberWordToLong(simpleText); + + return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text")); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index be5820de..44f9c7e5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -20,6 +20,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; * along with NewPipe. If not, see . */ +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; import java.io.IOException; @@ -41,6 +45,7 @@ import java.io.IOException; public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; + private JsonObject initialData; public YoutubeTrendingExtractor(StreamingService service, ListLinkHandler linkHandler, @@ -55,6 +60,16 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + initialData = getInitialData(); + } + + private JsonObject getInitialData() throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } } @Override @@ -70,14 +85,17 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Nonnull @Override public String getName() throws ParsingException { + String name; try { - Element a = doc.select("a[href*=\"/feed/trending\"]").first(); - Element span = a.select("span[class*=\"display-name\"]").first(); - Element nameSpan = span.select("span").first(); - return nameSpan.text(); + name = initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title") + .getArray("runs").getObject(0).getString("text"); } catch (Exception e) { throw new ParsingException("Could not get Trending name", e); } + if (name != null && !name.isEmpty()) { + return name; + } + throw new ParsingException("Could not get Trending name"); } @Nonnull From 21253abd9b6d45695198733c16e638d67e597eb8 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 17 Feb 2020 17:52:29 +0100 Subject: [PATCH 032/149] Use Firefox ESR User-Agent: 68 --- .../src/test/java/org/schabi/newpipe/DownloaderTestImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java index 4fdab77e..e524ac8d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java +++ b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java @@ -20,7 +20,7 @@ import java.util.Map; public class DownloaderTestImpl extends Downloader { - private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/73.0"; + private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0"; private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en"; private static DownloaderTestImpl instance = null; From b705515da06ff2ebca28bed2bb513e4f94f6a94a Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 17 Feb 2020 18:58:12 +0100 Subject: [PATCH 033/149] Adapt most playlist fields --- .../extractors/YoutubePlaylistExtractor.java | 101 ++++++++++++++---- 1 file changed, 79 insertions(+), 22 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 520bda80..692b9cc3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; @@ -19,6 +20,7 @@ import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingH import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; @@ -29,6 +31,10 @@ import java.io.IOException; public class YoutubePlaylistExtractor extends PlaylistExtractor { private Document doc; + private JsonObject initialData; + private JsonObject uploaderInfo; + private JsonObject playlistInfo; + private JsonObject playlistVideos; public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -39,6 +45,62 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final String url = getUrl(); final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + initialData = getInitialData(); + uploaderInfo = getUploaderInfo(); + playlistInfo = getPlaylistInfo(); + playlistVideos = getPlaylistVideos(); + } + + private JsonObject getInitialData() throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } + } + + private JsonObject getUploaderInfo() throws ParsingException { + JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items"); + try { + JsonObject uploaderInfo = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer") + .getObject("videoOwner").getObject("videoOwnerRenderer"); + if (uploaderInfo != null) { + return uploaderInfo; + } + } catch (Exception ignored) {} + + // we might want to create a loop here instead of using duplicated code + try { + JsonObject uploaderInfo = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer") + .getObject("videoOwner").getObject("videoOwnerRenderer"); + if (uploaderInfo != null) { + return uploaderInfo; + } + } catch (Exception e) { + throw new ParsingException("Could not get uploader info", e); + } + throw new ParsingException("Could not get uploader info"); + } + + private JsonObject getPlaylistInfo() throws ParsingException { + try { + return initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items") + .getObject(0).getObject("playlistSidebarPrimaryInfoRenderer"); + } catch (Exception e) { + throw new ParsingException("Could not get PlaylistInfo", e); + } + } + + private JsonObject getPlaylistVideos() throws ParsingException { + try { + return initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer").getArray("contents") + .getObject(0).getObject("playlistVideoListRenderer"); + } catch (Exception e) { + throw new ParsingException("Could not get playlist info", e); + } } @Override @@ -50,7 +112,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { try { - return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text(); + String name = playlistInfo.getObject("title").getArray("runs").getObject(0).getString("text"); + if (name != null) return name; + } catch (Exception ignored) {} + try { + return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); } catch (Exception e) { throw new ParsingException("Could not get playlist name", e); } @@ -59,7 +125,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src"); + return playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception ignored) {} + try { + return initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") + .getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -75,8 +146,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public String getUploaderUrl() throws ParsingException { try { return YoutubeChannelExtractor.CHANNEL_URL_BASE + - doc.select("button[class*=\"yt-uix-subscription-button\"]") - .first().attr("data-channel-external-id"); + uploaderInfo.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } @@ -85,7 +155,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderName() throws ParsingException { try { - return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); + return uploaderInfo.getObject("title").getArray("runs").getObject(0).getString("text"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader name", e); } @@ -94,7 +164,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { try { - return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src"); + return uploaderInfo.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader avatar", e); } @@ -102,25 +172,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public long getStreamCount() throws ParsingException { - String input; - try { - input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text(); - } catch (IndexOutOfBoundsException e) { + String viewsText = getPlaylistInfo().getArray("stats").getObject(0).getArray("runs").getObject(0).getString("text"); + return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); + } catch (Exception e) { throw new ParsingException("Could not get video count from playlist", e); } - - try { - return Long.parseLong(Utils.removeNonDigitCharacters(input)); - } catch (NumberFormatException e) { - // When there's no videos in a playlist, there's no number in the "innerHtml", - // all characters that is not a number is removed, so we try to parse a empty string - if (!input.isEmpty()) { - return 0; - } else { - throw new ParsingException("Could not handle input: " + input, e); - } - } } @Nonnull From a34e5c18e1fc838715414542d55f9e3a638bdfd9 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 17 Feb 2020 20:24:48 +0100 Subject: [PATCH 034/149] Trending StreamInfoItemCollecetor --- .../YoutubeStreamInfoItemExtractor.java | 15 ++- .../extractors/YoutubeTrendingExtractor.java | 127 +++++++++--------- 2 files changed, 76 insertions(+), 66 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index eda36d21..1c127180 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonObject; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -35,7 +36,8 @@ import java.util.Date; public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - private final Element item; + private JsonObject videoInfoItem; + private Element item; private final TimeAgoParser timeAgoParser; private String cachedUploadDate; @@ -51,6 +53,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { this.timeAgoParser = timeAgoParser; } + /** + * Creates an extractor of StreamInfoItems from a YouTube page. + * + * @param videoInfoItem The JSON page element + * @param timeAgoParser A parser of the textual dates or {@code null}. + */ + public YoutubeStreamInfoItemExtractor(JsonObject videoInfoItem, @Nullable TimeAgoParser timeAgoParser) { + this.videoInfoItem = videoInfoItem; + this.timeAgoParser = timeAgoParser; + } + @Override public StreamType getStreamType() throws ParsingException { if (isLiveStream(item)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 44f9c7e5..05a33a35 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -20,6 +20,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; * along with NewPipe. If not, see . */ +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; @@ -34,7 +35,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Parser; @@ -102,85 +105,79 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public InfoItemsPage getInitialPage() throws ParsingException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]"); + JsonArray firstPageElements = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") + .getObject("expandedShelfContentsRenderer").getArray("items"); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Element ul : uls) { - for (final Element li : ul.children()) { - final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - try { - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } + for (Object ul : firstPageElements) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser) { + @Override + public String getUrl() throws ParsingException { + try { + String videoId = videoInfo.getString("videoId"); + return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); } + } - @Override - public String getName() throws ParsingException { - try { - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } + @Override + public String getName() throws ParsingException { + String name = null; + try { + name = videoInfo.getObject("title").getObject("accessibility") + .getObject("accessibilityData").getString("label"); + } catch (Exception ignored) { } + if (name != null && !name.isEmpty()) return name; + try { + name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text"); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get web page url for the video"); + } - @Override - public String getUploaderUrl() throws ParsingException { - try { - String link = getUploaderLink().attr("abs:href"); - if (link.isEmpty()) { - throw new IllegalArgumentException("is empty"); - } - return link; - } catch (Exception e) { - throw new ParsingException("Could not get Uploader name"); + @Override + public String getUploaderUrl() throws ParsingException { + try { + String id = videoInfo.getObject("ownerText").getArray("runs").getObject(0). + getObject("browseEndpoint").getString("browseId"); + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("is empty"); } + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); + } catch (Exception e) { + throw new ParsingException("Could not get Uploader url"); } + } - private Element getUploaderLink() { - // this url is not always in the form "/channel/..." - // sometimes Youtube provides urls in the from "/user/..." - Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first(); - return uploaderEl.select("a").first(); + @Override + public String getUploaderName() throws ParsingException { + try { + return videoInfo.getObject("ownerText").getArray("runs").getObject(0).getString("text"); + } catch (Exception e) { + throw new ParsingException("Could not get uploader name"); } + } - @Override - public String getUploaderName() throws ParsingException { - try { - return getUploaderLink().text(); - } catch (Exception e) { - throw new ParsingException("Could not get Uploader name"); - } + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return videoInfo.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - String url; - Element te = li.select("span[class=\"yt-thumb-simple\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); - } + } + }); } - return new InfoItemsPage<>(collector, getNextPageUrl()); + } } From 3ec422a54253e6267b6359924b7ee0c02a3b537e Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 18 Feb 2020 11:10:41 +0100 Subject: [PATCH 035/149] Fix like/dislike parsing --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b60fa875..a0ee0bf3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -388,9 +388,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String likesString = ""; try { - Element button = doc.select("button.like-button-renderer-like-button").first(); try { - likesString = button.select("span.yt-uix-button-content").first().text(); + likesString = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -413,7 +412,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { Element button = doc.select("button.like-button-renderer-dislike-button").first(); try { - dislikesString = button.select("span.yt-uix-button-content").first().text(); + dislikesString = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { From 12166ca60ce88544140e3aba1f88ff3c7356eefe Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 18 Feb 2020 13:05:11 +0100 Subject: [PATCH 036/149] [StreamExtractor] Add fallback method for uploader avatar url --- .../youtube/extractors/YoutubeStreamExtractor.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a0ee0bf3..4bb491d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -479,8 +479,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") .getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail") .getArray("thumbnails").getObject(0).getString("url"); + if (uploaderAvatarUrl != null && !uploaderAvatarUrl.isEmpty()) { + return uploaderAvatarUrl; + } + } catch (Exception ignored) {} - } catch (Exception e) {//todo: add fallback method + try { + uploaderAvatarUrl = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") + .getObject("results").getArray("contents").getObject(1).getObject("videoSecondaryInfoRenderer") + .getObject("owner").getObject("videoOwnerRenderer").getObject("thumbnail").getArray("thumbnails") + .getObject(0).getString("url"); + } catch (Exception e) { throw new ParsingException("Could not get uploader avatar url", e); } From 7bfc0e62c665863ac3e3be0716400eca58162be2 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 18 Feb 2020 17:04:22 +0100 Subject: [PATCH 037/149] Fix getRelatedStreams() in YoutubeStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 115 ++++++++++++++++-- 1 file changed, 107 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 4bb491d0..7c69d2e3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -25,7 +25,9 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; @@ -653,21 +655,118 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Override - public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + JsonArray results = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("secondaryResults").getObject("secondaryResults").getArray("results"); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); - Element ul = doc.select("ul[id=\"watch-related\"]").first(); - if (ul != null) { - for (Element li : ul.children()) { - // first check if we have a playlist. If so leave them out - if (li.select("a[class*=\"content-link\"]").first() != null) { - collector.commit(extractVideoPreviewInfo(li, timeAgoParser)); + for (Object ul : results) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer"); + + if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser) { + @Override + public StreamType getStreamType() { + return StreamType.VIDEO_STREAM; } - } + + @Override + public boolean isAd() { + return false; + } + + @Override + public String getUrl() throws ParsingException { + try { + String videoId = videoInfo.getString("videoId"); + return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); + } catch (Exception e) { + throw new ParsingException("Could not get url", e); + } + } + + @Override + public String getName() throws ParsingException { + String name = null; + try { + name = videoInfo.getObject("title").getString("simpleText"); + } catch (Exception ignored) {} + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get title"); + } + + @Override + public long getDuration() throws ParsingException { + try { + return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); + } catch (Exception e) { + throw new ParsingException("Could not get duration", e); + } + } + + @Override + public String getUploaderUrl() throws ParsingException { + try { + String id = videoInfo.getObject("longBylineText").getArray("runs") + .getObject(0).getObject("navigationEndpoint") + .getObject("browseEndpoint").getString("browseId"); + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("is empty"); + } + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); + } catch (Exception e) { + throw new ParsingException("Could not get uploader url"); + } + } + + @Nullable + @Override + public String getTextualUploadDate() { + return null; + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Override + public long getViewCount() throws ParsingException { + try { + String viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); + if (viewCount.equals("Recommended for you")) return -1; + return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); + } catch (Exception e) { + throw new ParsingException("Could not get view count", e); + } + } + + @Override + public String getUploaderName() throws ParsingException { + try { + return videoInfo.getObject("longBylineText").getArray("runs") + .getObject(0).getString("text"); + } catch (Exception e) { + throw new ParsingException("Could not get uploader name", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return videoInfo.getObject("thumbnail").getArray("thumbnails") + .getObject(0).getString("url"); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); } return collector; } catch (Exception e) { From 127c4d589323a48c5561e39a82c6a32e51afaa7f Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 19 Feb 2020 19:14:05 +0100 Subject: [PATCH 038/149] Fix getNextStream() in YoutubeStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 211 +++++++----------- 1 file changed, 82 insertions(+), 129 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 7c69d2e3..d8059ed4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -640,15 +640,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { public StreamInfoItem getNextStream() throws IOException, ExtractionException { assertPageFetched(); try { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonObject videoInfo = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("secondaryResults").getObject("secondaryResults").getArray("results") + .getObject(0).getObject("compactAutoplayRenderer").getArray("contents") + .getObject(0).getObject("compactVideoRenderer"); final TimeAgoParser timeAgoParser = getTimeAgoParser(); + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Elements watch = doc.select("div[class=\"watch-sidebar-section\"]"); - if (watch.size() < 1) { - return null;// prevent the snackbar notification "report error" on age-restricted videos - } - - collector.commit(extractVideoPreviewInfo(watch.first().select("li").first(), timeAgoParser)); + collector.commit(extractVideoPreviewInfo(videoInfo, timeAgoParser)); return collector.getItems().get(0); } catch (Exception e) { throw new ParsingException("Could not get next video", e); @@ -669,104 +668,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { for (Object ul : results) { final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer"); - if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser) { - @Override - public StreamType getStreamType() { - return StreamType.VIDEO_STREAM; - } - - @Override - public boolean isAd() { - return false; - } - - @Override - public String getUrl() throws ParsingException { - try { - String videoId = videoInfo.getString("videoId"); - return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); - } catch (Exception e) { - throw new ParsingException("Could not get url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = null; - try { - name = videoInfo.getObject("title").getString("simpleText"); - } catch (Exception ignored) {} - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get title"); - } - - @Override - public long getDuration() throws ParsingException { - try { - return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); - } catch (Exception e) { - throw new ParsingException("Could not get duration", e); - } - } - - @Override - public String getUploaderUrl() throws ParsingException { - try { - String id = videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); - if (id == null || id.isEmpty()) { - throw new IllegalArgumentException("is empty"); - } - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); - } catch (Exception e) { - throw new ParsingException("Could not get uploader url"); - } - } - - @Nullable - @Override - public String getTextualUploadDate() { - return null; - } - - @Nullable - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - try { - String viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); - if (viewCount.equals("Recommended for you")) return -1; - return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); - } catch (Exception e) { - throw new ParsingException("Could not get view count", e); - } - } - - @Override - public String getUploaderName() throws ParsingException { - try { - return videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getString("text"); - } catch (Exception e) { - throw new ParsingException("Could not get uploader name", e); - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return videoInfo.getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url"); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); + if (videoInfo != null) collector.commit(extractVideoPreviewInfo(videoInfo, timeAgoParser)); } return collector; } catch (Exception e) { @@ -1150,52 +1052,103 @@ public class YoutubeStreamExtractor extends StreamExtractor { * Provides information about links to other videos on the video page, such as related videos. * This is encapsulated in a StreamInfoItem object, which is a subset of the fields in a full StreamInfo. */ - private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li, final TimeAgoParser timeAgoParser) { - return new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { + private StreamInfoItemExtractor extractVideoPreviewInfo(final JsonObject videoInfo, final TimeAgoParser timeAgoParser) { + return new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser) { + @Override + public StreamType getStreamType() { + return StreamType.VIDEO_STREAM; + } + + @Override + public boolean isAd() { + return false; + } @Override public String getUrl() throws ParsingException { - return li.select("a.content-link").first().attr("abs:href"); + try { + String videoId = videoInfo.getString("videoId"); + return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); + } catch (Exception e) { + throw new ParsingException("Could not get url", e); + } } @Override public String getName() throws ParsingException { - //todo: check NullPointerException causing - return li.select("span.title").first().text(); - //this page causes the NullPointerException, after finding it by searching for "tjvg": - //https://www.youtube.com/watch?v=Uqg0aEhLFAg + String name = null; + try { + name = videoInfo.getObject("title").getString("simpleText"); + } catch (Exception ignored) {} + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get title"); } @Override - public String getUploaderName() throws ParsingException { - return li.select("span[class*=\"attribution\"").first() - .select("span").first().text(); + public long getDuration() throws ParsingException { + try { + return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); + } catch (Exception e) { + throw new ParsingException("Could not get duration", e); + } } @Override public String getUploaderUrl() throws ParsingException { - return ""; // The uploader is not linked + try { + String id = videoInfo.getObject("longBylineText").getArray("runs") + .getObject(0).getObject("navigationEndpoint") + .getObject("browseEndpoint").getString("browseId"); + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("is empty"); + } + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); + } catch (Exception e) { + throw new ParsingException("Could not get uploader url"); + } + } + + @Nullable + @Override + public String getTextualUploadDate() { + return null; + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; } @Override - public String getTextualUploadDate() throws ParsingException { - return ""; + public long getViewCount() throws ParsingException { + try { + String viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); + if (viewCount.equals("Recommended for you")) return -1; + return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); + } catch (Exception e) { + throw new ParsingException("Could not get view count", e); + } + } + + @Override + public String getUploaderName() throws ParsingException { + try { + return videoInfo.getObject("longBylineText").getArray("runs") + .getObject(0).getString("text"); + } catch (Exception e) { + throw new ParsingException("Could not get uploader name", e); + } } @Override public String getThumbnailUrl() throws ParsingException { - Element img = li.select("img").first(); - String thumbnailUrl = img.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we caught such an item. - if (thumbnailUrl.contains(".gif")) { - thumbnailUrl = img.attr("data-thumb"); + try { + return videoInfo.getObject("thumbnail").getArray("thumbnails") + .getObject(0).getString("url"); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); } - if (thumbnailUrl.startsWith("//")) { - thumbnailUrl = HTTPS + thumbnailUrl; - } - return thumbnailUrl; } }; } From 10e9c16d8cca05e6fa76e18667de064c38c06edc Mon Sep 17 00:00:00 2001 From: TobiGr Date: Thu, 20 Feb 2020 13:26:38 +0100 Subject: [PATCH 039/149] Fix LiveStream view count --- .../extractors/YoutubeStreamExtractor.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index d8059ed4..802a28f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -323,7 +323,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { if (getStreamType().equals(StreamType.LIVE_STREAM)) { - return getLiveStreamWatchingCount(); + // The array index is variable, therefore we loop throw the complete array. + // videoPrimaryInfoRenderer is often stored at index 1 + JsonArray contents = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("results").getObject("results").getArray("contents"); + for (Object c : contents) { + try { + // this gets current view count, but there is also an overall view count which is stored here: + // contents.twoColumnWatchNextResults.secondaryResults.secondaryResults.results[0] + // .compactAutoplayRenderer.contents[0].compactVideoRenderer.viewCountText.simpleText + String views = ((JsonObject) c).getObject("videoPrimaryInfoRenderer") + .getObject("viewCount").getObject("videoViewCountRenderer").getObject("viewCount") + .getArray("runs").getObject(0).getString("text"); + return Long.parseLong(Utils.removeNonDigitCharacters(views)); + } catch (Exception ignored) {} + } + throw new ParsingException("Could not get view count from live stream"); + } else { return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount")); } From f33f9466ce6a09437523e0803ff56ee59b6d1679 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Thu, 20 Feb 2020 13:27:38 +0100 Subject: [PATCH 040/149] Fix testDescription() in ChannelExtractor --- .../extractor/services/youtube/YoutubeChannelExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 317bd4fa..fc4ffff3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -170,7 +170,7 @@ public class YoutubeChannelExtractorTest { @Test public void testDescription() throws Exception { assertTrue("What it actually was: " + extractor.getDescription(), - extractor.getDescription().contains("Our World is Amazing. Questions? Ideas? Tweet me:")); + extractor.getDescription().contains("Our World is Amazing. \n\nQuestions? Ideas? Tweet me:")); } @Test From af49b3c48777dfb002401cc7680766312ae96a6b Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 21 Feb 2020 20:08:34 +0100 Subject: [PATCH 041/149] Fix live streams in extractVideoPreviewInfo() --- .../youtube/extractors/YoutubeStreamExtractor.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 802a28f3..53f1585d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1072,6 +1072,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { return new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser) { @Override public StreamType getStreamType() { + try { + if (videoInfo.getArray("badges").getObject(0).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { + return StreamType.LIVE_STREAM; + } + } catch (Exception ignored) {} return StreamType.VIDEO_STREAM; } @@ -1103,6 +1108,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getDuration() throws ParsingException { try { + if (getStreamType() == StreamType.LIVE_STREAM) return -1; return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); } catch (Exception e) { throw new ParsingException("Could not get duration", e); @@ -1139,7 +1145,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getViewCount() throws ParsingException { try { - String viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); + String viewCount; + if (getStreamType() == StreamType.LIVE_STREAM) { + viewCount = videoInfo.getObject("viewCountText") + .getArray("runs").getObject(0).getString("text"); + } else { + viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); + } if (viewCount.equals("Recommended for you")) return -1; return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); } catch (Exception e) { From b88188d4198ec81d0d4d78c2b55a07a53a073769 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 22 Feb 2020 20:19:41 +0100 Subject: [PATCH 042/149] Move stuff from extractVideoPreviewInfo() into YoutubeStreamInfoItemExtractor and partially fix search --- .../extractors/YoutubePlaylistExtractor.java | 6 +- .../extractors/YoutubeSearchExtractor.java | 59 ++-- .../extractors/YoutubeStreamExtractor.java | 149 ++-------- .../YoutubeStreamInfoItemExtractor.java | 258 ++++-------------- 4 files changed, 118 insertions(+), 354 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 692b9cc3..1498ef68 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -23,9 +24,10 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import java.io.IOException; + import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { @@ -318,7 +320,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } @Override - public String getTextualUploadDate() throws ParsingException { + public String getTextualUploadDate() { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 48420814..c8ed9efd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -1,5 +1,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -16,12 +21,13 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Parser; -import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; +import javax.annotation.Nonnull; + /* * Created by Christian Schabesberger on 22.07.2018 * @@ -45,6 +51,7 @@ import java.net.URL; public class YoutubeSearchExtractor extends SearchExtractor { private Document doc; + private JsonObject ytInitialData; public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); @@ -55,6 +62,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final String url = getUrl(); final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + ytInitialData = getInitialData(); } @Nonnull @@ -86,6 +94,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + // TODO: Get extracting next pages working final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); doc = Jsoup.parse(response, pageUrl); @@ -108,37 +117,33 @@ public class YoutubeSearchExtractor extends SearchExtractor { InfoItemsSearchCollector collector = getInfoItemSearchCollector(); collector.reset(); - Element list = doc.select("ol[class=\"item-section\"]").first(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Element item : list.children()) { - /* First we need to determine which kind of item we are working with. - Youtube depicts five different kinds of items on its search result page. These are - regular videos, playlists, channels, two types of video suggestions, and a "no video - found" item. Since we only want videos, we need to filter out all the others. - An example for this can be seen here: - https://www.youtube.com/results?search_query=asdf&page=1 + JsonArray list = ytInitialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("contents"); - We already applied a filter to the url, so we don't need to care about channels and - playlists now. - */ - - Element el; - - if ((el = item.select("div[class*=\"search-message\"]").first()) != null) { - throw new NothingFoundException(el.text()); - - // video item type - } else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(el, timeAgoParser)); - } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) { - collector.commit(new YoutubeChannelInfoItemExtractor(el)); - } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null && - item.select(".yt-pl-icon-mix").isEmpty()) { - collector.commit(new YoutubePlaylistInfoItemExtractor(el)); + for (Object item : list) { + if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { + throw new NothingFoundException(((JsonObject) item).getObject("backgroundPromoRenderer") + .getObject("bodyText").getArray("runs").getObject(0).getString("text")); + } else if (((JsonObject) item).getObject("videoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); + } else if (((JsonObject) item).getObject("channelRenderer") != null) { +// collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer"))); + } else if (((JsonObject) item).getObject("playlistRenderer") != null) { +// collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); } } - return collector; } + + private JsonObject getInitialData() throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 53f1585d..a6c94732 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -4,10 +4,10 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -25,24 +25,38 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + /* * Created by Christian Schabesberger on 06.08.15. * @@ -663,7 +677,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collector.commit(extractVideoPreviewInfo(videoInfo, timeAgoParser)); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); return collector.getItems().get(0); } catch (Exception e) { throw new ParsingException("Could not get next video", e); @@ -684,7 +698,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { for (Object ul : results) { final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer"); - if (videoInfo != null) collector.commit(extractVideoPreviewInfo(videoInfo, timeAgoParser)); + if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); } return collector; } catch (Exception e) { @@ -1064,123 +1078,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { return urlAndItags; } - /** - * Provides information about links to other videos on the video page, such as related videos. - * This is encapsulated in a StreamInfoItem object, which is a subset of the fields in a full StreamInfo. - */ - private StreamInfoItemExtractor extractVideoPreviewInfo(final JsonObject videoInfo, final TimeAgoParser timeAgoParser) { - return new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser) { - @Override - public StreamType getStreamType() { - try { - if (videoInfo.getArray("badges").getObject(0).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { - return StreamType.LIVE_STREAM; - } - } catch (Exception ignored) {} - return StreamType.VIDEO_STREAM; - } - - @Override - public boolean isAd() { - return false; - } - - @Override - public String getUrl() throws ParsingException { - try { - String videoId = videoInfo.getString("videoId"); - return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); - } catch (Exception e) { - throw new ParsingException("Could not get url", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = null; - try { - name = videoInfo.getObject("title").getString("simpleText"); - } catch (Exception ignored) {} - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get title"); - } - - @Override - public long getDuration() throws ParsingException { - try { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); - } catch (Exception e) { - throw new ParsingException("Could not get duration", e); - } - } - - @Override - public String getUploaderUrl() throws ParsingException { - try { - String id = videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); - if (id == null || id.isEmpty()) { - throw new IllegalArgumentException("is empty"); - } - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); - } catch (Exception e) { - throw new ParsingException("Could not get uploader url"); - } - } - - @Nullable - @Override - public String getTextualUploadDate() { - return null; - } - - @Nullable - @Override - public DateWrapper getUploadDate() { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - try { - String viewCount; - if (getStreamType() == StreamType.LIVE_STREAM) { - viewCount = videoInfo.getObject("viewCountText") - .getArray("runs").getObject(0).getString("text"); - } else { - viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); - } - if (viewCount.equals("Recommended for you")) return -1; - return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); - } catch (Exception e) { - throw new ParsingException("Could not get view count", e); - } - } - - @Override - public String getUploaderName() throws ParsingException { - try { - return videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getString("text"); - } catch (Exception e) { - throw new ParsingException("Could not get uploader name", e); - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return videoInfo.getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url"); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }; - } - @Nonnull @Override public List getFrames() throws ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 1c127180..8970fd5d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -2,19 +2,17 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; /* * Copyright (C) Christian Schabesberger 2016 @@ -36,20 +34,10 @@ import java.util.Date; public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - private JsonObject videoInfoItem; - private Element item; + private JsonObject videoInfo; private final TimeAgoParser timeAgoParser; - private String cachedUploadDate; - - /** - * Creates an extractor of StreamInfoItems from a YouTube page. - * - * @param item The page element - * @param timeAgoParser A parser of the textual dates or {@code null}. - */ - public YoutubeStreamInfoItemExtractor(Element item, @Nullable TimeAgoParser timeAgoParser) { - this.item = item; + public YoutubeStreamInfoItemExtractor(Element a, @Nullable TimeAgoParser timeAgoParser) { this.timeAgoParser = timeAgoParser; } @@ -60,251 +48,123 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { * @param timeAgoParser A parser of the textual dates or {@code null}. */ public YoutubeStreamInfoItemExtractor(JsonObject videoInfoItem, @Nullable TimeAgoParser timeAgoParser) { - this.videoInfoItem = videoInfoItem; + this.videoInfo = videoInfoItem; this.timeAgoParser = timeAgoParser; } @Override - public StreamType getStreamType() throws ParsingException { - if (isLiveStream(item)) { - return StreamType.LIVE_STREAM; - } else { - return StreamType.VIDEO_STREAM; - } + public StreamType getStreamType() { + try { + if (videoInfo.getArray("badges").getObject(0).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { + return StreamType.LIVE_STREAM; + } + } catch (Exception ignored) {} + return StreamType.VIDEO_STREAM; } @Override - public boolean isAd() throws ParsingException { - return !item.select("span[class*=\"icon-not-available\"]").isEmpty() - || !item.select("span[class*=\"yt-badge-ad\"]").isEmpty() - || isPremiumVideo(); - } - - private boolean isPremiumVideo() { - Element premiumSpan = item.select("span[class=\"standalone-collection-badge-renderer-red-text\"]").first(); - if (premiumSpan == null) return false; - - // if this span has text it most likely says ("Free Video") so we can play this - if (premiumSpan.hasText()) return false; - return true; + public boolean isAd() { + return false; } @Override public String getUrl() throws ParsingException { try { - Element el = item.select("div[class*=\"yt-lockup-video\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); + String videoId = videoInfo.getString("videoId"); + return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); + throw new ParsingException("Could not get url", e); } } @Override public String getName() throws ParsingException { + String name = null; try { - Element el = item.select("div[class*=\"yt-lockup-video\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); + name = videoInfo.getObject("title").getString("simpleText"); + } catch (Exception ignored) {} + if (name == null) { + try { + name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text"); + } catch (Exception ignored) {} } + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get name"); } @Override public long getDuration() throws ParsingException { try { if (getStreamType() == StreamType.LIVE_STREAM) return -1; - - final Element duration = item.select("span[class*=\"video-time\"]").first(); - // apparently on youtube, video-time element will not show up if the video has a duration of 00:00 - // see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf - return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text()); + return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); } catch (Exception e) { - throw new ParsingException("Could not get Duration: " + getUrl(), e); + throw new ParsingException("Could not get duration", e); } } @Override public String getUploaderName() throws ParsingException { try { - return item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .text(); + return videoInfo.getObject("longBylineText").getArray("runs") + .getObject(0).getString("text"); } catch (Exception e) { - throw new ParsingException("Could not get uploader", e); + throw new ParsingException("Could not get uploader name", e); } } @Override public String getUploaderUrl() throws ParsingException { - // this url is not always in the form "/channel/..." - // sometimes Youtube provides urls in the from "/user/..." try { - try { - return item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .attr("abs:href"); - } catch (Exception e){} - - // try this if the first didn't work - return item.select("span[class=\"title\"") - .text().split(" - ")[0]; - } catch (Exception e) { - System.out.println(item.html()); - throw new ParsingException("Could not get uploader url", e); - } - } - - @Nullable - @Override - public String getTextualUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { - return null; - } - - if (cachedUploadDate != null) { - return cachedUploadDate; - } - - try { - if (isVideoReminder()) { - final Calendar calendar = getDateFromReminder(); - if (calendar != null) { - return cachedUploadDate = new SimpleDateFormat("yyyy-MM-dd HH:mm") - .format(calendar.getTime()); - } + String id = videoInfo.getObject("longBylineText").getArray("runs") + .getObject(0).getObject("navigationEndpoint") + .getObject("browseEndpoint").getString("browseId"); + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("is empty"); } - - - Element meta = item.select("div[class=\"yt-lockup-meta\"]").first(); - if (meta == null) return ""; - - final Elements li = meta.select("li"); - if (li.isEmpty()) return ""; - - return cachedUploadDate = li.first().text(); + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); } catch (Exception e) { - throw new ParsingException("Could not get upload date", e); + throw new ParsingException("Could not get uploader url"); } } @Nullable @Override - public DateWrapper getUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { - return null; - } + public String getTextualUploadDate() { + // TODO: Get upload date in case of a videoRenderer (not available in case of a compactVideoRenderer) + return null; + } - if (isVideoReminder()) { - return new DateWrapper(getDateFromReminder()); - } - - String textualUploadDate = getTextualUploadDate(); - if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) { - return timeAgoParser.parse(textualUploadDate); - } else { - return null; - } + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; } @Override public long getViewCount() throws ParsingException { - String input; - - final Element spanViewCount = item.select("span.view-count").first(); - if (spanViewCount != null) { - input = spanViewCount.text(); - - } else if (getStreamType().equals(StreamType.LIVE_STREAM)) { - Element meta = item.select("ul.yt-lockup-meta-info").first(); - if (meta == null) return 0; - - final Elements li = meta.select("li"); - if (li.isEmpty()) return 0; - - input = li.first().text(); - } else { - try { - Element meta = item.select("div.yt-lockup-meta").first(); - if (meta == null) return -1; - - // This case can happen if google releases a special video - if (meta.select("li").size() < 2) return -1; - - input = meta.select("li").get(1).text(); - } catch (IndexOutOfBoundsException e) { - throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e); - } - } - - if (input == null) { - throw new ParsingException("Input is null"); - } - try { - - return Long.parseLong(Utils.removeNonDigitCharacters(input)); - } catch (NumberFormatException e) { - // if this happens the video probably has no views - if (!input.isEmpty()) { - return 0; + String viewCount; + if (getStreamType() == StreamType.LIVE_STREAM) { + viewCount = videoInfo.getObject("viewCountText") + .getArray("runs").getObject(0).getString("text"); + } else { + viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); } - - throw new ParsingException("Could not handle input: " + input, e); + if (viewCount.equals("Recommended for you")) return -1; + return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); + } catch (Exception e) { + throw new ParsingException("Could not get view count", e); } } @Override public String getThumbnailUrl() throws ParsingException { try { - String url; - Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + return videoInfo.getObject("thumbnail").getArray("thumbnails") + .getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } } - - - private boolean isVideoReminder() { - return !item.select("span.yt-uix-livereminder").isEmpty(); - } - - private Calendar getDateFromReminder() throws ParsingException { - final Element timeFuture = item.select("span.yt-badge.localized-date").first(); - - if (timeFuture == null) { - throw new ParsingException("Span timeFuture is null"); - } - - final String timestamp = timeFuture.attr("data-timestamp"); - if (!timestamp.isEmpty()) { - try { - final Calendar calendar = Calendar.getInstance(); - calendar.setTime(new Date(Long.parseLong(timestamp) * 1000L)); - return calendar; - } catch (Exception e) { - throw new ParsingException("Could not parse = \"" + timestamp + "\""); - } - } - - throw new ParsingException("Could not parse date from reminder element: \"" + timeFuture + "\""); - } - - /** - * Generic method that checks if the element contains any clues that it's a livestream item - */ - protected static boolean isLiveStream(Element item) { - return !item.select("span[class*=\"yt-badge-live\"]").isEmpty() - || !item.select("span[class*=\"video-time-overlay-live\"]").isEmpty(); - } } From 957db062efeade07fb5c121308236a06c52fdad6 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 22 Feb 2020 20:33:05 +0100 Subject: [PATCH 043/149] Fix getting uploader for videoRenderer --- .../YoutubeStreamInfoItemExtractor.java | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 8970fd5d..91124fb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -104,20 +104,37 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderName() throws ParsingException { + String name = null; try { - return videoInfo.getObject("longBylineText").getArray("runs") + name = videoInfo.getObject("longBylineText").getArray("runs") .getObject(0).getString("text"); - } catch (Exception e) { - throw new ParsingException("Could not get uploader name", e); + } catch (Exception ignored) {} + if (name == null) { + try { + name = videoInfo.getObject("ownerText").getArray("runs") + .getObject(0).getString("text"); + } catch (Exception ignored) {} } + if (name != null && !name.isEmpty()) return name; + throw new ParsingException("Could not get uploader name"); } @Override public String getUploaderUrl() throws ParsingException { try { - String id = videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); + String id = null; + try { + id = videoInfo.getObject("longBylineText").getArray("runs") + .getObject(0).getObject("navigationEndpoint") + .getObject("browseEndpoint").getString("browseId"); + } catch (Exception ignored) {} + if (id == null) { + try { + id = videoInfo.getObject("ownerText").getArray("runs") + .getObject(0).getObject("navigationEndpoint") + .getObject("browseEndpoint").getString("browseId"); + } catch (Exception ignored) {} + } if (id == null || id.isEmpty()) { throw new IllegalArgumentException("is empty"); } From 38aabc6acac601b1d0343517ae0357bfcf76f9d6 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 22 Feb 2020 20:33:48 +0100 Subject: [PATCH 044/149] Use plain YoutubeStreamInfoItemExtractor in YoutubeTrendingExtractor --- .../extractors/YoutubeTrendingExtractor.java | 70 ++----------------- 1 file changed, 4 insertions(+), 66 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 05a33a35..467a1fed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -24,9 +24,8 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -35,16 +34,15 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Parser; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; @@ -115,67 +113,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { for (Object ul : firstPageElements) { final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); - collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - try { - String videoId = videoInfo.getString("videoId"); - return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - String name = null; - try { - name = videoInfo.getObject("title").getObject("accessibility") - .getObject("accessibilityData").getString("label"); - } catch (Exception ignored) { - } - if (name != null && !name.isEmpty()) return name; - try { - name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text"); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get web page url for the video"); - } - - @Override - public String getUploaderUrl() throws ParsingException { - try { - String id = videoInfo.getObject("ownerText").getArray("runs").getObject(0). - getObject("browseEndpoint").getString("browseId"); - if (id == null || id.isEmpty()) { - throw new IllegalArgumentException("is empty"); - } - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); - } catch (Exception e) { - throw new ParsingException("Could not get Uploader url"); - } - } - - @Override - public String getUploaderName() throws ParsingException { - try { - return videoInfo.getObject("ownerText").getArray("runs").getObject(0).getString("text"); - } catch (Exception e) { - throw new ParsingException("Could not get uploader name"); - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return videoInfo.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); } return new InfoItemsPage<>(collector, getNextPageUrl()); From 5816202cc7a6d055a996f9783f701cd60cb1cbe2 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 22 Feb 2020 23:51:02 +0100 Subject: [PATCH 045/149] Move getInitialData() method to YouTubeParsingHelper Rename ytInitialData to initialData --- .../extractors/YoutubeChannelExtractor.java | 22 +++++----------- .../extractors/YoutubePlaylistExtractor.java | 11 +------- .../extractors/YoutubeSearchExtractor.java | 14 +++------- .../extractors/YoutubeStreamExtractor.java | 26 +++++++------------ .../extractors/YoutubeTrendingExtractor.java | 11 +------- .../linkHandler/YoutubeParsingHelper.java | 14 ++++++++++ 6 files changed, 35 insertions(+), 63 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 544a9e46..8e47e779 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -52,7 +52,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; private Document doc; - private JsonObject ytInitialData; + private JsonObject initialData; public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -63,17 +63,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; final Response response = downloader.get(channelUrl, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); - ytInitialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } @Override public String getNextPageUrl() throws ExtractionException { @@ -97,7 +89,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return doc.select("meta[property=\"og:url\"]").first().attr("content").replace(CHANNEL_URL_BASE, ""); } catch (Exception ignored) {} try { - return ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); } catch (Exception ignored) {} // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) @@ -124,7 +116,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { try { - return ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails").getObject(0).getString("url"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -133,7 +125,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - String url = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); + String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); if (url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; } @@ -165,7 +157,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final JsonObject subscriberInfo = ytInitialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); + final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); if (subscriberInfo != null) { try { @@ -182,7 +174,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getDescription() throws ParsingException { try { - return ytInitialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); + return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); } catch (Exception e) { throw new ParsingException("Could not get channel description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 1498ef68..3a4cbffd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -47,21 +47,12 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final String url = getUrl(); final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); uploaderInfo = getUploaderInfo(); playlistInfo = getPlaylistInfo(); playlistVideos = getPlaylistVideos(); } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } - private JsonObject getUploaderInfo() throws ParsingException { JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items"); try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index c8ed9efd..57ce3aab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -51,7 +51,7 @@ import javax.annotation.Nonnull; public class YoutubeSearchExtractor extends SearchExtractor { private Document doc; - private JsonObject ytInitialData; + private JsonObject initialData; public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { super(service, linkHandler); @@ -62,7 +62,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final String url = getUrl(); final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - ytInitialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); } @Nonnull @@ -119,7 +119,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - JsonArray list = ytInitialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + JsonArray list = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") .getObject(0).getObject("itemSectionRenderer").getArray("contents"); @@ -138,12 +138,4 @@ public class YoutubeSearchExtractor extends SearchExtractor { return collector; } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a6c94732..a0bf1c8f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -104,7 +104,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private final Map videoInfoPage = new HashMap<>(); private JsonObject playerResponse; - private JsonObject ytInitialData; + private JsonObject initialData; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -339,7 +339,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (getStreamType().equals(StreamType.LIVE_STREAM)) { // The array index is variable, therefore we loop throw the complete array. // videoPrimaryInfoRenderer is often stored at index 1 - JsonArray contents = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("results").getObject("results").getArray("contents"); for (Object c : contents) { try { @@ -421,7 +421,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String likesString = ""; try { try { - likesString = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; + likesString = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -444,7 +444,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { Element button = doc.select("button.like-button-renderer-dislike-button").first(); try { - dislikesString = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; + dislikesString = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -507,7 +507,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String uploaderAvatarUrl = null; try { - uploaderAvatarUrl = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") + uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") .getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail") .getArray("thumbnails").getObject(0).getString("url"); @@ -517,7 +517,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} try { - uploaderAvatarUrl = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") + uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") .getObject("results").getArray("contents").getObject(1).getObject("videoSecondaryInfoRenderer") .getObject("owner").getObject("videoOwnerRenderer").getObject("thumbnail").getArray("thumbnails") .getObject(0).getString("url"); @@ -670,7 +670,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public StreamInfoItem getNextStream() throws IOException, ExtractionException { assertPageFetched(); try { - final JsonObject videoInfo = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults").getObject("secondaryResults").getArray("results") .getObject(0).getObject("compactAutoplayRenderer").getArray("contents") .getObject(0).getObject("compactVideoRenderer"); @@ -690,7 +690,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray results = ytInitialData.getObject("contents").getObject("twoColumnWatchNextResults") + JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults").getObject("secondaryResults").getArray("results"); final TimeAgoParser timeAgoParser = getTimeAgoParser(); @@ -778,7 +778,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { isAgeRestricted = false; } playerResponse = getPlayerResponse(); - ytInitialData = getInitialData(); + initialData = YoutubeParsingHelper.getInitialData(pageHtml); if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); @@ -852,14 +852,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } - } @Nonnull private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 467a1fed..d815e9e1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -61,16 +61,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final Response response = downloader.get(url, getExtractorLocalization()); doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = getInitialData(); - } - - private JsonObject getInitialData() throws ParsingException { - try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", doc.toString()); - return JsonParser.object().from(initialData); - } catch (JsonParserException | Parser.RegexException e) { - throw new ParsingException("Could not get ytInitialData", e); - } + initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 65ec7e3f..78516f6d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -1,11 +1,15 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.Parser; import java.net.URL; import java.text.ParseException; @@ -143,4 +147,14 @@ public class YoutubeParsingHelper { uploadDate.setTime(date); return uploadDate; } + + public static JsonObject getInitialData(String html) throws ParsingException { + try { + String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); + return JsonParser.object().from(initialData); + } catch (JsonParserException | Parser.RegexException e) { + throw new ParsingException("Could not get ytInitialData", e); + } + } + } From 4e57e589cee1842104fcb9af2b8e1d668c304187 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 23 Feb 2020 11:23:33 +0100 Subject: [PATCH 046/149] Fix first channel page --- .../extractors/YoutubeChannelExtractor.java | 62 +++++-------------- 1 file changed, 15 insertions(+), 47 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 8e47e779..1cb448e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,8 +1,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -17,12 +19,12 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; +import javax.annotation.Nonnull; + import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @@ -191,6 +193,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + // TODO: Get extracting next pages working if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } @@ -245,59 +248,24 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String uploaderUrl = getUrl(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Element li : element.children()) { - if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - @Override - public String getUrl() throws ParsingException { - try { - Element el = li.select("div[class=\"feed-item-dismissable\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } + JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(1).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents"); + for (Object video : videos) { + JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer") + .getArray("contents").getObject(0); + if (videoInfo.getObject("videoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo.getObject("videoRenderer"), timeAgoParser) { @Override - public String getName() throws ParsingException { - try { - Element el = li.select("div[class=\"feed-item-dismissable\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } - } - - @Override - public String getUploaderName() throws ParsingException { + public String getUploaderName() { return uploaderName; } @Override - public String getUploaderUrl() throws ParsingException { + public String getUploaderUrl() { return uploaderUrl; } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - String url; - Element te = li.select("span[class=\"yt-thumb-clip\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } }); } } From 316fe0109df23dd3e6db25274d85508203308090 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 23 Feb 2020 13:48:54 +0100 Subject: [PATCH 047/149] Fix first playlist page --- .../extractors/YoutubePlaylistExtractor.java | 120 ++---------------- .../YoutubeStreamInfoItemExtractor.java | 19 ++- 2 files changed, 28 insertions(+), 111 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 3a4cbffd..f0fb91a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -13,15 +13,12 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -233,112 +230,23 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private void collectStreamsFrom(@Nonnull StreamInfoItemsCollector collector, @Nullable Element element) { collector.reset(); - if (element == null) { - return; - } - - final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Element li : element.children()) { - if (isDeletedItem(li)) { - continue; + JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("playlistVideoListRenderer").getArray("contents"); + + for (Object video : videos) { + if (((JsonObject) video).getObject("playlistVideoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { + @Override + public long getViewCount() { + return -1; + } + }); } - - collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { - public Element uploaderLink; - - @Override - public boolean isAd() { - return false; - } - - @Override - public String getUrl() throws ParsingException { - try { - return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getName() throws ParsingException { - try { - return li.attr("data-title"); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } - } - - @Override - public long getDuration() throws ParsingException { - try { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - - Element first = li.select("div[class=\"timestamp\"] span").first(); - if (first == null) { - // Video unavailable (private, deleted, etc.), this is a thing that happens specifically with playlists, - // because in other cases, those videos don't even show up - return -1; - } - - return YoutubeParsingHelper.parseDurationString(first.text()); - } catch (Exception e) { - throw new ParsingException("Could not get duration" + getUrl(), e); - } - } - - - private Element getUploaderLink() { - // should always be present since we filter deleted items - if (uploaderLink == null) { - uploaderLink = li.select("div[class=pl-video-owner] a").first(); - } - return uploaderLink; - } - - @Override - public String getUploaderName() throws ParsingException { - return getUploaderLink().text(); - } - - @Override - public String getUploaderUrl() throws ParsingException { - // this url is not always in the form "/channel/..." - // sometimes Youtube provides urls in the from "/user/..." - return getUploaderLink().attr("abs:href"); - } - - @Override - public String getTextualUploadDate() { - return ""; - } - - @Override - public long getViewCount() throws ParsingException { - return -1; - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - return "https://i.ytimg.com/vi/" + streamLinkHandlerFactory.fromUrl(getUrl()).getId() + "/hqdefault.jpg"; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - }); } } - - /** - * Check if the playlist item is deleted - * - * @param li the list item - * @return true if the item is deleted - */ - private boolean isDeletedItem(Element li) { - return li.select("div[class=pl-video-owner] a").isEmpty(); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 91124fb4..a53194f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; -import org.jsoup.nodes.Element; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -37,10 +37,6 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private JsonObject videoInfo; private final TimeAgoParser timeAgoParser; - public YoutubeStreamInfoItemExtractor(Element a, @Nullable TimeAgoParser timeAgoParser) { - this.timeAgoParser = timeAgoParser; - } - /** * Creates an extractor of StreamInfoItems from a YouTube page. * @@ -115,6 +111,12 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { .getObject(0).getString("text"); } catch (Exception ignored) {} } + if (name == null) { + try { + name = videoInfo.getObject("shortBylineText").getArray("runs") + .getObject(0).getString("text"); + } catch (Exception ignored) {} + } if (name != null && !name.isEmpty()) return name; throw new ParsingException("Could not get uploader name"); } @@ -135,6 +137,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { .getObject("browseEndpoint").getString("browseId"); } catch (Exception ignored) {} } + if (id == null) { + try { + id = videoInfo.getObject("shortBylineText").getArray("runs") + .getObject(0).getObject("navigationEndpoint") + .getObject("browseEndpoint").getString("browseId"); + } catch (Exception ignored) {} + } if (id == null || id.isEmpty()) { throw new IllegalArgumentException("is empty"); } From 7442f91aa619a57b73ece4ca43160420585e1464 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 23 Feb 2020 14:18:21 +0100 Subject: [PATCH 048/149] Don't assume which channel tab is videos --- .../extractors/YoutubeChannelExtractor.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 1cb448e4..ed987f24 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -248,9 +248,22 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String uploaderUrl = getUrl(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs").getObject(1).getObject("tabRenderer").getObject("content") - .getObject("sectionListRenderer").getArray("contents"); + JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs"); + JsonArray videos = null; + + for (Object tab : tabs) { + if (((JsonObject) tab).getObject("tabRenderer") != null) { + if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) { + videos = ((JsonObject) tab).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents"); + } + } + } + + if (videos == null) { + throw new ParsingException("Could not find Videos tab"); + } for (Object video : videos) { JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer") From 3187116a63cda7ddc2e7f68d3571fc7f353f476e Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 23 Feb 2020 14:19:13 +0100 Subject: [PATCH 049/149] Handle premium videos --- .../YoutubeStreamInfoItemExtractor.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index a53194f1..bf06774c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -60,7 +61,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public boolean isAd() { - return false; + return isPremium(); } @Override @@ -169,6 +170,9 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getViewCount() throws ParsingException { try { + if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { + return -1; + } String viewCount; if (getStreamType() == StreamType.LIVE_STREAM) { viewCount = videoInfo.getObject("viewCountText") @@ -193,4 +197,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { throw new ParsingException("Could not get thumbnail url", e); } } + + private boolean isPremium() { + try { + JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("Premium")) { + return true; + } + } + } catch (Exception ignored) {} + return false; + } } From 8aea4d445b62b92ffeeb7fc22b3d0af5e244e809 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 23 Feb 2020 18:27:28 +0100 Subject: [PATCH 050/149] Reimplement YoutubeChannelInfoItemExtractor --- .../YoutubeChannelInfoItemExtractor.java | 86 +++++++------------ .../extractors/YoutubeSearchExtractor.java | 2 +- 2 files changed, 31 insertions(+), 57 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index a687c050..d0df1581 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,13 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import org.jsoup.nodes.Element; +import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - /* * Created by Christian Schabesberger on 12.02.17. * @@ -29,87 +28,62 @@ import java.util.regex.Pattern; */ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private final Element el; + private JsonObject channelInfoItem; - public YoutubeChannelInfoItemExtractor(Element el) { - this.el = el; + public YoutubeChannelInfoItemExtractor(JsonObject channelInfoItem) { + this.channelInfoItem = channelInfoItem; } @Override public String getThumbnailUrl() throws ParsingException { - Element img = el.select("span[class*=\"yt-thumb-simple\"]").first() - .select("img").first(); - - String url = img.attr("abs:src"); - - if (url.contains("gif")) { - url = img.attr("abs:data-thumb"); + try { + return channelInfoItem.getObject("thumbnails").getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); } - return url; } @Override public String getName() throws ParsingException { - return el.select("a[class*=\"yt-uix-tile-link\"]").first() - .text(); + try { + return channelInfoItem.getObject("title").getString("simpleText"); + } catch (Exception e) { + throw new ParsingException("Could not get name", e); + } } @Override public String getUrl() throws ParsingException { try { - String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first() - .attr("abs:data-href"); - - Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)"); - Matcher match = channelIdPattern.matcher(buttonTrackingUrl); - - if (match.matches()) { - return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); - } - } catch(Exception ignored) {} - - // fallback method for channels without "Subscribe" button (or just in case yt changes things) - // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail - try { - return el.select("a[class*=\"yt-uix-tile-link\"]").first() - .attr("abs:href"); + String id = "channel/" + channelInfoItem.getString("channelId"); // Does prepending 'channel/' always work? + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); } catch (Exception e) { - throw new ParsingException("Could not get channel url", e); + throw new ParsingException("Could not get url", e); } } @Override - public long getSubscriberCount() throws ParsingException { - final Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first(); - if (subsEl != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text())); - } catch (NumberFormatException e) { - throw new ParsingException("Could not get subscriber count", e); - } - } else { - // If the element is null, the channel have the subscriber count disabled - return -1; - } + public long getSubscriberCount() { + // TODO: get subscriber count, it's in subscriberCountText.simpleText as a string like "103M subscribers" + return -1; } @Override public long getStreamCount() throws ParsingException { - Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first(); - if (metaEl == null) { - return 0; - } else { - return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text())); + try { + return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText") + .getArray("runs").getObject(0).getString("text"))); + } catch (Exception e) { + throw new ParsingException("Could not get name", e); } } @Override public String getDescription() throws ParsingException { - Element desEl = el.select("div[class*=\"yt-lockup-description\"]").first(); - if (desEl == null) { - return ""; - } else { - return desEl.text(); + try { + return channelInfoItem.getObject("descriptionSnippet").getArray("runs").getObject(0).getString("text"); + } catch (Exception e) { + throw new ParsingException("Could not get description url", e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 57ce3aab..fdf09e4d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -130,7 +130,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } else if (((JsonObject) item).getObject("videoRenderer") != null) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); } else if (((JsonObject) item).getObject("channelRenderer") != null) { -// collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer"))); + collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer"))); } else if (((JsonObject) item).getObject("playlistRenderer") != null) { // collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); } From 4462cbe3f154e8fbf08c9d5f0bb1be9febe8ee71 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 23 Feb 2020 19:45:45 +0100 Subject: [PATCH 051/149] Reimplement YoutubePlaylistInfoItemExtractor --- .../YoutubeChannelInfoItemExtractor.java | 6 +- .../YoutubePlaylistInfoItemExtractor.java | 70 +++++-------------- .../extractors/YoutubeSearchExtractor.java | 2 +- 3 files changed, 22 insertions(+), 56 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index d0df1581..d987af0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -37,7 +37,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getThumbnailUrl() throws ParsingException { try { - return channelInfoItem.getObject("thumbnails").getArray("thumbnails").getObject(0).getString("url"); + return channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -74,7 +74,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText") .getArray("runs").getObject(0).getString("text"))); } catch (Exception e) { - throw new ParsingException("Could not get name", e); + throw new ParsingException("Could not get stream count", e); } } @@ -83,7 +83,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor try { return channelInfoItem.getObject("descriptionSnippet").getArray("runs").getObject(0).getString("text"); } catch (Exception e) { - throw new ParsingException("Could not get description url", e); + throw new ParsingException("Could not get description", e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 63fef225..358fa2e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -1,97 +1,63 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import org.jsoup.nodes.Element; +import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { - private final Element el; + private JsonObject playlistInfoItem; - public YoutubePlaylistInfoItemExtractor(Element el) { - this.el = el; + public YoutubePlaylistInfoItemExtractor(JsonObject playlistInfoItem) { + this.playlistInfoItem = playlistInfoItem; } @Override public String getThumbnailUrl() throws ParsingException { - String url; - try { - Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } + return playlistInfoItem.getArray("thumbnails").getObject(0).getArray("thumbnails") + .getObject(0).getString("url"); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist thumbnail url", e); + throw new ParsingException("Could not get thumbnail url", e); } - - return url; } @Override public String getName() throws ParsingException { - String name; try { - final Element title = el.select("[class=\"yt-lockup-title\"]").first() - .select("a").first(); - - name = title == null ? "" : title.text(); + return playlistInfoItem.getObject("title").getString("simpleText"); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist name", e); + throw new ParsingException("Could not get name", e); } - - return name; } @Override public String getUrl() throws ParsingException { try { - final Element a = el.select("div[class=\"yt-lockup-meta\"]") - .select("ul[class=\"yt-lockup-meta-info\"]") - .select("li").select("a").first(); - - if (a != null) { - return a.attr("abs:href"); - } - - // this is for yt premium playlists - return el.select("h3[class=\"yt-lockup-title\"").first() - .select("a").first() - .attr("abs:href"); - + String id = playlistInfoItem.getString("playlistId"); + return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(id); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist url", e); + throw new ParsingException("Could not get url", e); } } @Override public String getUploaderName() throws ParsingException { - String name; - try { - final Element div = el.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first(); - - name = div.text(); + return playlistInfoItem.getObject("longBylineText").getArray("runs").getObject(0).getString("text"); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist uploader", e); + throw new ParsingException("Could not get uploader name", e); } - - return name; } @Override public long getStreamCount() throws ParsingException { try { - final Element count = el.select("span[class=\"formatted-video-count-label\"]").first() - .select("b").first(); - - return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text())); + return Long.parseLong(Utils.removeNonDigitCharacters(playlistInfoItem.getString("videoCount"))); } catch (Exception e) { - throw new ParsingException("Failed to extract playlist stream count", e); + throw new ParsingException("Could not get stream count", e); } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index fdf09e4d..02dd2ada 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -132,7 +132,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } else if (((JsonObject) item).getObject("channelRenderer") != null) { collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer"))); } else if (((JsonObject) item).getObject("playlistRenderer") != null) { -// collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); + collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); } } return collector; From 6d20b2b1d0011f6f26adfbd02715cd3d8b7b80c6 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 23 Feb 2020 22:24:47 +0100 Subject: [PATCH 052/149] Fix NPE on collecting page items --- .../services/youtube/extractors/YoutubeSearchExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 02dd2ada..5d04a389 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -113,12 +113,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { "&page=" + Integer.toString(pageNr + 1)); } - private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException { + private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException, ParsingException { InfoItemsSearchCollector collector = getInfoItemSearchCollector(); collector.reset(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); + if (initialData == null) initialData = YoutubeParsingHelper.getInitialData(doc.toString()); JsonArray list = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") .getObject(0).getObject("itemSectionRenderer").getArray("contents"); From 51fb26625a9db58de7c7647ae4461cd3c09fe98e Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 10:39:52 +0100 Subject: [PATCH 053/149] Implement getSubscriberCount() in YoutubeChannelInfoItemExtractor --- .../extractors/YoutubeChannelInfoItemExtractor.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index d987af0d..7f1e0129 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -63,9 +63,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor } @Override - public long getSubscriberCount() { - // TODO: get subscriber count, it's in subscriberCountText.simpleText as a string like "103M subscribers" - return -1; + public long getSubscriberCount() throws ParsingException { + try { + String subscribers = channelInfoItem.getObject("subscriberCountText").getString("simpleText").split(" ")[0]; + return Utils.mixedNumberWordToLong(subscribers); + } catch (Exception e) { + throw new ParsingException("Could not get subscriber count", e); + } } @Override From c7360e4a468b5c657b2a95e640282d0773881156 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 12:48:16 +0100 Subject: [PATCH 054/149] Fix getThumbnailUrl() in YoutubeChannelInfoItemExtractor --- .../YoutubeChannelInfoItemExtractor.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 7f1e0129..483cd894 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -7,6 +7,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; + /* * Created by Christian Schabesberger on 12.02.17. * @@ -37,7 +40,16 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getThumbnailUrl() throws ParsingException { try { - return channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + if (url.startsWith("//")) { + url = url.substring(2); + } + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + return url; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } From 1eca63089b88e4c27c38e70fe17760cf89ab00db Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 12:55:51 +0100 Subject: [PATCH 055/149] Count private and deleted videos as ad --- .../youtube/extractors/YoutubeStreamInfoItemExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index bf06774c..605053bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -60,8 +60,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } @Override - public boolean isAd() { - return isPremium(); + public boolean isAd() throws ParsingException { + return isPremium() || getName().equals("[Private video]") || getName().equals("[Deleted video]"); } @Override From a38ab9b791ac118e3fbd7e33fc5d2dc9e3bb795c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 13:01:13 +0100 Subject: [PATCH 056/149] Return null if channel has no banner --- .../youtube/extractors/YoutubeChannelExtractor.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index ed987f24..839f9203 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -127,8 +127,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { try { - String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); - if (url.contains("s.ytimg.com") || url.contains("default_banner")) { + String url = null; + try { + url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception ignored) {} + if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; } // the first characters of the banner URLs are different for each channel and some are not even valid URLs From 2dfa2187ffb6c8a6a04143d2811aa9e368936fbb Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 15:48:23 +0100 Subject: [PATCH 057/149] Implement pagination in YoutubeChannelExtractor --- .../extractors/YoutubeChannelExtractor.java | 101 +++++++++--------- 1 file changed, 50 insertions(+), 51 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 839f9203..2e68a276 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; @@ -22,6 +21,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; @@ -71,7 +74,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(doc); + return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("continuations")); } @Nonnull @@ -189,8 +192,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); - collectStreamsFrom(collector, ul); + + JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents"); + collectStreamsFrom(collector, videos); + return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -203,71 +208,44 @@ public class YoutubeChannelExtractor extends ChannelExtractor { // Unfortunately, we have to fetch the page even if we are only getting next streams, // as they don't deliver enough information on their own (the channel name, for example). - fetchPage(); +// fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonObject ajaxJson; + JsonArray ajaxJson; try { - final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); - ajaxJson = JsonParser.object().from(response); + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + ajaxJson = JsonParser.array().from(response); } catch (JsonParserException pe) { throw new ParsingException("Could not parse json data for next streams", pe); } - final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl); - collectStreamsFrom(collector, ajaxHtml.select("body").first()); + JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("sectionListContinuation"); - return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); + collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); } - private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) - throws ParsingException { - String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); - if (!loadMoreHtmlDataRaw.isEmpty()) { - return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl)); - } else { - return ""; - } + + private String getNextPageUrlFrom(JsonArray continuations) { + JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + String continuation = nextContinuationData.getString("continuation"); + String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } - private String getNextPageUrlFrom(Document d) throws ParsingException { - try { - Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); - if (button != null) { - return button.attr("abs:data-uix-load-more-href"); - } else { - // Sometimes channels are simply so small, they don't have a more streams/videos - return ""; - } - } catch (Exception e) { - throw new ParsingException("Could not get next page url", e); - } - } - - private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) throws ParsingException { + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException { collector.reset(); final String uploaderName = getName(); final String uploaderUrl = getUrl(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs"); - JsonArray videos = null; - - for (Object tab : tabs) { - if (((JsonObject) tab).getObject("tabRenderer") != null) { - if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) { - videos = ((JsonObject) tab).getObject("tabRenderer").getObject("content") - .getObject("sectionListRenderer").getArray("contents"); - } - } - } - - if (videos == null) { - throw new ParsingException("Could not find Videos tab"); - } - for (Object video : videos) { JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer") .getArray("contents").getObject(0); @@ -286,4 +264,25 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } } } + + private JsonObject getVideoTab() throws ParsingException { + JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs"); + JsonObject videoTab = null; + + for (Object tab : tabs) { + if (((JsonObject) tab).getObject("tabRenderer") != null) { + if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) { + videoTab = ((JsonObject) tab).getObject("tabRenderer"); + break; + } + } + } + + if (videoTab == null) { + throw new ParsingException("Could not find Videos tab"); + } + + return videoTab; + } } From 627ab2459f90969c987e7463fcfcaf2fc93c8a10 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 15:51:08 +0100 Subject: [PATCH 058/149] Uncomment fetchPage() --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 2e68a276..b42514fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -208,7 +208,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { // Unfortunately, we have to fetch the page even if we are only getting next streams, // as they don't deliver enough information on their own (the channel name, for example). -// fetchPage(); + fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray ajaxJson; From 4039409820e9a3f82f9727c25d4bb1759c8576ec Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 16:04:01 +0100 Subject: [PATCH 059/149] Fix getLikeCount() for certain streams --- .../extractors/YoutubeChannelExtractor.java | 1 - .../extractors/YoutubeStreamExtractor.java | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index b42514fc..c60e1c8f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -201,7 +201,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - // TODO: Get extracting next pages working if (pageUrl == null || pageUrl.isEmpty()) { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a0bf1c8f..dd2987a4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -415,13 +415,33 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ExtractionException("Could not find correct results in response"); } + private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("results").getObject("results").getArray("contents"); + JsonObject videoPrimaryInfoRenderer = null; + + for (Object content : contents) { + if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) { + videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); + break; + } + } + + if (videoPrimaryInfoRenderer == null) { + throw new ParsingException("Could not find videoPrimaryInfoRenderer"); + } + + return videoPrimaryInfoRenderer; + } + @Override public long getLikeCount() throws ParsingException { assertPageFetched(); String likesString = ""; try { try { - likesString = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; + likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar") + .getObject("sentimentBarRenderer").getString("tooltip").split("/")[0]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { @@ -442,9 +462,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String dislikesString = ""; try { - Element button = doc.select("button.like-button-renderer-dislike-button").first(); try { - dislikesString = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results").getObject("results").getArray("contents").getObject(0).getObject("videoPrimaryInfoRenderer").getObject("sentimentBar").getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; + dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar") + .getObject("sentimentBarRenderer").getString("tooltip").split("/")[1]; } catch (NullPointerException e) { //if this kicks in our button has no content and therefore ratings must be disabled if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { From c0a8e0188967db4e8ae84b5f0acbe2fd61b1119c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 16:45:36 +0100 Subject: [PATCH 060/149] Implement pagination in YoutubePlaylistExtractor --- .../extractors/YoutubeChannelExtractor.java | 4 + .../extractors/YoutubePlaylistExtractor.java | 79 +++++++++---------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index c60e1c8f..27678657 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -231,6 +231,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private String getNextPageUrlFrom(JsonArray continuations) { + if (continuations == null) { + return ""; + } + JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index f0fb91a1..8abee5f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -5,9 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -22,9 +20,12 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; -import javax.annotation.Nullable; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { @@ -95,7 +96,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(doc); + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("playlistVideoListRenderer").getArray("continuations")); } @Nonnull @@ -174,8 +179,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); - collectStreamsFrom(collector, tbody); + + JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("playlistVideoListRenderer").getArray("contents"); + + collectStreamsFrom(collector, videos); return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -186,58 +197,42 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonObject pageJson; + JsonArray ajaxJson; try { - final String responseBody = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); - pageJson = JsonParser.object().from(responseBody); + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + ajaxJson = JsonParser.array().from(response); } catch (JsonParserException pe) { - throw new ParsingException("Could not parse ajax json", pe); + throw new ParsingException("Could not parse json data for next streams", pe); } - final Document pageHtml = Jsoup.parse("" - + pageJson.getString("content_html") - + "
    ", pageUrl); + JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("playlistVideoListContinuation"); - collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first()); + collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); - return new InfoItemsPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl)); + return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); } - private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl) - throws ParsingException { - String nextPageHtml = pageJson.getString("load_more_widget_html"); - if (!nextPageHtml.isEmpty()) { - return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl)); - } else { + private String getNextPageUrlFrom(JsonArray continuations) { + if (continuations == null) { return ""; } + + JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + String continuation = nextContinuationData.getString("continuation"); + String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; } - private String getNextPageUrlFrom(Document d) throws ParsingException { - try { - Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); - if (button != null) { - return button.attr("abs:data-uix-load-more-href"); - } else { - // Sometimes playlists are simply so small, they don't have a more streams/videos - return ""; - } - } catch (Exception e) { - throw new ParsingException("could not get next streams' url", e); - } - } - - private void collectStreamsFrom(@Nonnull StreamInfoItemsCollector collector, @Nullable Element element) { + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) { collector.reset(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") - .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("playlistVideoListRenderer").getArray("contents"); - for (Object video : videos) { if (((JsonObject) video).getObject("playlistVideoRenderer") != null) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { From 5d883d100c96233a4f068f1853e11297bb5533fe Mon Sep 17 00:00:00 2001 From: wb9688 Date: Mon, 24 Feb 2020 18:24:36 +0100 Subject: [PATCH 061/149] Implement pagination in YoutubeSearchExtractor --- .../extractors/YoutubeSearchExtractor.java | 96 +++++++++++-------- .../YoutubeSearchQueryHandlerFactory.java | 8 +- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 5d04a389..07954334 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -5,9 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -19,12 +17,12 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.utils.Parser; import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; @@ -73,58 +71,70 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() { - final Element el = doc.select("div[class*=\"spell-correction\"]").first(); - if (el != null) { - return el.select("a").first().text(); - } else { + JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + if (showingResultsForRenderer == null) { return ""; + } else { + return showingResultsForRenderer.getObject("correctedQuery").getArray("runs") + .getObject(0).getString("text"); } } @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException { - return new InfoItemsPage<>(collectItems(doc), getNextPageUrl()); + InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + JsonArray videos = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("contents"); + + collectStreamsFrom(collector, videos); + return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override public String getNextPageUrl() throws ExtractionException { - return getUrl() + "&page=" + 2; + return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("continuations")); } @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - // TODO: Get extracting next pages working - final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); - doc = Jsoup.parse(response, pageUrl); + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } - return new InfoItemsPage<>(collectItems(doc), getNextPageUrlFromCurrentUrl(pageUrl)); - } - - private String getNextPageUrlFromCurrentUrl(String currentUrl) - throws MalformedURLException, UnsupportedEncodingException { - final int pageNr = Integer.parseInt( - Parser.compatParseMap( - new URL(currentUrl) - .getQuery()) - .get("page")); - - return currentUrl.replace("&page=" + pageNr, - "&page=" + Integer.toString(pageNr + 1)); - } - - private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException, ParsingException { InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + JsonArray ajaxJson; + try { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException pe) { + throw new ParsingException("Could not parse json data for next streams", pe); + } + + JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") + .getObject("continuationContents").getObject("itemSectionContinuation"); + + collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); + + return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations"))); + } + + private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { collector.reset(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - if (initialData == null) initialData = YoutubeParsingHelper.getInitialData(doc.toString()); - JsonArray list = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("contents"); - - for (Object item : list) { + for (Object item : videos) { if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { throw new NothingFoundException(((JsonObject) item).getObject("backgroundPromoRenderer") .getObject("bodyText").getArray("runs").getObject(0).getString("text")); @@ -136,7 +146,17 @@ public class YoutubeSearchExtractor extends SearchExtractor { collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer"))); } } - return collector; } + private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException { + if (continuations == null) { + return ""; + } + + JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); + String continuation = nextContinuationData.getString("continuation"); + String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation + + "&itct=" + clickTrackingParams; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index c1760074..13481b34 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -24,13 +24,13 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { try { final String url = "https://www.youtube.com/results" - + "?q=" + URLEncoder.encode(searchString, CHARSET_UTF_8); + + "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8); if (contentFilters.size() > 0) { switch (contentFilters.get(0)) { - case VIDEOS: return url + "&sp=EgIQAVAU"; - case CHANNELS: return url + "&sp=EgIQAlAU"; - case PLAYLISTS: return url + "&sp=EgIQA1AU"; + case VIDEOS: return url + "&sp=EgIQAQ%253D%253D"; + case CHANNELS: return url + "&sp=EgIQAg%253D%253D"; + case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D"; case ALL: default: } From 5842b9ad37f760289f74856832e1db0076a5502e Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 24 Feb 2020 19:03:54 +0100 Subject: [PATCH 062/149] Add getClientVersion() and HARDCODED_CLIENT_VERSION to YouTubeParsingHelper Prefer hardcoded client version above the current one when making requests to retrieve the same JSON structure for each request. --- .../extractors/YoutubeChannelExtractor.java | 26 +++++++-- .../extractors/YoutubePlaylistExtractor.java | 26 +++++++-- .../linkHandler/YoutubeParsingHelper.java | 58 +++++++++++++++++++ 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 27678657..0cdddde7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -211,14 +211,30 @@ public class YoutubeChannelExtractor extends ChannelExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); try { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + // Use the hardcoded client version first to get JSON with a structure we know + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException pe) { - throw new ParsingException("Could not parse json data for next streams", pe); + } catch (Exception e) { + try { + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException ignored) { + throw new ParsingException("Could not parse json data for next streams", e); + } } JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 8abee5f1..73a2044b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -198,14 +198,30 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); try { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + // Use the hardcoded client version first to get JSON with a structure we know + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException pe) { - throw new ParsingException("Could not parse json data for next streams", pe); + } catch (Exception e) { + try { + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException ignored) { + throw new ParsingException("Could not parse json data for next streams", e); + } } JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 78516f6d..81e4feb3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; @@ -42,6 +43,8 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -157,4 +160,59 @@ public class YoutubeParsingHelper { } } + /** + * Get the client version from a page + * @param initialData + * @param html The page HTML + * @return + * @throws ParsingException + */ + public static String getClientVersion(JsonObject initialData, String html) throws ParsingException { + if (initialData == null) initialData = getInitialData(html); + JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + String shortClientVersion = null; + + // try to get version from initial data first + for (Object service : serviceTrackingParams) { + JsonObject s = (JsonObject) service; + if (s.getString("service").equals("CSI")) { + JsonArray params = s.getArray("params"); + for (Object param: params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("cver")) { + return p.getString("value"); + } + } + } else if (s.getString("service").equals("ECATCHER")) { + // fallback to get a shortened client version which does not contain the last do digits + JsonArray params = s.getArray("params"); + for (Object param: params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("client.version")) { + shortClientVersion = p.getString("value"); + } + } + } + } + + String clientVersion; + String[] patterns = { + "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", + "innertube_context_client_version\":\"([0-9\\.]+?)\"", + "client.version=([0-9\\.]+)" + }; + for (String pattern: patterns) { + try { + clientVersion = Parser.matchGroup1(pattern, html); + if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; + } catch (Exception ignored) {} + } + + if (shortClientVersion != null) return shortClientVersion; + + throw new ParsingException("Could not get client version"); + } + } From 583e9c157182d05f9e8b7bbd5dae211216a7ab76 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 24 Feb 2020 19:09:27 +0100 Subject: [PATCH 063/149] Fix getPage() Someone should check if there are unstaged changes before committing... --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 4 ++-- .../services/youtube/extractors/YoutubePlaylistExtractor.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 0cdddde7..930d3441 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -219,7 +219,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { headers.put("X-YouTube-Client-Version", Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() > 50) { // ensure to have a valid response + if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); } ajaxJson = JsonParser.array().from(response); @@ -228,7 +228,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { headers.put("X-YouTube-Client-Version", Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() > 50) { // ensure to have a valid response + if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); } ajaxJson = JsonParser.array().from(response); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 73a2044b..ce104a1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -206,7 +206,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { headers.put("X-YouTube-Client-Version", Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() > 50) { // ensure to have a valid response + if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); } ajaxJson = JsonParser.array().from(response); @@ -215,7 +215,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { headers.put("X-YouTube-Client-Version", Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() > 50) { // ensure to have a valid response + if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); } ajaxJson = JsonParser.array().from(response); From dd4dd849dce1238df9a818f0e97a68282037b288 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 24 Feb 2020 20:02:45 +0100 Subject: [PATCH 064/149] Get client version dynamically in YouTubeSearchExtractor --- .../extractors/YoutubeSearchExtractor.java | 28 +++++++++++++++---- .../linkHandler/YoutubeParsingHelper.java | 1 - 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 07954334..a35529aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; import java.util.Collections; @@ -111,14 +112,31 @@ public class YoutubeSearchExtractor extends SearchExtractor { InfoItemsSearchCollector collector = getInfoItemSearchCollector(); JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + try { - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow + // Use the hardcoded client version first to get JSON with a structure we know + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException pe) { - throw new ParsingException("Could not parse json data for next streams", pe); + } catch (Exception e) { + try { + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException ignored) { + throw new ParsingException("Could not parse json data for next streams", e); + } } JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 81e4feb3..51347d42 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -214,5 +214,4 @@ public class YoutubeParsingHelper { throw new ParsingException("Could not get client version"); } - } From 02b59903fa5a3c8d0c83166862d12ecf5febf1ff Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 09:07:22 +0100 Subject: [PATCH 065/149] Remove useless code --- .../extractors/YoutubePlaylistExtractor.java | 17 +-- .../extractors/YoutubeSearchExtractor.java | 3 +- .../extractors/YoutubeStreamExtractor.java | 114 +++++------------- .../extractors/YoutubeTrendingExtractor.java | 9 +- 4 files changed, 36 insertions(+), 107 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index ce104a1d..0ac2dcf0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -34,7 +34,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private JsonObject initialData; private JsonObject uploaderInfo; private JsonObject playlistInfo; - private JsonObject playlistVideos; public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -48,7 +47,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); uploaderInfo = getUploaderInfo(); playlistInfo = getPlaylistInfo(); - playlistVideos = getPlaylistVideos(); } private JsonObject getUploaderInfo() throws ParsingException { @@ -83,19 +81,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } - private JsonObject getPlaylistVideos() throws ParsingException { - try { - return initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer").getArray("contents") - .getObject(0).getObject("playlistVideoListRenderer"); - } catch (Exception e) { - throw new ParsingException("Could not get playlist info", e); - } - } - @Override - public String getNextPageUrl() throws ExtractionException { + public String getNextPageUrl() { return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") .getObject("sectionListRenderer").getArray("contents").getObject(0) @@ -177,7 +164,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException { + public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index a35529aa..b0669909 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -17,7 +17,6 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; import java.util.Collections; @@ -147,7 +146,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations"))); } - private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { + private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException { collector.reset(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index dd2987a4..29871b17 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -15,7 +14,6 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Request; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -36,7 +34,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -366,55 +363,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private long getLiveStreamWatchingCount() throws ExtractionException, IOException, JsonParserException { - // https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key= - String innerTubeKey = null, clientVersion = null; - if (playerArgs != null && !playerArgs.isEmpty()) { - innerTubeKey = playerArgs.getString("innertube_api_key"); - clientVersion = playerArgs.getString("innertube_context_client_version"); - } else if (!videoInfoPage.isEmpty()) { - innerTubeKey = videoInfoPage.get("innertube_api_key"); - clientVersion = videoInfoPage.get("innertube_context_client_version"); - } - - if (innerTubeKey == null || innerTubeKey.isEmpty()) { - throw new ExtractionException("Couldn't get innerTube key"); - } - - if (clientVersion == null || clientVersion.isEmpty()) { - throw new ExtractionException("Couldn't get innerTube client version"); - } - - final String metadataUrl = "https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=" + innerTubeKey; - final byte[] dataBody = ("{\"context\":{\"client\":{\"clientName\":1,\"clientVersion\":\"" + clientVersion + "\"}}" + - ",\"videoId\":\"" + getId() + "\"}").getBytes("UTF-8"); - final Response response = getDownloader().execute(Request.newBuilder() - .post(metadataUrl, dataBody) - .addHeader("Content-Type", "application/json") - .build()); - final JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - - for (Object actionEntry : jsonObject.getArray("actions")) { - if (!(actionEntry instanceof JsonObject)) continue; - final JsonObject entry = (JsonObject) actionEntry; - - final JsonObject updateViewershipAction = entry.getObject("updateViewershipAction", null); - if (updateViewershipAction == null) continue; - - final JsonArray viewCountRuns = JsonUtils.getArray(updateViewershipAction, "viewership.videoViewCountRenderer.viewCount.runs"); - if (viewCountRuns.isEmpty()) continue; - - final JsonObject textObject = viewCountRuns.getObject(0); - if (!textObject.has("text")) { - throw new ExtractionException("Response don't have \"text\" element"); - } - - return Long.parseLong(Utils.removeNonDigitCharacters(textObject.getString("text"))); - } - - throw new ExtractionException("Could not find correct results in response"); - } - private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("results").getObject("results").getArray("contents"); @@ -525,7 +473,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderAvatarUrl() throws ParsingException { assertPageFetched(); - String uploaderAvatarUrl = null; + String uploaderAvatarUrl; try { uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") @@ -657,13 +605,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitlesDefault() { return getSubtitles(MediaFormat.TTML); } @Override @Nonnull - public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { + public List getSubtitles(final MediaFormat format) { assertPageFetched(); List subtitles = new ArrayList<>(); for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) { @@ -687,7 +635,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamInfoItem getNextStream() throws IOException, ExtractionException { + public StreamInfoItem getNextStream() throws ExtractionException { assertPageFetched(); try { final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") @@ -815,12 +763,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { return JsonParser.object().from(ytPlayerConfigRaw); } catch (Parser.RegexException e) { String errorReason = getErrorMessage(); - switch (errorReason) { - case "": - throw new ContentNotAvailableException("Content not available: player config empty", e); - default: - throw new ContentNotAvailableException("Content not available", e); + if (errorReason.isEmpty()) { + throw new ContentNotAvailableException("Content not available: player config empty", e); } + throw new ContentNotAvailableException("Content not available", e); } catch (Exception e) { throw new ParsingException("Could not parse yt player config", e); } @@ -976,7 +922,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Nonnull - private List getAvailableSubtitlesInfo() throws SubtitlesException { + private List getAvailableSubtitlesInfo() { // If the video is age restricted getPlayerConfig will fail if (isAgeRestricted) return Collections.emptyList(); @@ -990,7 +936,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject()); final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray()); // todo: use this to apply auto translation to different language from a source language - final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray()); +// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray()); // This check is necessary since there may be cases where subtitles metadata do not contain caption track info // e.g. https://www.youtube.com/watch?v=-Vpwatutnko @@ -1147,40 +1093,44 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nonnull @Override - public String getHost() throws ParsingException { + public String getHost() { + return ""; + } + + @Nonnull + @Override + public String getPrivacy() { + return ""; + } + + @Nonnull + @Override + public String getCategory() { + return ""; + } + + @Nonnull + @Override + public String getLicence() { return ""; } @Override - public String getPrivacy() throws ParsingException { - return ""; - } - - @Override - public String getCategory() throws ParsingException { - return ""; - } - - @Override - public String getLicence() throws ParsingException { - return ""; - } - - @Override - public Locale getLanguageInfo() throws ParsingException { + public Locale getLanguageInfo() { return null; } @Nonnull @Override - public List getTags() throws ParsingException { + public List getTags() { return new ArrayList<>(); } @Nonnull @Override - public String getSupportInfo() throws ParsingException { + public String getSupportInfo() { return ""; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index d815e9e1..649cdf4e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -22,10 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -37,15 +34,12 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.utils.Parser; import java.io.IOException; import javax.annotation.Nonnull; public class YoutubeTrendingExtractor extends KioskExtractor { - - private Document doc; private JsonObject initialData; public YoutubeTrendingExtractor(StreamingService service, @@ -60,7 +54,6 @@ public class YoutubeTrendingExtractor extends KioskExtractor { "?gl=" + getExtractorContentCountry().getCountryCode(); final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); } @@ -92,7 +85,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws ParsingException { + public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray firstPageElements = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") From f13c0288cc0c8a94f741095bc207c0e744a3f5bf Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 09:50:22 +0100 Subject: [PATCH 066/149] Reimplement some methods in YoutubeStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 221 +++++------------- 1 file changed, 52 insertions(+), 169 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 29871b17..caea8ac5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.mozilla.javascript.Context; @@ -39,8 +38,6 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; -import java.net.URL; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -48,8 +45,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -75,8 +70,6 @@ import javax.annotation.Nullable; */ public class YoutubeStreamExtractor extends StreamExtractor { - private static final String TAG = YoutubeStreamExtractor.class.getSimpleName(); - /*////////////////////////////////////////////////////////////////////////// // Exceptions //////////////////////////////////////////////////////////////////////////*/ @@ -87,12 +80,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - public class SubtitlesException extends ContentNotAvailableException { - SubtitlesException(String message, Throwable cause) { - super(message, cause); - } - } - /*//////////////////////////////////////////////////////////////////////////*/ private Document doc; @@ -120,22 +107,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getName() throws ParsingException { assertPageFetched(); + String title = null; try { - return playerResponse.getObject("videoDetails").getString("title"); - - } catch (Exception e) { - // fallback HTML method - String name = null; + title = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs").getObject(0).getString("text"); + } catch (Exception ignored) {} + if (title == null) { try { - name = doc.select("meta[name=title]").attr(CONTENT); - } catch (Exception ignored) { - } - - if (name == null) { - throw new ParsingException("Could not get name", e); - } - return name; + title = playerResponse.getObject("videoDetails").getString("title"); + } catch (Exception ignored) {} } + if (title != null) return title; + throw new ParsingException("Could not get name"); } @Override @@ -144,19 +126,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { return null; } + // TODO: try videoPrimaryInfoRenderer.dateText.simpleText + try { return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); } catch (Exception e) { - String uploadDate = null; - try { - uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT); - } catch (Exception ignored) { - } - - if (uploadDate == null) { - throw new ParsingException("Could not get upload date", e); - } - return uploadDate; + throw new ParsingException("Could not get upload date"); } } @@ -181,15 +156,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return thumbnails.getObject(thumbnails.size() - 1).getString("url"); } catch (Exception e) { - String url = null; - try { - url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); - } catch (Exception ignored) {} - - if (url == null) { - throw new ParsingException("Could not get thumbnail url", e); - } - return url; + throw new ParsingException("Could not get thumbnail url"); } } @@ -198,93 +165,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public Description getDescription() throws ParsingException { assertPageFetched(); + // TODO: Parse videoSecondaryInfoRenderer.description try { - // first try to get html-formatted description - return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML); - } catch (Exception e) { - try { - // fallback to raw non-html description - return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); - } catch (Exception ignored) { - throw new ParsingException("Could not get the description", e); - } + // raw non-html description + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); + } catch (Exception ignored) { + throw new ParsingException("Could not get the description"); } } - // onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;" - // :00 is NOT recognized as a timestamp in description or comments. - // 0:00 is recognized in both description and comments. - // https://www.youtube.com/watch?v=4cccfDXu1vA - private final static Pattern DESCRIPTION_TIMESTAMP_ONCLICK_REGEX = Pattern.compile( - "seekTo\\(" - + "(?:(\\d+)\\*3600\\+)?" // hours? - + "(\\d+)\\*60\\+" // minutes - + "(\\d+)" // seconds - + "\\)"); - - @SafeVarargs - private static T coalesce(T... args) { - for (T arg : args) { - if (arg != null) return arg; - } - throw new IllegalArgumentException("all arguments to coalesce() were null"); - } - - private String parseHtmlAndGetFullLinks(String descriptionHtml) - throws MalformedURLException, UnsupportedEncodingException, ParsingException { - final Document description = Jsoup.parse(descriptionHtml, getUrl()); - for (Element a : description.select("a")) { - final String rawUrl = a.attr("abs:href"); - final URL redirectLink = new URL(rawUrl); - - final Matcher onClickTimestamp; - final String queryString; - if ((onClickTimestamp = DESCRIPTION_TIMESTAMP_ONCLICK_REGEX.matcher(a.attr("onclick"))) - .find()) { - a.removeAttr("onclick"); - - String hours = coalesce(onClickTimestamp.group(1), "0"); - String minutes = onClickTimestamp.group(2); - String seconds = onClickTimestamp.group(3); - - int timestamp = 0; - timestamp += Integer.parseInt(hours) * 3600; - timestamp += Integer.parseInt(minutes) * 60; - timestamp += Integer.parseInt(seconds); - - String setTimestamp = "&t=" + timestamp; - - // Even after clicking https://youtu.be/...?t=6, - // getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=. - a.attr("href", getUrl() + setTimestamp); - - } else if ((queryString = redirectLink.getQuery()) != null) { - // if the query string is null we are not dealing with a redirect link, - // so we don't need to override it. - final String link = - Parser.compatParseMap(queryString).get("q"); - - if (link != null) { - // if link is null the a tag is a hashtag. - // They refer to the youtube search. We do not handle them. - a.text(link); - a.attr("href", link); - } else if (redirectLink.toString().contains("https://www.youtube.com/")) { - a.text(redirectLink.toString()); - a.attr("href", redirectLink.toString()); - } - } else if (redirectLink.toString().contains("https://www.youtube.com/")) { - descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString()); - a.text(redirectLink.toString()); - a.attr("href", redirectLink.toString()); - } - } - return description.select("body").first().html(); - } - @Override public int getAgeLimit() throws ParsingException { assertPageFetched(); + // TODO: Find new way to get age limit if (!isAgeRestricted) { return NO_AGE_LIMIT; } @@ -332,54 +225,25 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getViewCount() throws ParsingException { assertPageFetched(); + String views = null; try { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { - // The array index is variable, therefore we loop throw the complete array. - // videoPrimaryInfoRenderer is often stored at index 1 - JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("results").getObject("results").getArray("contents"); - for (Object c : contents) { - try { - // this gets current view count, but there is also an overall view count which is stored here: - // contents.twoColumnWatchNextResults.secondaryResults.secondaryResults.results[0] - // .compactAutoplayRenderer.contents[0].compactVideoRenderer.viewCountText.simpleText - String views = ((JsonObject) c).getObject("videoPrimaryInfoRenderer") - .getObject("viewCount").getObject("videoViewCountRenderer").getObject("viewCount") - .getArray("runs").getObject(0).getString("text"); - return Long.parseLong(Utils.removeNonDigitCharacters(views)); - } catch (Exception ignored) {} - } - throw new ParsingException("Could not get view count from live stream"); - - } else { - return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount")); - } - } catch (Exception e) { + views = getVideoPrimaryInfoRenderer().getObject("viewCount") + .getObject("videoViewCountRenderer").getObject("viewCount") + .getArray("runs").getObject(0).getString("text"); + } catch (Exception ignored) {} + if (views == null) { try { - return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT)); - } catch (Exception ignored) { - throw new ParsingException("Could not get view count", e); - } + views = getVideoPrimaryInfoRenderer().getObject("viewCount") + .getObject("videoViewCountRenderer").getObject("viewCount").getString("simpleText"); + } catch (Exception ignored) {} } - } - - private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { - JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("results").getObject("results").getArray("contents"); - JsonObject videoPrimaryInfoRenderer = null; - - for (Object content : contents) { - if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) { - videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); - break; - } + if (views == null) { + try { + views = playerResponse.getObject("videoDetails").getString("viewCount"); + } catch (Exception ignored) {} } - - if (videoPrimaryInfoRenderer == null) { - throw new ParsingException("Could not find videoPrimaryInfoRenderer"); - } - - return videoPrimaryInfoRenderer; + if (views != null) return Long.parseLong(views); + throw new ParsingException("Could not get view count"); } @Override @@ -993,6 +857,25 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Utils //////////////////////////////////////////////////////////////////////////*/ + private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("results").getObject("results").getArray("contents"); + JsonObject videoPrimaryInfoRenderer = null; + + for (Object content : contents) { + if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) { + videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); + break; + } + } + + if (videoPrimaryInfoRenderer == null) { + throw new ParsingException("Could not find videoPrimaryInfoRenderer"); + } + + return videoPrimaryInfoRenderer; + } + @Nonnull private static String getVideoInfoUrl(final String id, final String sts) { return "https://www.youtube.com/get_video_info?" + "video_id=" + id + From 76d54abdbf00729c7ced10e753a13c39e11f7794 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 10:05:53 +0100 Subject: [PATCH 067/149] Reimplement more methods in YoutubeStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index caea8ac5..4afc7c1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -113,7 +113,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} if (title == null) { try { - title = playerResponse.getObject("videoDetails").getString("title"); + title = playerResponse.getObject("videoDetails").getString("title"); } catch (Exception ignored) {} } if (title != null) return title; @@ -296,40 +296,36 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderUrl() throws ParsingException { assertPageFetched(); + String uploaderId = null; try { - return "https://www.youtube.com/channel/" + - playerResponse.getObject("videoDetails").getString("channelId"); - } catch (Exception e) { - String uploaderUrl = null; + uploaderId = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") + .getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + } catch (Exception ignored) {} + if (uploaderId == null) { try { - uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children() - .select("a").first().attr("abs:href"); + uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); } catch (Exception ignored) {} - - if (uploaderUrl == null) { - throw new ParsingException("Could not get channel link", e); - } - return uploaderUrl; } + if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId; + throw new ParsingException("Could not get uploader url"); } @Nonnull @Override public String getUploaderName() throws ParsingException { assertPageFetched(); + String uploaderName = null; try { - return playerResponse.getObject("videoDetails").getString("author"); - } catch (Exception e) { - String name = null; + uploaderName = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") + .getObject("title").getArray("runs").getObject(0).getString("text"); + } catch (Exception ignored) {} + if (uploaderName == null) { try { - name = doc.select("div.yt-user-info").first().text(); + uploaderName = playerResponse.getObject("videoDetails").getString("author"); } catch (Exception ignored) {} - - if (name == null) { - throw new ParsingException("Could not get uploader name"); - } - return name; } + if (uploaderName != null) return uploaderName; + throw new ParsingException("Could not get uploader name"); } @Nonnull @@ -876,6 +872,25 @@ public class YoutubeStreamExtractor extends StreamExtractor { return videoPrimaryInfoRenderer; } + private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("results").getObject("results").getArray("contents"); + JsonObject videoSecondaryInfoRenderer = null; + + for (Object content : contents) { + if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) { + videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer"); + break; + } + } + + if (videoSecondaryInfoRenderer == null) { + throw new ParsingException("Could not find videoSecondaryInfoRenderer"); + } + + return videoSecondaryInfoRenderer; + } + @Nonnull private static String getVideoInfoUrl(final String id, final String sts) { return "https://www.youtube.com/get_video_info?" + "video_id=" + id + From 26ea3dceb6ea696b09a018f009c14d9cbe210d4d Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 10:08:52 +0100 Subject: [PATCH 068/149] Fix getViewCount() in YoutubeStreamExtractor --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 4afc7c1d..929c6c4d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -242,7 +242,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { views = playerResponse.getObject("videoDetails").getString("viewCount"); } catch (Exception ignored) {} } - if (views != null) return Long.parseLong(views); + if (views != null) return Long.parseLong(Utils.removeNonDigitCharacters(views)); throw new ParsingException("Could not get view count"); } From f39603f6ef75dbdcde341b723868c89f086e8d6c Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 25 Feb 2020 10:38:54 +0100 Subject: [PATCH 069/149] Implement getUploadDate() in YouTubeStreamInfoItemExtractor --- .../extractors/YoutubeChannelExtractor.java | 2 -- .../extractors/YoutubeStreamExtractor.java | 2 -- .../YoutubeStreamInfoItemExtractor.java | 18 ++++++++++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 930d3441..4c7185f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -164,11 +164,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); if (subscriberInfo != null) { try { - return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text")); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 929c6c4d..62876d38 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -512,7 +512,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - @Override public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); @@ -678,7 +677,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - @Nonnull private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 605053bb..950bab0f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -158,12 +158,26 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getTextualUploadDate() { // TODO: Get upload date in case of a videoRenderer (not available in case of a compactVideoRenderer) - return null; + try { + String s =videoInfo.getObject("publishedTimeText").getString("simpleText"); + return s; + } catch (Exception e) { + // upload date is not always available, e.g. in playlists + return null; + } } @Nullable @Override - public DateWrapper getUploadDate() { + public DateWrapper getUploadDate() throws ParsingException { + String textualUploadDate = getTextualUploadDate(); + if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) { + try { + return timeAgoParser.parse(textualUploadDate); + } catch (ParsingException e) { + throw new ParsingException("Could not get upload date", e); + } + } return null; } From 9efcc61ca6ed25499d4367f8ff95c34ea8db6047 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 25 Feb 2020 10:40:06 +0100 Subject: [PATCH 070/149] Adapt some tests to match the new URLs --- .../search/YoutubeSearchExtractorChannelOnlyTest.java | 9 ++++++++- .../services/youtube/search/YoutubeSearchQHTest.java | 8 ++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 63fc0375..7add4126 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -12,6 +12,8 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import java.util.regex.Pattern; + import static java.util.Arrays.asList; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -51,7 +53,12 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl()); + // check that ctoken, continuation and itct are longer than 5 characters + Pattern pattern = Pattern.compile( + "https:\\/\\/www.youtube.com\\/results\\?search_query=pewdiepie&sp=EgIQAg%253D%253D&gl=GB&pbj=1" + + "&ctoken=[\\w%]{5,}?&continuation=[\\w%]{5,}?&itct=[\\w]{5,}?" + ); + assertTrue(pattern.matcher(extractor.getNextPageUrl()).find()); } @Ignore diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 60bbf2ff..8777cc70 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -28,13 +28,13 @@ public class YoutubeSearchQHTest { @Test public void testWithContentfilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAVAU", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAlAU", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQA1AU", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory() + assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); } From eed29ea734ae3008d80f5754ca2cab7eed0578df Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 16:14:49 +0100 Subject: [PATCH 071/149] Return null for related streams at age restricted videos --- .../youtube/extractors/YoutubeStreamExtractor.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 62876d38..aa610d36 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -177,7 +177,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() throws ParsingException { assertPageFetched(); - // TODO: Find new way to get age limit if (!isAgeRestricted) { return NO_AGE_LIMIT; } @@ -497,6 +496,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItem getNextStream() throws ExtractionException { assertPageFetched(); + if (isAgeRestricted) { + return null; + } try { final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults").getObject("secondaryResults").getArray("results") @@ -515,6 +517,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); + if (isAgeRestricted) { + return null; + } try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") From ecfc27a92c9a57345eca43305d7eb3ce625ce2ce Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 25 Feb 2020 16:24:18 +0100 Subject: [PATCH 072/149] Implement fallback methods for getDescription() and getTextualUploadDate() in YouTubeStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 49 +++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index aa610d36..7f8e84d3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -38,13 +38,8 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.text.SimpleDateFormat; +import java.util.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -126,13 +121,28 @@ public class YoutubeStreamExtractor extends StreamExtractor { return null; } - // TODO: try videoPrimaryInfoRenderer.dateText.simpleText + try { + //return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); + } catch (Exception ignored) {} try { - return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") + .getObject("results").getArray("contents"); + for (Object c: contents) { + String unformattedDate = ""; + try { + JsonObject o = (JsonObject) c; + unformattedDate = o.getObject("videoPrimaryInfoRenderer").getObject("dateText").getString("simpleText"); + + } catch (Exception ignored) {/* we got the wrong element form the array */} + // TODO this parses English formatted dates only, we need a better approach to parse teh textual date + Date d = new SimpleDateFormat("dd MMM yyy").parse(unformattedDate); + return new SimpleDateFormat("yyyy-MM-dd").format(d); + } } catch (Exception e) { - throw new ParsingException("Could not get upload date"); + throw new ParsingException("Could not get upload date", e); } + throw new ParsingException("Could not get upload date"); } @Override @@ -165,13 +175,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public Description getDescription() throws ParsingException { assertPageFetched(); - // TODO: Parse videoSecondaryInfoRenderer.description + // raw non-html description try { - // raw non-html description return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); - } catch (Exception ignored) { - throw new ParsingException("Could not get the description"); - } + } catch (Exception ignored) { } + try { + JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs"); + StringBuilder descriptionBuilder = new StringBuilder(descriptions.size()); + for (Object textObjectHolder : descriptions) { + JsonObject textHolder = (JsonObject) textObjectHolder; + String text = textHolder.getString("text"); + if (text != null) descriptionBuilder.append(text); + } + String description = descriptionBuilder.toString(); + if (!description.isEmpty()) return new Description(description, Description.PLAIN_TEXT); + } catch (Exception ignored) { } + throw new ParsingException("Could not get description"); } @Override From 8e6be880d7641d3a722a12da308cb5046ab49249 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 16:32:08 +0100 Subject: [PATCH 073/149] Simplify getId() in YoutubeChannelExtractor --- .../extractors/YoutubeChannelExtractor.java | 24 ++++++------------- .../YoutubeStreamInfoItemExtractor.java | 4 +--- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 4c7185f1..7c448662 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -6,7 +6,6 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -91,18 +90,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getId() throws ParsingException { try { - return doc.select("meta[property=\"og:url\"]").first().attr("content").replace(CHANNEL_URL_BASE, ""); - } catch (Exception ignored) {} - try { - return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); - } catch (Exception ignored) {} - - // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) - try { - Element element = doc.getElementsByClass("yt-uix-subscription-button").first(); - if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first(); - - return element.attr("data-channel-external-id"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("channelId"); } catch (Exception e) { throw new ParsingException("Could not get channel id", e); } @@ -112,7 +100,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getName() throws ParsingException { try { - return doc.select("meta[property=\"og:title\"]").first().attr("content"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title"); } catch (Exception e) { throw new ParsingException("Could not get channel name", e); } @@ -121,7 +109,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { try { - return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar").getArray("thumbnails").getObject(0).getString("url"); + return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") + .getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -132,7 +121,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { try { String url = null; try { - url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner").getArray("thumbnails").getObject(0).getString("url"); + url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner") + .getArray("thumbnails").getObject(0).getString("url"); } catch (Exception ignored) {} if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; @@ -149,7 +139,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return url; } catch (Exception e) { - throw new ParsingException("Could not get Banner", e); + throw new ParsingException("Could not get banner", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 950bab0f..2010cfb5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -157,10 +157,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public String getTextualUploadDate() { - // TODO: Get upload date in case of a videoRenderer (not available in case of a compactVideoRenderer) try { - String s =videoInfo.getObject("publishedTimeText").getString("simpleText"); - return s; + return videoInfo.getObject("publishedTimeText").getString("simpleText"); } catch (Exception e) { // upload date is not always available, e.g. in playlists return null; From bbe1a3cd6263388876397a6a14940e76933f0bd3 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 17:40:23 +0100 Subject: [PATCH 074/149] Use getVideo(Primary|Secondary)InfoRenderer() --- .../extractors/YoutubeStreamExtractor.java | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 7f8e84d3..5d2b73bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -126,22 +126,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} try { - JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") - .getObject("results").getArray("contents"); - for (Object c: contents) { - String unformattedDate = ""; - try { - JsonObject o = (JsonObject) c; - unformattedDate = o.getObject("videoPrimaryInfoRenderer").getObject("dateText").getString("simpleText"); - - } catch (Exception ignored) {/* we got the wrong element form the array */} - // TODO this parses English formatted dates only, we need a better approach to parse teh textual date - Date d = new SimpleDateFormat("dd MMM yyy").parse(unformattedDate); - return new SimpleDateFormat("yyyy-MM-dd").format(d); - } - } catch (Exception e) { - throw new ParsingException("Could not get upload date", e); - } + // TODO this parses English formatted dates only, we need a better approach to parse the textual date + Date d = new SimpleDateFormat("dd MMM yyy").parse(getVideoPrimaryInfoRenderer() + .getObject("dateText").getString("simpleText")); + return new SimpleDateFormat("yyyy-MM-dd").format(d); + } catch (Exception ignored) {} throw new ParsingException("Could not get upload date"); } @@ -351,7 +340,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderAvatarUrl() throws ParsingException { assertPageFetched(); - String uploaderAvatarUrl; + String uploaderAvatarUrl = null; try { uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") @@ -363,13 +352,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} try { - uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("results") - .getObject("results").getArray("contents").getObject(1).getObject("videoSecondaryInfoRenderer") - .getObject("owner").getObject("videoOwnerRenderer").getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url"); - } catch (Exception e) { - throw new ParsingException("Could not get uploader avatar url", e); - } + uploaderAvatarUrl = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") + .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception ignored) {} if (uploaderAvatarUrl == null) { throw new ParsingException("Could not get uploader avatar url"); From 5d08c343227e44ebc6bb8acca73072e2e2878fa5 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 19:47:26 +0100 Subject: [PATCH 075/149] Fix parsing upload date in YoutubeStreamExtractor --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 5d2b73bd..26dc2443 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -127,8 +127,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { // TODO this parses English formatted dates only, we need a better approach to parse the textual date - Date d = new SimpleDateFormat("dd MMM yyy").parse(getVideoPrimaryInfoRenderer() - .getObject("dateText").getString("simpleText")); + Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( + getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText")); return new SimpleDateFormat("yyyy-MM-dd").format(d); } catch (Exception ignored) {} throw new ParsingException("Could not get upload date"); From 0fff03038ed285a2e1f2610b5d6eaab97c75ee35 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 25 Feb 2020 18:27:39 +0100 Subject: [PATCH 076/149] Get description with correct links --- .../extractors/YoutubeStreamExtractor.java | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 26dc2443..09a00ef6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -38,6 +38,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.*; @@ -164,22 +165,57 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public Description getDescription() throws ParsingException { assertPageFetched(); - // raw non-html description - try { - return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); - } catch (Exception ignored) { } + + // description with more info on links try { + boolean htmlConversionRequired = false; JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs"); StringBuilder descriptionBuilder = new StringBuilder(descriptions.size()); for (Object textObjectHolder : descriptions) { JsonObject textHolder = (JsonObject) textObjectHolder; String text = textHolder.getString("text"); - if (text != null) descriptionBuilder.append(text); + if (textHolder.getObject("navigationEndpoint") != null) { + // The text is a link. Get the URL it points to and generate a HTML link of it + String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url"); + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + } + String[] params = internUrl.split("&"); + for (String param : params) { + if (param.charAt(0) == 'q') { + String url = java.net.URLDecoder.decode(param.substring(2), StandardCharsets.UTF_8.name()); + if (url != null && !url.isEmpty()) { + descriptionBuilder.append("").append(text).append(""); + htmlConversionRequired = true; + } else { + descriptionBuilder.append(text); + } + } + } + } else if (text != null) { + descriptionBuilder.append(text); + } } + String description = descriptionBuilder.toString(); - if (!description.isEmpty()) return new Description(description, Description.PLAIN_TEXT); + + if (!description.isEmpty()) { + if (htmlConversionRequired) { + description = description.replaceAll("\\n", "
    "); + return new Description(description, Description.HTML); + } + return new Description(description, Description.PLAIN_TEXT); + } + } catch (Exception ignored) { } - throw new ParsingException("Could not get description"); + + // raw non-html description + try { + return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT); + } catch (Exception ignored) { + throw new ParsingException("Could not get description"); + } } @Override From 26fb44595f7af1f00d50a0fec2b4ec84a3625337 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 20:40:47 +0100 Subject: [PATCH 077/149] Fix parsing description --- .../extractors/YoutubeStreamExtractor.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 09a00ef6..28817ae0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -38,9 +38,17 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -180,18 +188,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (internUrl.startsWith("/redirect?")) { // q parameter can be the first parameter internUrl = internUrl.substring(10); - } - String[] params = internUrl.split("&"); - for (String param : params) { - if (param.charAt(0) == 'q') { - String url = java.net.URLDecoder.decode(param.substring(2), StandardCharsets.UTF_8.name()); - if (url != null && !url.isEmpty()) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } else { - descriptionBuilder.append(text); + String[] params = internUrl.split("&"); + for (String param : params) { + if (param.split("=")[0].equals("q")) { + String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); + if (url != null && !url.isEmpty()) { + descriptionBuilder.append("").append(text).append(""); + htmlConversionRequired = true; + } else { + descriptionBuilder.append(text); + } + break; } } + } else if (internUrl.startsWith("http")) { + descriptionBuilder.append("").append(text).append(""); + htmlConversionRequired = true; + } else if (text != null) { + descriptionBuilder.append(text); } } else if (text != null) { descriptionBuilder.append(text); @@ -203,11 +217,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (!description.isEmpty()) { if (htmlConversionRequired) { description = description.replaceAll("\\n", "
    "); + description = description.replaceAll(" ", "  "); return new Description(description, Description.HTML); } return new Description(description, Description.PLAIN_TEXT); } - } catch (Exception ignored) { } // raw non-html description From 729fd2eaeec758019604b987c6d43522f4879c01 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 20:54:11 +0100 Subject: [PATCH 078/149] Fix parsing description for navigationEndpoint without urlEndpoint --- .../extractors/YoutubeStreamExtractor.java | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 28817ae0..b95ab08c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -173,7 +173,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public Description getDescription() throws ParsingException { assertPageFetched(); - // description with more info on links try { boolean htmlConversionRequired = false; @@ -184,30 +183,33 @@ public class YoutubeStreamExtractor extends StreamExtractor { String text = textHolder.getString("text"); if (textHolder.getObject("navigationEndpoint") != null) { // The text is a link. Get the URL it points to and generate a HTML link of it - String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url"); - if (internUrl.startsWith("/redirect?")) { - // q parameter can be the first parameter - internUrl = internUrl.substring(10); - String[] params = internUrl.split("&"); - for (String param : params) { - if (param.split("=")[0].equals("q")) { - String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); - if (url != null && !url.isEmpty()) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } else { - descriptionBuilder.append(text); + if (textHolder.getObject("navigationEndpoint").getObject("urlEndpoint") != null) { + String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url"); + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + String[] params = internUrl.split("&"); + for (String param : params) { + if (param.split("=")[0].equals("q")) { + String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); + if (url != null && !url.isEmpty()) { + descriptionBuilder.append("").append(text).append(""); + htmlConversionRequired = true; + } else { + descriptionBuilder.append(text); + } + break; } - break; } + } else if (internUrl.startsWith("http")) { + descriptionBuilder.append("").append(text).append(""); + htmlConversionRequired = true; } - } else if (internUrl.startsWith("http")) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } else if (text != null) { - descriptionBuilder.append(text); + continue; } - } else if (text != null) { + continue; + } + if (text != null) { descriptionBuilder.append(text); } } From 5a35300a5e3fc0221a6349351306a964e74adfb7 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 25 Feb 2020 21:19:53 +0100 Subject: [PATCH 079/149] Fix parsing upload date for premiered streams --- .../youtube/extractors/YoutubeStreamExtractor.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b95ab08c..91c088fa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -20,7 +20,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; @@ -42,6 +44,7 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -134,6 +137,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { //return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); } catch (Exception ignored) {} + try { + if (getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").startsWith("Premiered")) { + String timeAgo = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10); + TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); + Calendar parsedTimeAgo = timeAgoParser.parse(timeAgo).date(); + return new SimpleDateFormat("yyyy-MM-dd").format(parsedTimeAgo.getTime()); + } + } catch (Exception ignored) {} + try { // TODO this parses English formatted dates only, we need a better approach to parse the textual date Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( From 96285e08151ef5a48afdd7ba28b7857924ddd6ca Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 25 Feb 2020 21:50:11 +0100 Subject: [PATCH 080/149] Fix getUploadDate() for premiered videos with a given date --- .../extractors/YoutubeStreamExtractor.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 91c088fa..2408b5c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -134,15 +134,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - //return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); + // return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); } catch (Exception ignored) {} try { if (getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").startsWith("Premiered")) { - String timeAgo = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10); - TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); - Calendar parsedTimeAgo = timeAgoParser.parse(timeAgo).date(); - return new SimpleDateFormat("yyyy-MM-dd").format(parsedTimeAgo.getTime()); + String time = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10); + + try { // Premiered 20 hours ago + TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); + Calendar parsedTime = timeAgoParser.parse(time).date(); + return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); + } catch (Exception ignored) {} + + try { // Premiered Premiered Feb 21, 2020 + Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); + return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); + } catch (Exception ignored) {} } } catch (Exception ignored) {} From d1b7461d0e224454070ad798e070ded8f089fc92 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 25 Feb 2020 22:00:47 +0100 Subject: [PATCH 081/149] Version 0.18.5 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fcfa4fa9..81b48035 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.4' + version 'v0.18.5' group 'com.github.TeamNewPipe' repositories { From fc465c8bf7b70d9052fc5bf1f524f1c97b330718 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 25 Feb 2020 22:54:37 +0100 Subject: [PATCH 082/149] Fix getUploadDate() for premiered videos with a given date Detect publishDate and uploadDate --- .../youtube/extractors/YoutubeStreamExtractor.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 2408b5c0..b5d03edc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -134,7 +134,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - // return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate"); + JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); + if (micro.getString("uploadDate") != null && !micro.getString("uploadDate").isEmpty()) { + return micro.getString("uploadDate"); + } + if (micro.getString("publishDate") != null && !micro.getString("publishDate").isEmpty()) { + return micro.getString("publishDate"); + } } catch (Exception ignored) {} try { @@ -147,7 +153,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); } catch (Exception ignored) {} - try { // Premiered Premiered Feb 21, 2020 + try { // Premiered Feb 21, 2020 Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); } catch (Exception ignored) {} From 0973263aab14b75ab56216ea795b31c6f562f15d Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 09:01:57 +0100 Subject: [PATCH 083/149] Use pbj in YoutubeSearchExtractor --- .../extractors/YoutubeSearchExtractor.java | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index b0669909..2396498f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -47,8 +47,6 @@ import javax.annotation.Nonnull; */ public class YoutubeSearchExtractor extends SearchExtractor { - - private Document doc; private JsonObject initialData; public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { @@ -57,10 +55,28 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); + final String url = getUrl() + "&pbj=1"; + + JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + // Use the hardcoded client version first to get JSON with a structure we know + // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + + try { + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); + } + + initialData = ajaxJson.getObject(1).getObject("response"); } @Nonnull @@ -114,30 +130,22 @@ public class YoutubeSearchExtractor extends SearchExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + // Use the hardcoded client version first to get JSON with a structure we know + // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } try { - // Use the hardcoded client version first to get JSON with a structure we know - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } ajaxJson = JsonParser.array().from(response); - } catch (Exception e) { - try { - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException ignored) { - throw new ParsingException("Could not parse json data for next streams", e); - } + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); } + JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("itemSectionContinuation"); From 89a41a7f69b63c09e4cc8b7f381e07380718c297 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 09:31:26 +0100 Subject: [PATCH 084/149] Use pbj in YoutubeChannelExtractor --- .../extractors/YoutubeChannelExtractor.java | 77 +++++++++++-------- .../YoutubeStreamInfoItemExtractor.java | 19 ++++- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 7c448662..3b1a9e66 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -53,9 +53,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @SuppressWarnings("WeakerAccess") public class YoutubeChannelExtractor extends ChannelExtractor { /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; - private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; - private Document doc; private JsonObject initialData; public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -64,16 +62,36 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; - final Response response = downloader.get(channelUrl, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); + final String url = super.getUrl() + "/videos?pbj=1"; + + JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + // Use the hardcoded client version first to get JSON with a structure we know + // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + + try { + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); + } + + initialData = ajaxJson.getObject(1).getObject("response"); } @Override public String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("continuations")); + return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations")); } @Nonnull @@ -181,7 +199,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents"); + JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("gridRenderer").getArray("items"); collectStreamsFrom(collector, videos); return new InfoItemsPage<>(collector, getNextPageUrl()); @@ -202,33 +222,25 @@ public class YoutubeChannelExtractor extends ChannelExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + // Use the hardcoded client version first to get JSON with a structure we know + // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + try { - // Use the hardcoded client version first to get JSON with a structure we know - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } ajaxJson = JsonParser.array().from(response); - } catch (Exception e) { - try { - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException ignored) { - throw new ParsingException("Could not parse json data for next streams", e); - } + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); } JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("sectionListContinuation"); + .getObject("continuationContents").getObject("gridContinuation"); - collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); + collectStreamsFrom(collector, sectionListContinuation.getArray("items")); return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); } @@ -254,10 +266,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object video : videos) { - JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer") - .getArray("contents").getObject(0); - if (videoInfo.getObject("videoRenderer") != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo.getObject("videoRenderer"), timeAgoParser) { + if (((JsonObject) video).getObject("gridVideoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor( + ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) { @Override public String getUploaderName() { return uploaderName; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 2010cfb5..0f9c1c50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -91,12 +91,23 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getDuration() throws ParsingException { + if (getStreamType() == StreamType.LIVE_STREAM) return -1; + String duration = null; try { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); - } catch (Exception e) { - throw new ParsingException("Could not get duration", e); + duration = videoInfo.getObject("lengthText").getString("simpleText"); + } catch (Exception ignored) {} + if (duration == null) { + try { + for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { + if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) { + duration = ((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") + .getObject("text").getString("simpleText"); + } + } + } catch (Exception ignored) {} } + if (duration != null) return YoutubeParsingHelper.parseDurationString(duration); + throw new ParsingException("Could not get duration"); } @Override From 15afbea3e19f83b33cabdca71fbdce669179b2ef Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 14:45:50 +0100 Subject: [PATCH 085/149] Use pbj in YoutubePlaylistExtractor --- .../extractors/YoutubePlaylistExtractor.java | 60 ++++++++++--------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 0ac2dcf0..47620030 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -5,10 +5,8 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -29,8 +27,6 @@ import javax.annotation.Nonnull; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { - - private Document doc; private JsonObject initialData; private JsonObject uploaderInfo; private JsonObject playlistInfo; @@ -41,10 +37,28 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); + final String url = getUrl() + "&pbj=1"; + + JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + // Use the hardcoded client version first to get JSON with a structure we know + // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + + try { + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); + } + + initialData = ajaxJson.getObject(1).getObject("response"); uploaderInfo = getUploaderInfo(); playlistInfo = getPlaylistInfo(); } @@ -188,27 +202,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + // Use the hardcoded client version first to get JSON with a structure we know + // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + try { - // Use the hardcoded client version first to get JSON with a structure we know - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } ajaxJson = JsonParser.array().from(response); - } catch (Exception e) { - try { - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException ignored) { - throw new ParsingException("Could not parse json data for next streams", e); - } + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); } JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") From 985c3ec877fa8e62073bb1fc81135e08ac33fda2 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 15:22:59 +0100 Subject: [PATCH 086/149] Improve getClientVersion() --- .../extractors/YoutubeChannelExtractor.java | 10 +- .../extractors/YoutubePlaylistExtractor.java | 8 +- .../extractors/YoutubeSearchExtractor.java | 10 +- .../linkHandler/YoutubeParsingHelper.java | 117 ++++++++++++------ 4 files changed, 83 insertions(+), 62 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 3b1a9e66..41b0de1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; @@ -68,10 +66,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); @@ -222,10 +218,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 47620030..84af2146 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -43,10 +43,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); @@ -202,10 +200,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 2396498f..7ec0e5cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; @@ -61,10 +59,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); @@ -130,10 +126,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - // Use the hardcoded client version first to get JSON with a structure we know - // TODO: Use YoutubeParsingHelper.getClientVersion() as fallback headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("Could not parse json data for next streams"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 51347d42..10caa509 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; @@ -16,7 +17,13 @@ import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; /* * Created by Christian Schabesberger on 02.03.16. @@ -43,7 +50,9 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } - public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static String clientVersion; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -162,55 +171,83 @@ public class YoutubeParsingHelper { /** * Get the client version from a page - * @param initialData - * @param html The page HTML * @return * @throws ParsingException */ - public static String getClientVersion(JsonObject initialData, String html) throws ParsingException { - if (initialData == null) initialData = getInitialData(html); - JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); - String shortClientVersion = null; + public static String getClientVersion() throws ParsingException { + if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - // try to get version from initial data first - for (Object service : serviceTrackingParams) { - JsonObject s = (JsonObject) service; - if (s.getString("service").equals("CSI")) { - JsonArray params = s.getArray("params"); - for (Object param: params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("cver")) { - return p.getString("value"); + // Test if hard-coded client version is valid + try { + final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + clientVersion = HARDCODED_CLIENT_VERSION; + return clientVersion; + } + } catch (Exception ignored) {} + + // Try extracting it from YouTube's website otherwise + try { + final String url = "https://www.youtube.com/results?search_query=test"; + final String html = getDownloader().get(url).responseBody(); + JsonObject initialData = getInitialData(html); + JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + String shortClientVersion = null; + + // try to get version from initial data first + for (Object service : serviceTrackingParams) { + JsonObject s = (JsonObject) service; + if (s.getString("service").equals("CSI")) { + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("cver")) { + clientVersion = p.getString("value"); + return clientVersion; + } } - } - } else if (s.getString("service").equals("ECATCHER")) { - // fallback to get a shortened client version which does not contain the last do digits - JsonArray params = s.getArray("params"); - for (Object param: params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("client.version")) { - shortClientVersion = p.getString("value"); + } else if (s.getString("service").equals("ECATCHER")) { + // fallback to get a shortened client version which does not contain the last do digits + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("client.version")) { + shortClientVersion = p.getString("value"); + } } } } - } - String clientVersion; - String[] patterns = { - "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", - "innertube_context_client_version\":\"([0-9\\.]+?)\"", - "client.version=([0-9\\.]+)" - }; - for (String pattern: patterns) { - try { - clientVersion = Parser.matchGroup1(pattern, html); - if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - } catch (Exception ignored) {} - } + String contextClientVersion; + String[] patterns = { + "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", + "innertube_context_client_version\":\"([0-9\\.]+?)\"", + "client.version=([0-9\\.]+)" + }; + for (String pattern : patterns) { + try { + contextClientVersion = Parser.matchGroup1(pattern, html); + if (contextClientVersion != null && !contextClientVersion.isEmpty()) { + clientVersion = contextClientVersion; + return clientVersion; + } + } catch (Exception ignored) { + } + } - if (shortClientVersion != null) return shortClientVersion; + if (shortClientVersion != null) { + clientVersion = shortClientVersion; + return clientVersion; + } + } catch (Exception ignored) {} throw new ParsingException("Could not get client version"); } From 1c3a4e710ba820f43208a75045c03d61e914359a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 16:03:06 +0100 Subject: [PATCH 087/149] Check if there are no related streams in getAgeLimit() --- .../extractors/YoutubeStreamExtractor.java | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b5d03edc..516dc181 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -100,8 +100,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private List subtitlesInfos = new ArrayList<>(); - private boolean isAgeRestricted; - public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { super(service, linkHandler); } @@ -261,17 +259,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public int getAgeLimit() throws ParsingException { + public int getAgeLimit() { assertPageFetched(); - if (!isAgeRestricted) { + if (initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("secondaryResults") == null) { + return 18; + } else { return NO_AGE_LIMIT; } - try { - return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]") - .attr(CONTENT).replace("+", "")); - } catch (Exception e) { - throw new ParsingException("Could not get age restriction"); - } } @Override @@ -578,7 +573,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItem getNextStream() throws ExtractionException { assertPageFetched(); - if (isAgeRestricted) { + if (getAgeLimit() == 18) { return null; } try { @@ -599,7 +594,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); - if (isAgeRestricted) { + if (getAgeLimit() == 18) { return null; } try { @@ -678,18 +673,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String playerUrl; // Check if the video is age restricted - if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) { + if (getAgeLimit() == 18) { final EmbeddedInfo info = getEmbeddedInfo(); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); playerUrl = info.url; - isAgeRestricted = true; } else { final JsonObject ytPlayerConfig = getPlayerConfig(); playerArgs = getPlayerArgs(ytPlayerConfig); playerUrl = getPlayerUrl(ytPlayerConfig); - isAgeRestricted = false; } playerResponse = getPlayerResponse(); initialData = YoutubeParsingHelper.getInitialData(pageHtml); @@ -869,7 +862,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private List getAvailableSubtitlesInfo() { // If the video is age restricted getPlayerConfig will fail - if (isAgeRestricted) return Collections.emptyList(); + if (getAgeLimit() == 18) return Collections.emptyList(); final JsonObject captions; if (!playerResponse.has("captions")) { From bb3a3d70bf14cff501fbe514ce05f10cc6f4a148 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 16:09:32 +0100 Subject: [PATCH 088/149] Use pbj in YoutubeTrendingExtractor --- .../extractors/YoutubeTrendingExtractor.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 649cdf4e..0b1667cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -22,10 +22,11 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; @@ -36,6 +37,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; @@ -50,11 +55,27 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + - "?gl=" + getExtractorContentCountry().getCountryCode(); + final String url = getUrl() + "?pbj=1&gl=" + + getExtractorContentCountry().getCountryCode(); - final Response response = downloader.get(url, getExtractorLocalization()); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); + JsonArray ajaxJson; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); + final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + + try { + ajaxJson = JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); + } + + initialData = ajaxJson.getObject(1).getObject("response"); } @Override From 88282761f44921e0533afdf93a4d83de0c0a4c20 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 16:19:42 +0100 Subject: [PATCH 089/149] Fix YoutubeStreamExtractor not being able to getAgeLimit() in onFetchPage() --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 516dc181..8a725593 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -260,7 +260,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() { - assertPageFetched(); + if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet"); if (initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults") == null) { return 18; @@ -672,6 +672,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response); final String playerUrl; + initialData = YoutubeParsingHelper.getInitialData(pageHtml); // Check if the video is age restricted if (getAgeLimit() == 18) { final EmbeddedInfo info = getEmbeddedInfo(); @@ -685,7 +686,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerUrl = getPlayerUrl(ytPlayerConfig); } playerResponse = getPlayerResponse(); - initialData = YoutubeParsingHelper.getInitialData(pageHtml); if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); From 8dc39d517ae3e8080f902ceb28fe173cf0d75ff7 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 17:31:01 +0100 Subject: [PATCH 090/149] Fix tests Except testGetFullLinksInDescription() in DescriptionTestUnboxing, since that requires other changes first. --- .../linkHandler/YoutubeParsingHelper.java | 2 +- ...YoutubeSearchExtractorChannelOnlyTest.java | 37 +++++++++++++------ .../YoutubeSearchExtractorDefaultTest.java | 30 ++++++++++++--- .../youtube/search/YoutubeSearchQHTest.java | 10 ++--- .../YoutubeStreamExtractorDefaultTest.java | 28 +++++++------- .../YoutubeStreamExtractorLivestreamTest.java | 20 +++++----- 6 files changed, 82 insertions(+), 45 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 10caa509..24edef60 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -214,7 +214,7 @@ public class YoutubeParsingHelper { } } } else if (s.getString("service").equals("ECATCHER")) { - // fallback to get a shortened client version which does not contain the last do digits + // fallback to get a shortened client version which does not contain the last two digits JsonArray params = s.getArray("params"); for (Object param : params) { JsonObject p = (JsonObject) param; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 7add4126..eb56bb5e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -12,10 +12,17 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import java.util.regex.Pattern; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; import static java.util.Arrays.asList; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { @@ -47,18 +54,26 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - // check that ctoken, continuation and itct are longer than 5 characters - Pattern pattern = Pattern.compile( - "https:\\/\\/www.youtube.com\\/results\\?search_query=pewdiepie&sp=EgIQAg%253D%253D&gl=GB&pbj=1" - + "&ctoken=[\\w%]{5,}?&continuation=[\\w%]{5,}?&itct=[\\w]{5,}?" - ); - assertTrue(pattern.matcher(extractor.getNextPageUrl()).find()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Ignore @@ -77,7 +92,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto if (item instanceof ChannelInfoItem) { ChannelInfoItem channel = (ChannelInfoItem) item; - if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie + if (channel.getSubscriberCount() > 5e7 && channel.getName().equals("PewDiePie")) { // the real PewDiePie assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); } else { assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 3078d873..65ffe839 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -10,6 +10,11 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -48,13 +53,28 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testGetUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl()); + assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Test @@ -101,14 +121,12 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test - public void testSuggestionNotNull() throws Exception { + public void testSuggestionNotNull() { //todo write a real test - assertTrue(extractor.getSearchSuggestion() != null); + assertNotNull(extractor.getSearchSuggestion()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 8777cc70..fc6af4c4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -11,11 +11,11 @@ public class YoutubeSearchQHTest { @Test public void testRegularValues() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); - assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); - assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); - assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 2798e8db..333d0ee0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -10,17 +10,24 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -89,7 +96,6 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetFullLinksInDescription() throws ParsingException { assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); - assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test @@ -142,12 +148,12 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetAudioStreams() throws IOException, ExtractionException { + public void testGetAudioStreams() throws ExtractionException { assertFalse(extractor.getAudioStreams().isEmpty()); } @Test - public void testGetVideoStreams() throws IOException, ExtractionException { + public void testGetVideoStreams() throws ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { assertIsSecureUrl(s.url); assertTrue(s.resolution.length() > 0); @@ -169,7 +175,7 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -177,13 +183,13 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } @@ -223,10 +229,6 @@ public class YoutubeStreamExtractorDefaultTest { assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); - - assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub...")); - assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8...")); - assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/...")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index b1d6f53a..f5c7fe10 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -13,9 +13,12 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -26,7 +29,7 @@ public class YoutubeStreamExtractorLivestreamTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY"); + .getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A"); extractor.fetchPage(); } @@ -49,8 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/")); - assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba...")); + assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists")); } @Test @@ -119,7 +121,7 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -127,12 +129,12 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } From 8d503407ef32fe1cac5359f042e909ba4878a807 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 19:37:58 +0100 Subject: [PATCH 091/149] Parse browseEndpoint and watchEndpoint in descriptions --- .../extractors/YoutubeStreamExtractor.java | 20 ++++++++++++++++++- .../YoutubeStreamExtractorDefaultTest.java | 15 ++++++-------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8a725593..79e50e4b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -229,7 +229,25 @@ public class YoutubeStreamExtractor extends StreamExtractor { descriptionBuilder.append("").append(text).append(""); htmlConversionRequired = true; } - continue; + } else if (textHolder.getObject("navigationEndpoint").getObject("browseEndpoint") != null) { + descriptionBuilder.append("").append(text).append(""); + htmlConversionRequired = true; + } else if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint") != null) { + if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint").getString("playlistId") != null) { + descriptionBuilder.append("").append(text).append(""); + } else { + descriptionBuilder.append("").append(text).append(""); + } + htmlConversionRequired = true; } continue; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 333d0ee0..cb2b768d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -245,21 +245,18 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { + System.out.println(extractor.getDescription().getContent()); assertNotNull(extractor.getDescription()); assertFalse(extractor.getDescription().getContent().isEmpty()); } @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); + final String description = extractor.getDescription().getContent(); + assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); } } From a2d8489e169ce43d9d3646c9ccb21d56e7f79b2b Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 20:27:30 +0100 Subject: [PATCH 092/149] Parse timestamps in description --- .../extractors/YoutubeStreamExtractor.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 79e50e4b..24f16c6d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -235,18 +235,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getString("canonicalBaseUrl")).append("\">").append(text).append(""); htmlConversionRequired = true; } else if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint") != null) { + descriptionBuilder.append("").append(text).append(""); - } else { - descriptionBuilder.append("").append(text).append(""); + descriptionBuilder.append("&list=").append(textHolder.getObject("navigationEndpoint") + .getObject("watchEndpoint").getString("playlistId")); } + if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint").has("startTimeSeconds")) { + descriptionBuilder.append("&t=").append(textHolder.getObject("navigationEndpoint") + .getObject("watchEndpoint").getInt("startTimeSeconds")); + } + descriptionBuilder.append("\">").append(text).append(""); htmlConversionRequired = true; } continue; From db305408e906614fbdff44d963fc94aed1363c9c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 26 Feb 2020 20:37:20 +0100 Subject: [PATCH 093/149] Get all expandedShelfContentsRenderers in YoutubeTrendingExtractor --- .../extractors/YoutubeTrendingExtractor.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 0b1667cd..af1a91b3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -108,17 +108,21 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray firstPageElements = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") - .getObject("sectionListRenderer").getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") - .getObject("expandedShelfContentsRenderer").getArray("items"); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); + JsonArray itemSectionRenderers = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents"); - for (Object ul : firstPageElements) { - final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); - collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); + for (Object itemSectionRenderer : itemSectionRenderers) { + JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") + .getObject("expandedShelfContentsRenderer"); + if (expandedShelfContentsRenderer != null) { + for (Object ul : expandedShelfContentsRenderer.getArray("items")) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); + } + } } return new InfoItemsPage<>(collector, getNextPageUrl()); From cdbc751b874a5f096a885a4f09b45efe6bfb403a Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 09:05:41 +0100 Subject: [PATCH 094/149] Fix getUploaderUrl() in YoutubeStreamInfoItemExtractor --- .../youtube/extractors/YoutubeStreamInfoItemExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 0f9c1c50..4e40034b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -159,7 +159,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { if (id == null || id.isEmpty()) { throw new IllegalArgumentException("is empty"); } - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + id); } catch (Exception e) { throw new ParsingException("Could not get uploader url"); } From 2399d46ab039c8e07b27885e3240cb8087e0ef57 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 09:21:25 +0100 Subject: [PATCH 095/149] Fix YoutubeChannelExtractor for channels without videos --- .../extractors/YoutubeChannelExtractor.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 41b0de1d..b6c9bb95 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -85,6 +85,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getNextPageUrl() throws ExtractionException { + if (getVideoTab() == null) return ""; return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer") .getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations")); @@ -195,10 +196,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0) - .getObject("gridRenderer").getArray("items"); - collectStreamsFrom(collector, videos); + if (getVideoTab() != null) { + JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("gridRenderer").getArray("items"); + collectStreamsFrom(collector, videos); + } return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -241,9 +244,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null) { - return ""; - } + if (continuations == null) return ""; JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); @@ -295,6 +296,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ParsingException("Could not find Videos tab"); } + try { + if (videoTab.getObject("content").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("contents") + .getObject(0).getObject("messageRenderer").getObject("text").getArray("runs") + .getObject(0).getString("text").equals("This channel has no videos.")) + return null; + } catch (Exception ignored) {} + return videoTab; } } From 951ed9f0d85b5a526b40d5883c676fb8069ea312 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 09:27:56 +0100 Subject: [PATCH 096/149] Don't call getUploaderInfo() in onFetchPage() for playlists This way YouTube Music playlists will keep working in NewPipe --- .../youtube/extractors/YoutubePlaylistExtractor.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 84af2146..240fa537 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -28,7 +28,6 @@ import javax.annotation.Nonnull; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { private JsonObject initialData; - private JsonObject uploaderInfo; private JsonObject playlistInfo; public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -57,7 +56,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } initialData = ajaxJson.getObject(1).getObject("response"); - uploaderInfo = getUploaderInfo(); playlistInfo = getPlaylistInfo(); } @@ -140,7 +138,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public String getUploaderUrl() throws ParsingException { try { return YoutubeChannelExtractor.CHANNEL_URL_BASE + - uploaderInfo.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + getUploaderInfo().getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } @@ -149,7 +147,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderName() throws ParsingException { try { - return uploaderInfo.getObject("title").getArray("runs").getObject(0).getString("text"); + return getUploaderInfo().getObject("title").getArray("runs").getObject(0).getString("text"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader name", e); } @@ -158,7 +156,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { try { - return uploaderInfo.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + return getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader avatar", e); } From 8aa560bef4c23f5c0cb8c543a50a30226784c8ef Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 09:42:22 +0100 Subject: [PATCH 097/149] Fix video titles consisting of multiple parts due to @ --- .../youtube/extractors/YoutubeStreamExtractor.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 24f16c6d..9f52386d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -114,7 +114,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String title = null; try { - title = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs").getObject(0).getString("text"); + StringBuilder titleBuilder = new StringBuilder(); + JsonArray titleArray = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs"); + for (Object titlePart : titleArray) { + titleBuilder.append(((JsonObject) titlePart).getString("text")); + } + title = titleBuilder.toString(); } catch (Exception ignored) {} if (title == null) { try { From 880b9510880c9809dbd23e232bdbdb31d827d75c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 09:53:59 +0100 Subject: [PATCH 098/149] Fix getUploaderAvatarUrl() in YoutubeStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 9f52386d..e9228367 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -435,27 +435,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { assertPageFetched(); - - String uploaderAvatarUrl = null; try { - uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") - .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") - .getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail") - .getArray("thumbnails").getObject(0).getString("url"); - if (uploaderAvatarUrl != null && !uploaderAvatarUrl.isEmpty()) { - return uploaderAvatarUrl; - } - } catch (Exception ignored) {} - - try { - uploaderAvatarUrl = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") + return getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} - - if (uploaderAvatarUrl == null) { - throw new ParsingException("Could not get uploader avatar url"); + } catch (Exception e) { + throw new ParsingException("Could not get uploader avatar url", e); } - return uploaderAvatarUrl; } @Nonnull From 8ebd971648729e397f63c4c24ca7ccc523d4ec94 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 10:06:35 +0100 Subject: [PATCH 099/149] Fix channel descriptions consisting of multiple parts --- .../extractors/YoutubeChannelInfoItemExtractor.java | 7 ++++++- .../youtube/extractors/YoutubeStreamExtractor.java | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 483cd894..518d8ea0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; @@ -97,7 +98,11 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getDescription() throws ParsingException { try { - return channelInfoItem.getObject("descriptionSnippet").getArray("runs").getObject(0).getString("text"); + StringBuilder description = new StringBuilder(); + JsonArray descriptionArray = channelInfoItem.getObject("descriptionSnippet").getArray("runs"); + for (Object descriptionPart : descriptionArray) + description.append(((JsonObject) descriptionPart).getString("text")); + return description.toString(); } catch (Exception e) { throw new ParsingException("Could not get description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e9228367..d6a3c943 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -116,9 +116,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { StringBuilder titleBuilder = new StringBuilder(); JsonArray titleArray = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs"); - for (Object titlePart : titleArray) { + for (Object titlePart : titleArray) titleBuilder.append(((JsonObject) titlePart).getString("text")); - } title = titleBuilder.toString(); } catch (Exception ignored) {} if (title == null) { From 0798bdd5cd03bb44c0593a28b9c23df472e762ef Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 16:26:22 +0100 Subject: [PATCH 100/149] Fix getSubscriberCount() for 0 subscribers --- .../youtube/extractors/YoutubeChannelExtractor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index b6c9bb95..2d2fe917 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -177,8 +177,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ParsingException("Could not get subscriber count", e); } } else { - // If the element is null, the channel have the subscriber count disabled - return -1; + // If there's no subscribe button, the channel has the subscriber count disabled + if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) { + return -1; + } else { + return 0; + } } } From 365b0329f3dfb0dba257d1399624b5e9a4a56822 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 17:39:23 +0100 Subject: [PATCH 101/149] Implement getTextFromObject() function --- .../extractors/YoutubeChannelExtractor.java | 25 +++- .../YoutubeChannelInfoItemExtractor.java | 17 +-- .../extractors/YoutubePlaylistExtractor.java | 28 +++- .../YoutubePlaylistInfoItemExtractor.java | 6 +- .../extractors/YoutubeSearchExtractor.java | 9 +- .../extractors/YoutubeStreamExtractor.java | 132 +++++------------- .../YoutubeStreamInfoItemExtractor.java | 64 +++------ .../extractors/YoutubeTrendingExtractor.java | 5 +- .../linkHandler/YoutubeParsingHelper.java | 68 +++++++++ .../youtube/YoutubePlaylistExtractorTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 1 - 11 files changed, 185 insertions(+), 172 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 2d2fe917..4b52bee0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @@ -124,8 +125,20 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { try { - return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") + String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") .getArray("thumbnails").getObject(0).getString("url"); + + // the first characters of the avatar URLs are different for each channel and some are not even valid URLs + if (url.startsWith("//")) { + url = url.substring(2); + } + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + + return url; } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -172,7 +185,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); if (subscriberInfo != null) { try { - return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text")); + return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo)); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } @@ -301,10 +314,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } try { - if (videoTab.getObject("content").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("contents") - .getObject(0).getObject("messageRenderer").getObject("text").getArray("runs") - .getObject(0).getString("text").equals("This channel has no videos.")) + if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("messageRenderer") + .getObject("text")).equals("This channel has no videos.")) return null; } catch (Exception ignored) {} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 518d8ea0..e82c4377 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; @@ -8,6 +7,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @@ -59,7 +59,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getName() throws ParsingException { try { - return channelInfoItem.getObject("title").getString("simpleText"); + return getTextFromObject(channelInfoItem.getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get name", e); } @@ -68,7 +68,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getUrl() throws ParsingException { try { - String id = "channel/" + channelInfoItem.getString("channelId"); // Does prepending 'channel/' always work? + String id = "channel/" + channelInfoItem.getString("channelId"); return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); } catch (Exception e) { throw new ParsingException("Could not get url", e); @@ -78,7 +78,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getSubscriberCount() throws ParsingException { try { - String subscribers = channelInfoItem.getObject("subscriberCountText").getString("simpleText").split(" ")[0]; + String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText")); return Utils.mixedNumberWordToLong(subscribers); } catch (Exception e) { throw new ParsingException("Could not get subscriber count", e); @@ -88,8 +88,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getStreamCount() throws ParsingException { try { - return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText") - .getArray("runs").getObject(0).getString("text"))); + return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText")))); } catch (Exception e) { throw new ParsingException("Could not get stream count", e); } @@ -98,11 +97,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getDescription() throws ParsingException { try { - StringBuilder description = new StringBuilder(); - JsonArray descriptionArray = channelInfoItem.getObject("descriptionSnippet").getArray("runs"); - for (Object descriptionPart : descriptionArray) - description.append(((JsonObject) descriptionPart).getString("text")); - return description.toString(); + return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); } catch (Exception e) { throw new ParsingException("Could not get description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 240fa537..e32b42ef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -25,6 +25,11 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; + @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { private JsonObject initialData; @@ -104,7 +109,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { try { - String name = playlistInfo.getObject("title").getArray("runs").getObject(0).getString("text"); + String name = getTextFromObject(playlistInfo.getObject("title")); if (name != null) return name; } catch (Exception ignored) {} try { @@ -137,8 +142,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() throws ParsingException { try { - return YoutubeChannelExtractor.CHANNEL_URL_BASE + - getUploaderInfo().getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint")); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } @@ -147,7 +151,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderName() throws ParsingException { try { - return getUploaderInfo().getObject("title").getArray("runs").getObject(0).getString("text"); + return getTextFromObject(getUploaderInfo().getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader name", e); } @@ -156,7 +160,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { try { - return getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + + // the first characters of the avatar URLs are different for each channel and some are not even valid URLs + if (url.startsWith("//")) { + url = url.substring(2); + } + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + + return url; } catch (Exception e) { throw new ParsingException("Could not get playlist uploader avatar", e); } @@ -165,7 +181,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public long getStreamCount() throws ParsingException { try { - String viewsText = getPlaylistInfo().getArray("stats").getObject(0).getArray("runs").getObject(0).getString("text"); + String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0)); return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); } catch (Exception e) { throw new ParsingException("Could not get video count from playlist", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 358fa2e6..03bf9b22 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -7,6 +7,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private JsonObject playlistInfoItem; @@ -27,7 +29,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getName() throws ParsingException { try { - return playlistInfoItem.getObject("title").getString("simpleText"); + return getTextFromObject(playlistInfoItem.getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get name", e); } @@ -46,7 +48,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getUploaderName() throws ParsingException { try { - return playlistInfoItem.getObject("longBylineText").getArray("runs").getObject(0).getString("text"); + return getTextFromObject(playlistInfoItem.getObject("longBylineText")); } catch (Exception e) { throw new ParsingException("Could not get uploader name", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 7ec0e5cc..bdc46d19 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -24,6 +24,8 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + /* * Created by Christian Schabesberger on 22.07.2018 * @@ -91,8 +93,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { if (showingResultsForRenderer == null) { return ""; } else { - return showingResultsForRenderer.getObject("correctedQuery").getArray("runs") - .getObject(0).getString("text"); + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } } @@ -155,8 +156,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { for (Object item : videos) { if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { - throw new NothingFoundException(((JsonObject) item).getObject("backgroundPromoRenderer") - .getObject("bodyText").getArray("runs").getObject(0).getString("text")); + throw new NothingFoundException(getTextFromObject(((JsonObject) item) + .getObject("backgroundPromoRenderer").getObject("bodyText"))); } else if (((JsonObject) item).getObject("videoRenderer") != null) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); } else if (((JsonObject) item).getObject("channelRenderer") != null) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index d6a3c943..e985d1b9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -40,8 +40,6 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; @@ -56,6 +54,10 @@ import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; + /* * Created by Christian Schabesberger on 06.08.15. * @@ -114,11 +116,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String title = null; try { - StringBuilder titleBuilder = new StringBuilder(); - JsonArray titleArray = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs"); - for (Object titlePart : titleArray) - titleBuilder.append(((JsonObject) titlePart).getString("text")); - title = titleBuilder.toString(); + title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); } catch (Exception ignored) {} if (title == null) { try { @@ -146,8 +144,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} try { - if (getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").startsWith("Premiered")) { - String time = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10); + if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { + String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); try { // Premiered 20 hours ago TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); @@ -165,7 +163,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { // TODO this parses English formatted dates only, we need a better approach to parse the textual date Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( - getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText")); + getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); return new SimpleDateFormat("yyyy-MM-dd").format(d); } catch (Exception ignored) {} throw new ParsingException("Could not get upload date"); @@ -203,73 +201,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); // description with more info on links try { - boolean htmlConversionRequired = false; - JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs"); - StringBuilder descriptionBuilder = new StringBuilder(descriptions.size()); - for (Object textObjectHolder : descriptions) { - JsonObject textHolder = (JsonObject) textObjectHolder; - String text = textHolder.getString("text"); - if (textHolder.getObject("navigationEndpoint") != null) { - // The text is a link. Get the URL it points to and generate a HTML link of it - if (textHolder.getObject("navigationEndpoint").getObject("urlEndpoint") != null) { - String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url"); - if (internUrl.startsWith("/redirect?")) { - // q parameter can be the first parameter - internUrl = internUrl.substring(10); - String[] params = internUrl.split("&"); - for (String param : params) { - if (param.split("=")[0].equals("q")) { - String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); - if (url != null && !url.isEmpty()) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } else { - descriptionBuilder.append(text); - } - break; - } - } - } else if (internUrl.startsWith("http")) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } - } else if (textHolder.getObject("navigationEndpoint").getObject("browseEndpoint") != null) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } else if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint") != null) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } - continue; - } - if (text != null) { - descriptionBuilder.append(text); - } - } - - String description = descriptionBuilder.toString(); - - if (!description.isEmpty()) { - if (htmlConversionRequired) { - description = description.replaceAll("\\n", "
    "); - description = description.replaceAll(" ", "  "); - return new Description(description, Description.HTML); - } - return new Description(description, Description.PLAIN_TEXT); - } + String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); + return new Description(description, Description.HTML); } catch (Exception ignored) { } // raw non-html description @@ -329,16 +262,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String views = null; try { - views = getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount") - .getArray("runs").getObject(0).getString("text"); + views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") + .getObject("videoViewCountRenderer").getObject("viewCount")); } catch (Exception ignored) {} - if (views == null) { - try { - views = getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount").getString("simpleText"); - } catch (Exception ignored) {} - } if (views == null) { try { views = playerResponse.getObject("videoDetails").getString("viewCount"); @@ -398,17 +324,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderUrl() throws ParsingException { assertPageFetched(); - String uploaderId = null; try { - uploaderId = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") - .getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer() + .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint")); + if (uploaderUrl != null) return uploaderUrl; + } catch (Exception ignored) {} + try { + String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); + if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId; } catch (Exception ignored) {} - if (uploaderId == null) { - try { - uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); - } catch (Exception ignored) {} - } - if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId; throw new ParsingException("Could not get uploader url"); } @@ -418,8 +342,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); String uploaderName = null; try { - uploaderName = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") - .getObject("title").getArray("runs").getObject(0).getString("text"); + uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") + .getObject("videoOwnerRenderer").getObject("title")); } catch (Exception ignored) {} if (uploaderName == null) { try { @@ -435,8 +359,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderAvatarUrl() throws ParsingException { assertPageFetched(); try { - return getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") + String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + + // the first characters of the avatar URLs are different for each channel and some are not even valid URLs + if (url.startsWith("//")) { + url = url.substring(2); + } + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + + return url; } catch (Exception e) { throw new ParsingException("Could not get uploader avatar url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 4e40034b..d3cee8b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -6,7 +6,6 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; @@ -15,6 +14,9 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; + /* * Copyright (C) Christian Schabesberger 2016 * YoutubeStreamInfoItemExtractor.java is part of NewPipe. @@ -76,15 +78,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getName() throws ParsingException { - String name = null; - try { - name = videoInfo.getObject("title").getString("simpleText"); - } catch (Exception ignored) {} - if (name == null) { - try { - name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text"); - } catch (Exception ignored) {} - } + String name = getTextFromObject(videoInfo.getObject("title")); if (name != null && !name.isEmpty()) return name; throw new ParsingException("Could not get name"); } @@ -94,14 +88,14 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { if (getStreamType() == StreamType.LIVE_STREAM) return -1; String duration = null; try { - duration = videoInfo.getObject("lengthText").getString("simpleText"); + duration = getTextFromObject(videoInfo.getObject("lengthText")); } catch (Exception ignored) {} if (duration == null) { try { for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) { - duration = ((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") - .getObject("text").getString("simpleText"); + duration = getTextFromObject(((JsonObject) thumbnailOverlay) + .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); } } } catch (Exception ignored) {} @@ -114,19 +108,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { public String getUploaderName() throws ParsingException { String name = null; try { - name = videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getString("text"); + name = getTextFromObject(videoInfo.getObject("longBylineText")); } catch (Exception ignored) {} if (name == null) { try { - name = videoInfo.getObject("ownerText").getArray("runs") - .getObject(0).getString("text"); + name = getTextFromObject(videoInfo.getObject("ownerText")); } catch (Exception ignored) {} } if (name == null) { try { - name = videoInfo.getObject("shortBylineText").getArray("runs") - .getObject(0).getString("text"); + name = getTextFromObject(videoInfo.getObject("shortBylineText")); } catch (Exception ignored) {} } if (name != null && !name.isEmpty()) return name; @@ -136,30 +127,27 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getUploaderUrl() throws ParsingException { try { - String id = null; + String url = null; try { - id = videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); + url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); } catch (Exception ignored) {} - if (id == null) { + if (url == null) { try { - id = videoInfo.getObject("ownerText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); + url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); } catch (Exception ignored) {} } - if (id == null) { + if (url == null) { try { - id = videoInfo.getObject("shortBylineText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); + url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); } catch (Exception ignored) {} } - if (id == null || id.isEmpty()) { + if (url == null || url.isEmpty()) { throw new IllegalArgumentException("is empty"); } - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + id); + return url; } catch (Exception e) { throw new ParsingException("Could not get uploader url"); } @@ -169,7 +157,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getTextualUploadDate() { try { - return videoInfo.getObject("publishedTimeText").getString("simpleText"); + return getTextFromObject(videoInfo.getObject("publishedTimeText")); } catch (Exception e) { // upload date is not always available, e.g. in playlists return null; @@ -196,13 +184,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { return -1; } - String viewCount; - if (getStreamType() == StreamType.LIVE_STREAM) { - viewCount = videoInfo.getObject("viewCountText") - .getArray("runs").getObject(0).getString("text"); - } else { - viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); - } + String viewCount = getTextFromObject(videoInfo.getObject("viewCountText")); if (viewCount.equals("Recommended for you")) return -1; return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index af1a91b3..25aaa4db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -44,6 +44,8 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData; @@ -93,8 +95,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { public String getName() throws ParsingException { String name; try { - name = initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title") - .getArray("runs").getObject(0).getString("text"); + name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get Trending name", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 24edef60..63e20424 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -13,7 +13,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.Parser; +import java.io.UnsupportedEncodingException; import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -251,4 +254,69 @@ public class YoutubeParsingHelper { throw new ParsingException("Could not get client version"); } + + public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) { + if (navigationEndpoint.getObject("urlEndpoint") != null) { + String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + String[] params = internUrl.split("&"); + for (String param : params) { + if (param.split("=")[0].equals("q")) { + String url; + try { + url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + return null; + } + return url; + } + } + } else if (internUrl.startsWith("http")) { + return internUrl; + } + } else if (navigationEndpoint.getObject("browseEndpoint") != null) { + return "https://www.youtube.com" + navigationEndpoint.getObject("browseEndpoint").getString("canonicalBaseUrl"); + } else if (navigationEndpoint.getObject("watchEndpoint") != null) { + StringBuilder url = new StringBuilder(); + url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + return url.toString(); + } + return null; + } + + public static String getTextFromObject(JsonObject textObject, boolean html) { + if (textObject.has("simpleText")) return textObject.getString("simpleText"); + + StringBuilder textBuilder = new StringBuilder(); + for (Object textPart : textObject.getArray("runs")) { + String text = ((JsonObject) textPart).getString("text"); + if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) { + String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) { + textBuilder.append("").append(text).append(""); + continue; + } + } + textBuilder.append(text); + } + + String text = textBuilder.toString(); + + if (html) { + text = text.replaceAll("\\n", "
    "); + text = text.replaceAll(" ", "  "); + } + + return text; + } + + public static String getTextFromObject(JsonObject textObject) { + return getTextFromObject(textObject, false); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index d4de9175..7c911279 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -99,7 +99,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testUploaderUrl() throws Exception { - assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); + assertEquals("https://www.youtube.com/user/andre0y0you", extractor.getUploaderUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index cb2b768d..d6cf3815 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -245,7 +245,6 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDescription() throws ParsingException { - System.out.println(extractor.getDescription().getContent()); assertNotNull(extractor.getDescription()); assertFalse(extractor.getDescription().getContent().isEmpty()); } From 838d834ce217037a6c3212458356053b289cc316 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 18:54:27 +0100 Subject: [PATCH 102/149] Get Uploads view of Videos tab --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 4b52bee0..452cfbbf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -61,7 +61,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = super.getUrl() + "/videos?pbj=1"; + final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; JsonArray ajaxJson; From 94a2d5be463704dc551b1dca330110c36846ea7c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Thu, 27 Feb 2020 19:08:46 +0100 Subject: [PATCH 103/149] Fix thumbnail URLs starting with // --- .../extractors/YoutubeChannelExtractor.java | 9 +----- .../YoutubeChannelInfoItemExtractor.java | 5 ++-- .../extractors/YoutubePlaylistExtractor.java | 30 ++++++++++++------- .../YoutubePlaylistInfoItemExtractor.java | 14 +++++++-- .../extractors/YoutubeStreamExtractor.java | 13 ++++---- .../YoutubeStreamInfoItemExtractor.java | 12 +++++++- 6 files changed, 54 insertions(+), 29 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 452cfbbf..849953cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -128,10 +128,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor { String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") .getArray("thumbnails").getObject(0).getString("url"); - // the first characters of the avatar URLs are different for each channel and some are not even valid URLs - if (url.startsWith("//")) { - url = url.substring(2); - } if (url.startsWith(HTTP)) { url = Utils.replaceHttpWithHttps(url); } else if (!url.startsWith(HTTPS)) { @@ -155,10 +151,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; } - // the first characters of the banner URLs are different for each channel and some are not even valid URLs - if (url.startsWith("//")) { - url = url.substring(2); - } + if (url.startsWith(HTTP)) { url = Utils.replaceHttpWithHttps(url); } else if (!url.startsWith(HTTPS)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index e82c4377..6a571daa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -42,14 +42,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor public String getThumbnailUrl() throws ParsingException { try { String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - if (url.startsWith("//")) { - url = url.substring(2); - } + if (url.startsWith(HTTP)) { url = Utils.replaceHttpWithHttps(url); } else if (!url.startsWith(HTTPS)) { url = HTTPS + url; } + return url; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index e32b42ef..66768b9c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -121,16 +121,30 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { + String url = null; + try { - return playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") + url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception ignored) {} - try { - return initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") - .getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception e) { - throw new ParsingException("Could not get playlist thumbnail", e); + + if (url == null) { + try { + return initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") + .getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception ignored) {} } + + if (url != null && !url.isEmpty()) { + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + + return url; + } + throw new ParsingException("Could not get playlist thumbnail"); } @Override @@ -162,10 +176,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { try { String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - // the first characters of the avatar URLs are different for each channel and some are not even valid URLs - if (url.startsWith("//")) { - url = url.substring(2); - } if (url.startsWith(HTTP)) { url = Utils.replaceHttpWithHttps(url); } else if (!url.startsWith(HTTPS)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 03bf9b22..031b3737 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylist import org.schabi.newpipe.extractor.utils.Utils; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private JsonObject playlistInfoItem; @@ -19,8 +21,16 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getThumbnailUrl() throws ParsingException { try { - return playlistInfoItem.getArray("thumbnails").getObject(0).getArray("thumbnails") - .getObject(0).getString("url"); + String url = playlistInfoItem.getArray("thumbnails").getObject(0) + .getArray("thumbnails").getObject(0).getString("url"); + + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + + return url; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e985d1b9..144a7221 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -187,8 +187,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - return thumbnails.getObject(thumbnails.size() - 1).getString("url"); + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + + return url; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url"); } @@ -362,10 +369,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - // the first characters of the avatar URLs are different for each channel and some are not even valid URLs - if (url.startsWith("//")) { - url = url.substring(2); - } if (url.startsWith(HTTP)) { url = Utils.replaceHttpWithHttps(url); } else if (!url.startsWith(HTTPS)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index d3cee8b0..da22179a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -16,6 +16,8 @@ import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; /* * Copyright (C) Christian Schabesberger 2016 @@ -196,8 +198,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { public String getThumbnailUrl() throws ParsingException { try { // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return videoInfo.getObject("thumbnail").getArray("thumbnails") + String url = videoInfo.getObject("thumbnail").getArray("thumbnails") .getObject(0).getString("url"); + + if (url.startsWith(HTTP)) { + url = Utils.replaceHttpWithHttps(url); + } else if (!url.startsWith(HTTPS)) { + url = HTTPS + url; + } + + return url; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } From f946310af590f5ef9d428cadd59113380d971d03 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 08:57:44 +0100 Subject: [PATCH 104/149] Fix search queries with a promotedSparklesTextSearchRenderer returning 0 results --- .../youtube/extractors/YoutubeSearchExtractor.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index bdc46d19..222c411f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -101,11 +101,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - JsonArray videos = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("contents"); + JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); + + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); + } - collectStreamsFrom(collector, videos); return new InfoItemsPage<>(collector, getNextPageUrl()); } From 612ec06839bfef5db6616f99aba81e243bc6cd14 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 09:36:33 +0100 Subject: [PATCH 105/149] Actually fix thumbnail URLs starting with // --- .../extractors/YoutubeChannelExtractor.java | 19 +++---------------- .../YoutubeChannelInfoItemExtractor.java | 11 ++--------- .../extractors/YoutubePlaylistExtractor.java | 19 +++---------------- .../YoutubePlaylistInfoItemExtractor.java | 11 ++--------- .../extractors/YoutubeStreamExtractor.java | 18 +++--------------- .../YoutubeStreamInfoItemExtractor.java | 11 ++--------- .../linkHandler/YoutubeParsingHelper.java | 17 +++++++++++++++++ 7 files changed, 32 insertions(+), 74 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 849953cc..7b3f5b45 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -25,9 +25,8 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; /* * Created by Christian Schabesberger on 25.07.16. @@ -128,13 +127,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") .getArray("thumbnails").getObject(0).getString("url"); - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -152,13 +145,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return null; } - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get banner", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 6a571daa..29aa045b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -7,9 +7,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; /* * Created by Christian Schabesberger on 12.02.17. @@ -43,13 +42,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor try { String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 66768b9c..fdd75868 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -25,10 +25,9 @@ import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { @@ -136,13 +135,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } if (url != null && !url.isEmpty()) { - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } throw new ParsingException("Could not get playlist thumbnail"); } @@ -176,13 +169,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { try { String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader avatar", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 031b3737..5b4e7afc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -7,9 +7,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private JsonObject playlistInfoItem; @@ -24,13 +23,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract String url = playlistInfoItem.getArray("thumbnails").getObject(0) .getArray("thumbnails").getObject(0).getString("url"); - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 144a7221..a0535568 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -54,9 +54,9 @@ import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; /* * Created by Christian Schabesberger on 06.08.15. @@ -189,13 +189,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { // the last thumbnail is the one with the highest resolution String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url"); } @@ -369,13 +363,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get uploader avatar url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index da22179a..e981071c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -14,10 +14,9 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; /* * Copyright (C) Christian Schabesberger 2016 @@ -201,13 +200,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { String url = videoInfo.getObject("thumbnail").getArray("thumbnails") .getObject(0).getString("url"); - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 63e20424..0061d6d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.UnsupportedEncodingException; import java.net.URL; @@ -27,6 +28,8 @@ import java.util.List; import java.util.Map; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; /* * Created by Christian Schabesberger on 02.03.16. @@ -319,4 +322,18 @@ public class YoutubeParsingHelper { public static String getTextFromObject(JsonObject textObject) { return getTextFromObject(textObject, false); } + + public static String fixThumbnailUrl(String thumbnailUrl) { + if (thumbnailUrl.startsWith("//")) { + thumbnailUrl = thumbnailUrl.substring(2); + } + + if (thumbnailUrl.startsWith(HTTP)) { + thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl); + } else if (!thumbnailUrl.startsWith(HTTPS)) { + thumbnailUrl = "https://" + thumbnailUrl; + } + + return thumbnailUrl; + } } From 7898ba230d1ce192be343b16afc2e3d3dfb88ae0 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 15:17:47 +0100 Subject: [PATCH 106/149] Use pbj in YoutubeStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 93 ++++++------------- 1 file changed, 30 insertions(+), 63 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a0535568..e491c7aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,9 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -13,8 +12,6 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -91,7 +88,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { /*//////////////////////////////////////////////////////////////////////////*/ - private Document doc; + private JsonArray initialAjaxJson; @Nullable private JsonObject playerArgs; @Nonnull @@ -554,23 +551,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public String getErrorMessage() { - StringBuilder errorReason; - Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first(); - - if (errorElement == null) { - errorReason = null; - } else { - String errorMessage = errorElement.text(); - if (errorMessage == null || errorMessage.isEmpty()) { - errorReason = null; - } else { - errorReason = new StringBuilder(errorMessage); - errorReason.append(" "); - errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); - } - } - - return errorReason != null ? errorReason.toString() : ""; + return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus") + .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason")); } /*////////////////////////////////////////////////////////////////////////// @@ -580,11 +562,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; - private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; - private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; - private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 = @@ -596,29 +575,42 @@ public class YoutubeStreamExtractor extends StreamExtractor { private volatile String decryptionCode = ""; - private String pageHtml = null; - @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; - final Response response = downloader.get(verifiedUrl, getExtractorLocalization()); - pageHtml = response.responseBody(); - doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response); + final String url = getUrl() + "&pbj=1"; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); + final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("Could not parse json data for next streams"); + } + + try { + initialAjaxJson = JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for next streams", e); + } final String playerUrl; - initialData = YoutubeParsingHelper.getInitialData(pageHtml); - // Check if the video is age restricted - if (getAgeLimit() == 18) { + + if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos + initialData = initialAjaxJson.getObject(2).getObject("response"); + final EmbeddedInfo info = getEmbeddedInfo(); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); playerUrl = info.url; } else { - final JsonObject ytPlayerConfig = getPlayerConfig(); - playerArgs = getPlayerArgs(ytPlayerConfig); - playerUrl = getPlayerUrl(ytPlayerConfig); + initialData = initialAjaxJson.getObject(3).getObject("response"); + + playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player")); + playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player")); } + playerResponse = getPlayerResponse(); if (decryptionCode.isEmpty()) { @@ -630,21 +622,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JsonObject getPlayerConfig() throws ParsingException { - try { - String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml); - return JsonParser.object().from(ytPlayerConfigRaw); - } catch (Parser.RegexException e) { - String errorReason = getErrorMessage(); - if (errorReason.isEmpty()) { - throw new ContentNotAvailableException("Content not available: player config empty", e); - } - throw new ContentNotAvailableException("Content not available", e); - } catch (Exception e) { - throw new ParsingException("Could not parse yt player config", e); - } - } - private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException { JsonObject playerArgs; @@ -950,17 +927,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public List getFrames() throws ExtractionException { try { - final String script = doc.select("#player-api").first().siblingElements().select("script").html(); - int p = script.indexOf("ytplayer.config"); - if (p == -1) { - return Collections.emptyList(); - } - p = script.indexOf('{', p); - int e = script.indexOf("ytplayer.load", p); - if (e == -1) { - return Collections.emptyList(); - } - JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); + JsonObject jo = initialAjaxJson.getObject(2).getObject("player"); final String resp = jo.getObject("args").getString("player_response"); jo = JsonParser.object().from(resp); final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); From b89b6b1db229c522fd1eebbb6aed60ee41e8fc4d Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 16:35:24 +0100 Subject: [PATCH 107/149] Check if hard-coded client version is valid in separate function --- .../linkHandler/YoutubeParsingHelper.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 0061d6d0..5a33e4ce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -175,15 +175,7 @@ public class YoutubeParsingHelper { } } - /** - * Get the client version from a page - * @return - * @throws ParsingException - */ - public static String getClientVersion() throws ParsingException { - if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - - // Test if hard-coded client version is valid + public static boolean isHardcodedClientVersionValid() { try { final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; @@ -193,11 +185,26 @@ public class YoutubeParsingHelper { Collections.singletonList(HARDCODED_CLIENT_VERSION)); final String response = getDownloader().get(url, headers).responseBody(); if (response.length() > 50) { // ensure to have a valid response - clientVersion = HARDCODED_CLIENT_VERSION; - return clientVersion; + return true; } } catch (Exception ignored) {} + return false; + } + + /** + * Get the client version from a page + * @return + * @throws ParsingException + */ + public static String getClientVersion() throws ParsingException { + if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; + + if (isHardcodedClientVersionValid()) { + clientVersion = HARDCODED_CLIENT_VERSION; + return clientVersion; + } + // Try extracting it from YouTube's website otherwise try { final String url = "https://www.youtube.com/results?search_query=test"; From beb07a4852fc27c65ff2db56270085dd663a8012 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 16:36:14 +0100 Subject: [PATCH 108/149] Add test to check whether hardcoded client version is still valid --- .../youtube/YoutubeParsingHelperTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java new file mode 100644 index 00000000..06eedaef --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -0,0 +1,22 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; + +import static org.junit.Assert.assertTrue; + +public class YoutubeParsingHelperTest { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testIsHardcodedClientVersionValid() { + assertTrue("Hardcoded client version is not valid anymore", + YoutubeParsingHelper.isHardcodedClientVersionValid()); + } +} From 0b940f5cf47adbefc1f04df5d6232ef24b1cb895 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 16:40:50 +0100 Subject: [PATCH 109/149] Use "getAgeLimit() != NO_AGE_LIMIT" instead of "getAgeLimit() == 18" --- .../youtube/extractors/YoutubeStreamExtractor.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e491c7aa..e59c89a7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -504,9 +504,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItem getNextStream() throws ExtractionException { assertPageFetched(); - if (getAgeLimit() == 18) { - return null; - } + + if (getAgeLimit() != NO_AGE_LIMIT) return null; + try { final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults").getObject("secondaryResults").getArray("results") @@ -525,9 +525,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); - if (getAgeLimit() == 18) { - return null; - } + + if (getAgeLimit() != NO_AGE_LIMIT) return null; + try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") @@ -773,7 +773,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private List getAvailableSubtitlesInfo() { // If the video is age restricted getPlayerConfig will fail - if (getAgeLimit() == 18) return Collections.emptyList(); + if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList(); final JsonObject captions; if (!playerResponse.has("captions")) { From 8bfbae904d4b16ec3cc412f9877e725571094a66 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 17:03:21 +0100 Subject: [PATCH 110/149] Cache getVideo(Primary|Secondary)InfoRenderer() --- .../youtube/extractors/YoutubeStreamExtractor.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e59c89a7..fee13f3d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -95,6 +95,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { private final Map videoInfoPage = new HashMap<>(); private JsonObject playerResponse; private JsonObject initialData; + private JsonObject videoPrimaryInfoRenderer; + private JsonObject videoSecondaryInfoRenderer; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -843,6 +845,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { //////////////////////////////////////////////////////////////////////////*/ private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { + if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer; + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("results").getObject("results").getArray("contents"); JsonObject videoPrimaryInfoRenderer = null; @@ -858,10 +862,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ParsingException("Could not find videoPrimaryInfoRenderer"); } + this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer; return videoPrimaryInfoRenderer; } private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { + if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer; + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("results").getObject("results").getArray("contents"); JsonObject videoSecondaryInfoRenderer = null; @@ -877,6 +884,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ParsingException("Could not find videoSecondaryInfoRenderer"); } + this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer; return videoSecondaryInfoRenderer; } From 6ba03e542bac3a088787a5db556d97d6f1205dc0 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 17:05:31 +0100 Subject: [PATCH 111/149] Cache getVideoTab() --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 7b3f5b45..d82b3634 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -53,6 +53,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; private JsonObject initialData; + private JsonObject videoTab; public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -276,6 +277,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } private JsonObject getVideoTab() throws ParsingException { + if (this.videoTab != null) return this.videoTab; + JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); JsonObject videoTab = null; @@ -301,6 +304,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return null; } catch (Exception ignored) {} + this.videoTab = videoTab; return videoTab; } } From 4ff655352de245dacc901c179ea70e7512ad853c Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 28 Feb 2020 17:14:26 +0100 Subject: [PATCH 112/149] Reimplement getAgeLimit() --- .../youtube/extractors/YoutubeStreamExtractor.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index fee13f3d..ca9f2bfc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -97,6 +97,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject initialData; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; + private int ageLimit; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -216,12 +217,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() { if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet"); - if (initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("secondaryResults") == null) { - return 18; - } else { - return NO_AGE_LIMIT; - } + + return ageLimit; } @Override @@ -600,6 +597,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos initialData = initialAjaxJson.getObject(2).getObject("response"); + ageLimit = 18; final EmbeddedInfo info = getEmbeddedInfo(); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); @@ -608,6 +606,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerUrl = info.url; } else { initialData = initialAjaxJson.getObject(3).getObject("response"); + ageLimit = NO_AGE_LIMIT; playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player")); playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player")); From 4f406512c2eb0e7b2dfbdab6e17d94e36a8844bb Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 16:26:28 +0100 Subject: [PATCH 113/149] Use YoutubeChannelLinkHandlerFactory --- .../youtube/extractors/YoutubeChannelExtractor.java | 5 ++--- .../youtube/extractors/YoutubeStreamExtractor.java | 4 +++- .../linkHandler/YoutubeChannelLinkHandlerFactory.java | 8 ++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d82b3634..c051e31f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -50,8 +51,6 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP @SuppressWarnings("WeakerAccess") public class YoutubeChannelExtractor extends ChannelExtractor { - /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; - private JsonObject initialData; private JsonObject videoTab; @@ -96,7 +95,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getUrl() throws ParsingException { try { - return CHANNEL_URL_BASE + getId(); + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId()); } catch (ParsingException e) { return super.getUrl(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ca9f2bfc..92096d01 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -21,6 +21,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; @@ -328,7 +329,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} try { String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); - if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId; + if (uploaderId != null) + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); } catch (Exception ignored) {} throw new ParsingException("Could not get uploader url"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index eb34cf06..77eaf069 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -35,6 +35,14 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { return instance; } + /** + * Returns URL to channel from an ID + * + * @param id Channel ID including e.g. 'channel/' + * @param contentFilters + * @param searchFilter + * @return URL to channel + */ @Override public String getUrl(String id, List contentFilters, String searchFilter) { return "https://www.youtube.com/" + id; From 157055fbfd7d9b125bdbcf286c7e8b33eb736def Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 16:42:04 +0100 Subject: [PATCH 114/149] Create getJsonResponse() function --- .../extractors/YoutubeChannelExtractor.java | 41 ++---------------- .../extractors/YoutubePlaylistExtractor.java | 42 ++---------------- .../extractors/YoutubeSearchExtractor.java | 43 ++----------------- .../extractors/YoutubeStreamExtractor.java | 17 +------- .../extractors/YoutubeTrendingExtractor.java | 25 +---------- .../linkHandler/YoutubeParsingHelper.java | 20 +++++++++ 6 files changed, 33 insertions(+), 155 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index c051e31f..f28c0586 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -19,14 +17,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* @@ -62,22 +57,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); } @@ -214,22 +194,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(pageUrl); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("gridContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index fdd75868..2405df12 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,20 +10,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -42,22 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); playlistInfo = getPlaylistInfo(); @@ -207,22 +186,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(pageUrl); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("playlistVideoListContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 222c411f..62d1b3f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; @@ -14,16 +12,12 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* @@ -57,22 +51,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); } @@ -125,23 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } - + final JsonArray ajaxJson = getJsonResponse(pageUrl); JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("itemSectionContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 92096d01..75aa9284 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; @@ -53,6 +52,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -580,20 +580,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - initialAjaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + initialAjaxJson = getJsonResponse(url); final String playerUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 25aaa4db..30eceb9b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -22,8 +22,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -32,18 +30,14 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; public class YoutubeTrendingExtractor extends KioskExtractor { @@ -60,22 +54,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final String url = getUrl() + "?pbj=1&gl=" + getExtractorContentCountry().getCountryCode(); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - - try { - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json data for next streams", e); - } + final JsonArray ajaxJson = getJsonResponse(url); initialData = ajaxJson.getObject(1).getObject("response"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 5a33e4ce..02306c02 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -9,11 +9,13 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLDecoder; @@ -343,4 +345,22 @@ public class YoutubeParsingHelper { return thumbnailUrl; } + + public static JsonArray getJsonResponse(String url) throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(YoutubeParsingHelper.getClientVersion())); + final String response = getDownloader().get(url, headers).responseBody(); + + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + try { + return JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } } From 45df8248bfdb83696bf4f6f255e20abcdd162438 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 16:55:07 +0100 Subject: [PATCH 115/149] Use Localization in getJsonResponse() --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 4 ++-- .../youtube/extractors/YoutubePlaylistExtractor.java | 4 ++-- .../services/youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- .../youtube/extractors/YoutubeTrendingExtractor.java | 2 +- .../services/youtube/linkHandler/YoutubeParsingHelper.java | 5 +++-- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index f28c0586..91450d47 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -57,7 +57,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; - final JsonArray ajaxJson = getJsonResponse(url); + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(1).getObject("response"); } @@ -194,7 +194,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("gridContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 2405df12..1a1c8fab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -36,7 +36,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url); + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(1).getObject("response"); playlistInfo = getPlaylistInfo(); @@ -186,7 +186,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray ajaxJson = getJsonResponse(pageUrl); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("playlistVideoListContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 62d1b3f5..0449a9db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -51,7 +51,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - final JsonArray ajaxJson = getJsonResponse(url); + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(1).getObject("response"); } @@ -104,7 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - final JsonArray ajaxJson = getJsonResponse(pageUrl); + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("itemSectionContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 75aa9284..aa690fb9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -580,7 +580,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; - initialAjaxJson = getJsonResponse(url); + initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); final String playerUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 30eceb9b..e15463b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -54,7 +54,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { final String url = getUrl() + "?pbj=1&gl=" + getExtractorContentCountry().getCountryCode(); - final JsonArray ajaxJson = getJsonResponse(url); + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(1).getObject("response"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 02306c02..5986ddf4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -346,12 +347,12 @@ public class YoutubeParsingHelper { return thumbnailUrl; } - public static JsonArray getJsonResponse(String url) throws IOException, ExtractionException { + public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Version", Collections.singletonList(YoutubeParsingHelper.getClientVersion())); - final String response = getDownloader().get(url, headers).responseBody(); + final String response = getDownloader().get(url, headers, localization).responseBody(); if (response.length() < 50) { // ensure to have a valid response throw new ParsingException("JSON response is too short"); From a6a63e957030429223ad824aa1e8ad46408e6c65 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 17:01:36 +0100 Subject: [PATCH 116/149] Let isHardcodedClientVersionValid() throw IOException --- .../youtube/linkHandler/YoutubeParsingHelper.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 5986ddf4..643a6313 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -178,7 +178,7 @@ public class YoutubeParsingHelper { } } - public static boolean isHardcodedClientVersionValid() { + public static boolean isHardcodedClientVersionValid() throws IOException { try { final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; @@ -190,7 +190,7 @@ public class YoutubeParsingHelper { if (response.length() > 50) { // ensure to have a valid response return true; } - } catch (Exception ignored) {} + } catch (ReCaptchaException ignored) {} return false; } @@ -200,7 +200,7 @@ public class YoutubeParsingHelper { * @return * @throws ParsingException */ - public static String getClientVersion() throws ParsingException { + public static String getClientVersion() throws ParsingException, IOException { if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; if (isHardcodedClientVersionValid()) { @@ -350,8 +350,7 @@ public class YoutubeParsingHelper { public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion())); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final String response = getDownloader().get(url, headers, localization).responseBody(); if (response.length() < 50) { // ensure to have a valid response From bfe3eb1409a63a2c83269a452940a141beec7007 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 17:18:50 +0100 Subject: [PATCH 117/149] Use suggested try-if code style --- .../extractors/YoutubePlaylistExtractor.java | 9 ++- .../extractors/YoutubeStreamExtractor.java | 28 +++++++--- .../YoutubeStreamInfoItemExtractor.java | 55 +++++++++++-------- .../youtube/YoutubeParsingHelperTest.java | 4 +- 4 files changed, 58 insertions(+), 38 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 1a1c8fab..0f57a21a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -108,15 +108,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { if (url == null) { try { - return initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") + url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") .getArray("thumbnails").getObject(0).getString("url"); } catch (Exception ignored) {} + + if (url == null) throw new ParsingException("Could not get playlist thumbnail"); } - if (url != null && !url.isEmpty()) { - return fixThumbnailUrl(url); - } - throw new ParsingException("Could not get playlist thumbnail"); + return fixThumbnailUrl(url); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index aa690fb9..3a66dd3c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -116,16 +116,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getName() throws ParsingException { assertPageFetched(); String title = null; + try { title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); } catch (Exception ignored) {} + if (title == null) { try { title = playerResponse.getObject("videoDetails").getString("title"); } catch (Exception ignored) {} + + if (title == null) throw new ParsingException("Could not get name"); } - if (title != null) return title; - throw new ParsingException("Could not get name"); + + return title; } @Override @@ -259,17 +263,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { public long getViewCount() throws ParsingException { assertPageFetched(); String views = null; + try { views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") .getObject("videoViewCountRenderer").getObject("viewCount")); } catch (Exception ignored) {} + if (views == null) { try { views = playerResponse.getObject("videoDetails").getString("viewCount"); } catch (Exception ignored) {} + + if (views == null) throw new ParsingException("Could not get view count"); } - if (views != null) return Long.parseLong(Utils.removeNonDigitCharacters(views)); - throw new ParsingException("Could not get view count"); + + return Long.parseLong(Utils.removeNonDigitCharacters(views)); } @Override @@ -340,17 +348,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderName() throws ParsingException { assertPageFetched(); String uploaderName = null; + try { uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") .getObject("videoOwnerRenderer").getObject("title")); } catch (Exception ignored) {} + if (uploaderName == null) { try { uploaderName = playerResponse.getObject("videoDetails").getString("author"); } catch (Exception ignored) {} + + if (uploaderName == null) throw new ParsingException("Could not get uploader name"); } - if (uploaderName != null) return uploaderName; - throw new ParsingException("Could not get uploader name"); + + return uploaderName; } @Nonnull @@ -910,9 +922,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { urlAndItags.put(streamUrl, itagItem); } - } catch (UnsupportedEncodingException ignored) { - - } + } catch (UnsupportedEncodingException ignored) {} } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index e981071c..1e7ad5f9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -87,10 +87,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getDuration() throws ParsingException { if (getStreamType() == StreamType.LIVE_STREAM) return -1; + String duration = null; + try { duration = getTextFromObject(videoInfo.getObject("lengthText")); } catch (Exception ignored) {} + if (duration == null) { try { for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { @@ -100,58 +103,64 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } } } catch (Exception ignored) {} + + if (duration == null) throw new ParsingException("Could not get duration"); } - if (duration != null) return YoutubeParsingHelper.parseDurationString(duration); - throw new ParsingException("Could not get duration"); + + return YoutubeParsingHelper.parseDurationString(duration); } @Override public String getUploaderName() throws ParsingException { String name = null; + try { name = getTextFromObject(videoInfo.getObject("longBylineText")); } catch (Exception ignored) {} + if (name == null) { try { name = getTextFromObject(videoInfo.getObject("ownerText")); } catch (Exception ignored) {} + + if (name == null) { + try { + name = getTextFromObject(videoInfo.getObject("shortBylineText")); + } catch (Exception ignored) {} + + if (name == null) throw new ParsingException("Could not get uploader name"); + } } - if (name == null) { - try { - name = getTextFromObject(videoInfo.getObject("shortBylineText")); - } catch (Exception ignored) {} - } - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); + + return name; } @Override public String getUploaderUrl() throws ParsingException { + String url = null; + try { - String url = null; + url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); + } catch (Exception ignored) {} + + if (url == null) { try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); } catch (Exception ignored) {} - if (url == null) { - try { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - } catch (Exception ignored) {} - } + if (url == null) { try { url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") .getArray("runs").getObject(0).getObject("navigationEndpoint")); } catch (Exception ignored) {} + + if (url == null) throw new ParsingException("Could not get uploader url"); } - if (url == null || url.isEmpty()) { - throw new IllegalArgumentException("is empty"); - } - return url; - } catch (Exception e) { - throw new ParsingException("Could not get uploader url"); } + + return url; } @Nullable diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 06eedaef..f8ff1235 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -6,6 +6,8 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import java.io.IOException; + import static org.junit.Assert.assertTrue; public class YoutubeParsingHelperTest { @@ -15,7 +17,7 @@ public class YoutubeParsingHelperTest { } @Test - public void testIsHardcodedClientVersionValid() { + public void testIsHardcodedClientVersionValid() throws IOException { assertTrue("Hardcoded client version is not valid anymore", YoutubeParsingHelper.isHardcodedClientVersionValid()); } From 3e4d2fd64e4fd00f02ebacb8c31d9e3b2de7d6ed Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 18:20:35 +0100 Subject: [PATCH 118/149] Improve testChannelUrl() --- .../search/YoutubeSearchExtractorChannelOnlyTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index eb56bb5e..cb72622b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -92,13 +92,18 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto if (item instanceof ChannelInfoItem) { ChannelInfoItem channel = (ChannelInfoItem) item; - if (channel.getSubscriberCount() > 5e7 && channel.getName().equals("PewDiePie")) { // the real PewDiePie + if (channel.getSubscriberCount() > 1e8) { // the real PewDiePie assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); - } else { - assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); + break; } } } + + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof ChannelInfoItem) { + assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); + } + } } @Test From 280707953106ffa9f6ef719c4b76cac2865910eb Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 29 Feb 2020 21:28:38 +0100 Subject: [PATCH 119/149] Catch NumberFormatException instead of checking whether viewCount equals "Recommended for you" --- .../youtube/extractors/YoutubeStreamInfoItemExtractor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 1e7ad5f9..2ee89e24 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -195,8 +195,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return -1; } String viewCount = getTextFromObject(videoInfo.getObject("viewCountText")); - if (viewCount.equals("Recommended for you")) return -1; + return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); + } catch (NumberFormatException e) { + return -1; } catch (Exception e) { throw new ParsingException("Could not get view count", e); } From 705896aca0563d4555c4302df007f8413b3ae30b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 29 Feb 2020 21:39:45 +0100 Subject: [PATCH 120/149] Version 0.18.6 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 81b48035..97759443 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.5' + version 'v0.18.6' group 'com.github.TeamNewPipe' repositories { From df219a9453793800f3389e98c95206d13587d8ae Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sun, 1 Mar 2020 09:41:06 +0100 Subject: [PATCH 121/149] Don't use StandardCharsets class --- .../services/youtube/linkHandler/YoutubeParsingHelper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 643a6313..1c603f5b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -279,7 +278,7 @@ public class YoutubeParsingHelper { if (param.split("=")[0].equals("q")) { String url; try { - url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); + url = URLDecoder.decode(param.split("=")[1], "UTF-8"); } catch (UnsupportedEncodingException e) { return null; } From 3441946bea9c172a4c96dd6deae450cecab875a4 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 17:52:12 -0300 Subject: [PATCH 122/149] Make test downloader return a response instead of throwing an exception The test implementation was throwing an exception instead of just returning the response and letting the caller handle it. --- .../test/java/org/schabi/newpipe/DownloaderTestImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java index e524ac8d..8d7a3437 100644 --- a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java +++ b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java @@ -102,16 +102,20 @@ public class DownloaderTestImpl extends Downloader { return new Response(responseCode, responseMessage, responseHeaders, response.toString()); } catch (Exception e) { + final int responseCode = connection.getResponseCode(); + /* * HTTP 429 == Too Many Request * Receive from Youtube.com = ReCaptcha challenge request * See : https://github.com/rg3/youtube-dl/issues/5138 */ - if (connection.getResponseCode() == 429) { + if (responseCode == 429) { throw new ReCaptchaException("reCaptcha Challenge requested", url); + } else if (responseCode != -1) { + return new Response(responseCode, connection.getResponseMessage(), connection.getHeaderFields(), null); } - throw new IOException(connection.getResponseCode() + " " + connection.getResponseMessage(), e); + throw new IOException("Error occurred while fetching the content", e); } finally { if (outputStream != null) outputStream.close(); if (input != null) input.close(); From 5edd774fc44b037608840e1e968aa72a581040a8 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 17:57:23 -0300 Subject: [PATCH 123/149] Add latest url to the response to make detection of a redirect possible Will be latest one in this commit because there's need to check the history of redirects as of now. --- .../newpipe/extractor/downloader/Response.java | 16 +++++++++++++++- .../org/schabi/newpipe/DownloaderTestImpl.java | 6 ++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index 59c3911c..5fa432c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -15,12 +15,16 @@ public class Response { private final Map> responseHeaders; private final String responseBody; - public Response(int responseCode, String responseMessage, Map> responseHeaders, @Nullable String responseBody) { + private final String latestUrl; + + public Response(int responseCode, String responseMessage, Map> responseHeaders, + @Nullable String responseBody, @Nullable String latestUrl) { this.responseCode = responseCode; this.responseMessage = responseMessage; this.responseHeaders = responseHeaders != null ? responseHeaders : Collections.>emptyMap(); this.responseBody = responseBody == null ? "" : responseBody; + this.latestUrl = latestUrl; } public int responseCode() { @@ -40,6 +44,16 @@ public class Response { return responseBody; } + /** + * Used for detecting a possible redirection, limited to the latest one. + * + * @return latest url known right before this response object was created + */ + @Nonnull + public String latestUrl() { + return latestUrl; + } + /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ diff --git a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java index 8d7a3437..c93f31b7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java +++ b/extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java @@ -99,8 +99,9 @@ public class DownloaderTestImpl extends Downloader { final int responseCode = connection.getResponseCode(); final String responseMessage = connection.getResponseMessage(); final Map> responseHeaders = connection.getHeaderFields(); + final String latestUrl = connection.getURL().toString(); - return new Response(responseCode, responseMessage, responseHeaders, response.toString()); + return new Response(responseCode, responseMessage, responseHeaders, response.toString(), latestUrl); } catch (Exception e) { final int responseCode = connection.getResponseCode(); @@ -112,7 +113,8 @@ public class DownloaderTestImpl extends Downloader { if (responseCode == 429) { throw new ReCaptchaException("reCaptcha Challenge requested", url); } else if (responseCode != -1) { - return new Response(responseCode, connection.getResponseMessage(), connection.getHeaderFields(), null); + final String latestUrl = connection.getURL().toString(); + return new Response(responseCode, connection.getResponseMessage(), connection.getHeaderFields(), null, latestUrl); } throw new IOException("Error occurred while fetching the content", e); From f66c20de54418b464d529bebf0a49f66a1f27e74 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 17:59:51 -0300 Subject: [PATCH 124/149] Ignore null-keyed entries when iterating through the response headers --- .../java/org/schabi/newpipe/extractor/downloader/Response.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index 5fa432c2..b252d463 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -68,7 +68,8 @@ public class Response { @Nullable public String getHeader(String name) { for (Map.Entry> headerEntry : responseHeaders.entrySet()) { - if (headerEntry.getKey().equalsIgnoreCase(name)) { + final String key = headerEntry.getKey(); + if (key != null && key.equalsIgnoreCase(name)) { if (headerEntry.getValue().size() > 0) { return headerEntry.getValue().get(0); } From da2c562eb0fe5244e6078b8d6d038d0341dddd32 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 18:01:43 -0300 Subject: [PATCH 125/149] [YouTube] Take into account videos that have their views hidden Also remove catch because it may hide bugs when the page changes. --- .../extractors/YoutubeStreamInfoItemExtractor.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 2ee89e24..0ed79eec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -194,11 +194,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { return -1; } - String viewCount = getTextFromObject(videoInfo.getObject("viewCountText")); + + final JsonObject viewCountObject = videoInfo.getObject("viewCountText"); + if (viewCountObject == null) { + // This object is null when a video has its views hidden. + return -1; + } + + final String viewCount = getTextFromObject(viewCountObject); return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); - } catch (NumberFormatException e) { - return -1; } catch (Exception e) { throw new ParsingException("Could not get view count", e); } From 2a470ac4f740f9232e8fe7a7ec0adab7b90bf659 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 18:02:00 -0300 Subject: [PATCH 126/149] [YouTube] Handle videos with no views or with "Recommended to you" text --- .../youtube/extractors/YoutubeStreamInfoItemExtractor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 0ed79eec..b9d61046 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -203,6 +203,12 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { final String viewCount = getTextFromObject(viewCountObject); + if (viewCount.toLowerCase().contains("no views")) { + return 0; + } else if (viewCount.toLowerCase().contains("recommended")) { + return -1; + } + return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); } catch (Exception e) { throw new ParsingException("Could not get view count", e); From e9644e6216c9aaa20a6fefa4a0c94b471c09b1bf Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 18:19:34 -0300 Subject: [PATCH 127/149] [YouTube] Handle video premiere's date and duration --- .../YoutubeStreamInfoItemExtractor.java | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index b9d61046..be2aaf97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -13,10 +12,11 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*; /* * Copyright (C) Christian Schabesberger 2016 @@ -86,7 +86,9 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public long getDuration() throws ParsingException { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; + if (getStreamType() == StreamType.LIVE_STREAM || isPremiere()) { + return -1; + } String duration = null; @@ -165,7 +167,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override - public String getTextualUploadDate() { + public String getTextualUploadDate() throws ParsingException { + if (getStreamType().equals(StreamType.LIVE_STREAM)) { + return null; + } + + if (isPremiere()) { + final Date date = getDateFromPremiere().getTime(); + return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date); + } + try { return getTextFromObject(videoInfo.getObject("publishedTimeText")); } catch (Exception e) { @@ -177,7 +188,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - String textualUploadDate = getTextualUploadDate(); + if (getStreamType().equals(StreamType.LIVE_STREAM)) { + return null; + } + + if (isPremiere()) { + return new DateWrapper(getDateFromPremiere()); + } + + final String textualUploadDate = getTextualUploadDate(); if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) { try { return timeAgoParser.parse(textualUploadDate); @@ -236,7 +255,26 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return true; } } - } catch (Exception ignored) {} + } catch (Exception ignored) { + } return false; } + + private boolean isPremiere() { + return videoInfo.has("upcomingEventData"); + } + + private Calendar getDateFromPremiere() throws ParsingException { + final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData"); + final String startTime = upcomingEventData.getString("startTime"); + + try { + final long startTimeTimestamp = Long.parseLong(startTime); + final Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date(startTimeTimestamp * 1000L)); + return calendar; + } catch (Exception e) { + throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); + } + } } From 342bdbb85256a510a7c9b4aeeea148d8a34481bf Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 18:42:43 -0300 Subject: [PATCH 128/149] [YouTube] Avoid crashing by letting exceptions bubble up --- .../linkHandler/YoutubeParsingHelper.java | 126 ++++++++---------- .../youtube/YoutubeParsingHelperTest.java | 3 +- 2 files changed, 58 insertions(+), 71 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 1c603f5b..5b37e4c9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -5,7 +5,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; @@ -22,12 +21,7 @@ import java.net.URL; import java.net.URLDecoder; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; @@ -177,21 +171,16 @@ public class YoutubeParsingHelper { } } - public static boolean isHardcodedClientVersionValid() throws IOException { - try { - final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; + public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException { + final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - headers.put("X-YouTube-Client-Version", - Collections.singletonList(HARDCODED_CLIENT_VERSION)); - final String response = getDownloader().get(url, headers).responseBody(); - if (response.length() > 50) { // ensure to have a valid response - return true; - } - } catch (ReCaptchaException ignored) {} + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers).responseBody(); - return false; + return response.length() > 50; // ensure to have a valid response } /** @@ -199,7 +188,7 @@ public class YoutubeParsingHelper { * @return * @throws ParsingException */ - public static String getClientVersion() throws ParsingException, IOException { + public static String getClientVersion() throws IOException, ExtractionException { if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; if (isHardcodedClientVersionValid()) { @@ -207,62 +196,59 @@ public class YoutubeParsingHelper { return clientVersion; } - // Try extracting it from YouTube's website otherwise - try { - final String url = "https://www.youtube.com/results?search_query=test"; - final String html = getDownloader().get(url).responseBody(); - JsonObject initialData = getInitialData(html); - JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); - String shortClientVersion = null; + final String url = "https://www.youtube.com/results?search_query=test"; + final String html = getDownloader().get(url).responseBody(); + JsonObject initialData = getInitialData(html); + JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + String shortClientVersion = null; - // try to get version from initial data first - for (Object service : serviceTrackingParams) { - JsonObject s = (JsonObject) service; - if (s.getString("service").equals("CSI")) { - JsonArray params = s.getArray("params"); - for (Object param : params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("cver")) { - clientVersion = p.getString("value"); - return clientVersion; - } - } - } else if (s.getString("service").equals("ECATCHER")) { - // fallback to get a shortened client version which does not contain the last two digits - JsonArray params = s.getArray("params"); - for (Object param : params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("client.version")) { - shortClientVersion = p.getString("value"); - } - } - } - } - - String contextClientVersion; - String[] patterns = { - "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", - "innertube_context_client_version\":\"([0-9\\.]+?)\"", - "client.version=([0-9\\.]+)" - }; - for (String pattern : patterns) { - try { - contextClientVersion = Parser.matchGroup1(pattern, html); - if (contextClientVersion != null && !contextClientVersion.isEmpty()) { - clientVersion = contextClientVersion; + // try to get version from initial data first + for (Object service : serviceTrackingParams) { + JsonObject s = (JsonObject) service; + if (s.getString("service").equals("CSI")) { + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("cver")) { + clientVersion = p.getString("value"); return clientVersion; } - } catch (Exception ignored) { + } + } else if (s.getString("service").equals("ECATCHER")) { + // fallback to get a shortened client version which does not contain the last two digits + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("client.version")) { + shortClientVersion = p.getString("value"); + } } } + } - if (shortClientVersion != null) { - clientVersion = shortClientVersion; - return clientVersion; + String contextClientVersion; + String[] patterns = { + "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", + "innertube_context_client_version\":\"([0-9\\.]+?)\"", + "client.version=([0-9\\.]+)" + }; + for (String pattern : patterns) { + try { + contextClientVersion = Parser.matchGroup1(pattern, html); + if (contextClientVersion != null && !contextClientVersion.isEmpty()) { + clientVersion = contextClientVersion; + return clientVersion; + } + } catch (Exception ignored) { } - } catch (Exception ignored) {} + } + + if (shortClientVersion != null) { + clientVersion = shortClientVersion; + return clientVersion; + } throw new ParsingException("Could not get client version"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index f8ff1235..87dbbd75 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -4,6 +4,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import java.io.IOException; @@ -17,7 +18,7 @@ public class YoutubeParsingHelperTest { } @Test - public void testIsHardcodedClientVersionValid() throws IOException { + public void testIsHardcodedClientVersionValid() throws IOException, ExtractionException { assertTrue("Hardcoded client version is not valid anymore", YoutubeParsingHelper.isHardcodedClientVersionValid()); } From 408f0421276da45410af1206e7de761865829f84 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 18:57:25 -0300 Subject: [PATCH 129/149] [YouTube] Fix bug when url isn't present in the browseEndpoint object --- .../extractors/YoutubeSearchExtractor.java | 4 ++-- .../extractors/YoutubeStreamExtractor.java | 8 +++++-- .../linkHandler/YoutubeParsingHelper.java | 21 +++++++++++++++---- .../youtube/YoutubePlaylistExtractorTest.java | 2 +- .../YoutubeSearchExtractorDefaultTest.java | 2 +- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 0449a9db..6fec1164 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -63,7 +63,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } @Override - public String getSearchSuggestion() { + public String getSearchSuggestion() throws ParsingException { JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) @@ -114,7 +114,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations"))); } - private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException { + private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException { collector.reset(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 3a66dd3c..6aad8cf2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -564,8 +564,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public String getErrorMessage() { - return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus") - .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason")); + try { + return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus") + .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason")); + } catch (ParsingException e) { + return null; + } } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 5b37e4c9..32f6769f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -253,7 +253,7 @@ public class YoutubeParsingHelper { throw new ParsingException("Could not get client version"); } - public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) { + public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { if (navigationEndpoint.getObject("urlEndpoint") != null) { String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); if (internUrl.startsWith("/redirect?")) { @@ -275,7 +275,20 @@ public class YoutubeParsingHelper { return internUrl; } } else if (navigationEndpoint.getObject("browseEndpoint") != null) { - return "https://www.youtube.com" + navigationEndpoint.getObject("browseEndpoint").getString("canonicalBaseUrl"); + final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); + final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl"); + final String browseId = browseEndpoint.getString("browseId"); + + // All channel ids are prefixed with UC + if (browseId != null && browseId.startsWith("UC")) { + return "https://www.youtube.com/channel/" + browseId; + } + + if (canonicalBaseUrl != null && !canonicalBaseUrl.isEmpty()) { + return "https://www.youtube.com" + canonicalBaseUrl; + } + + throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")"); } else if (navigationEndpoint.getObject("watchEndpoint") != null) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); @@ -288,7 +301,7 @@ public class YoutubeParsingHelper { return null; } - public static String getTextFromObject(JsonObject textObject, boolean html) { + public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException { if (textObject.has("simpleText")) return textObject.getString("simpleText"); StringBuilder textBuilder = new StringBuilder(); @@ -314,7 +327,7 @@ public class YoutubeParsingHelper { return text; } - public static String getTextFromObject(JsonObject textObject) { + public static String getTextFromObject(JsonObject textObject) throws ParsingException { return getTextFromObject(textObject, false); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 7c911279..d4de9175 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -99,7 +99,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testUploaderUrl() throws Exception { - assertEquals("https://www.youtube.com/user/andre0y0you", extractor.getUploaderUrl()); + assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 65ffe839..1f905bde 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -124,7 +124,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas } @Test - public void testSuggestionNotNull() { + public void testSuggestionNotNull() throws Exception { //todo write a real test assertNotNull(extractor.getSearchSuggestion()); } From 98e359438a4bdcd252b7fa6bb5fe9a48fde5d0be Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 21:50:31 -0300 Subject: [PATCH 130/149] [YouTube] Detect simple 404s in the standard fetch method --- .../youtube/linkHandler/YoutubeParsingHelper.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 32f6769f..87d37f8e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -8,6 +8,7 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -349,14 +350,20 @@ public class YoutubeParsingHelper { Map> headers = new HashMap<>(); headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); - final String response = getDownloader().get(url, headers, localization).responseBody(); + final Response response = getDownloader().get(url, headers, localization); - if (response.length() < 50) { // ensure to have a valid response + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + } + + final String responseBody = response.responseBody(); + if (responseBody.length() < 50) { // ensure to have a valid response throw new ParsingException("JSON response is too short"); } try { - return JsonParser.array().from(response); + return JsonParser.array().from(responseBody); } catch (JsonParserException e) { throw new ParsingException("Could not parse JSON", e); } From e65333c3cea86b6fa8197044b8969b21df47291e Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 21:52:25 -0300 Subject: [PATCH 131/149] [YouTube] Detect deleted/nonexistent/invalid channels and playlists - Added tests for these cases. --- .../extractors/YoutubeChannelExtractor.java | 2 +- .../extractors/YoutubePlaylistExtractor.java | 3 ++ .../linkHandler/YoutubeParsingHelper.java | 35 +++++++++++++++++++ .../youtube/YoutubeChannelExtractorTest.java | 23 ++++++++++++ .../youtube/YoutubePlaylistExtractorTest.java | 23 ++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 91450d47..11dd8985 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -58,8 +58,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - initialData = ajaxJson.getObject(1).getObject("response"); + YoutubeParsingHelper.defaultAlertsCheck(initialData); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 0f57a21a..eef85aa9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; @@ -39,6 +40,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(1).getObject("response"); + YoutubeParsingHelper.defaultAlertsCheck(initialData); + playlistInfo = getPlaylistInfo(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 87d37f8e..d2b0a7da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -362,10 +362,45 @@ public class YoutubeParsingHelper { throw new ParsingException("JSON response is too short"); } + // Check if the request was redirected to the error page. + final URL latestUrl = new URL(response.latestUrl()); + if (latestUrl.getHost().equalsIgnoreCase("www.youtube.com")) { + final String path = latestUrl.getPath(); + if (path.equalsIgnoreCase("/oops") || path.equalsIgnoreCase("/error")) { + throw new ContentNotAvailableException("Content unavailable"); + } + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { throw new ParsingException("Could not parse JSON", e); } } + + /** + * Shared alert detection function, multiple endpoints return the error similarly structured. + *

    + * Will check if the object has an alert of the type "ERROR". + * + * @param initialData the object which will be checked if an alert is present + * @throws ContentNotAvailableException if an alert is detected + */ + public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException { + final JsonArray alerts = initialData.getArray("alerts"); + if (alerts != null && !alerts.isEmpty()) { + final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); + final String alertText = alertRenderer.getObject("text").getString("simpleText"); + final String alertType = alertRenderer.getString("type"); + if (alertType.equalsIgnoreCase("ERROR")) { + throw new ContentNotAvailableException("Got error: \"" + alertText + "\""); + } + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index fc4ffff3..4d69a6c8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; @@ -19,6 +20,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link ChannelExtractor} */ public class YoutubeChannelExtractorTest { + + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void deletedFetch() throws Exception { + final ChannelExtractor extractor = + YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw"); + extractor.fetchPage(); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final ChannelExtractor extractor = + YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST"); + extractor.fetchPage(); + } + } + public static class Gronkh implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index d4de9175..67584195 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; @@ -23,6 +24,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link YoutubePlaylistExtractor} */ public class YoutubePlaylistExtractorTest { + + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final PlaylistExtractor extractor = + YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL11111111111111111111111111111111"); + extractor.fetchPage(); + } + + @Test(expected = ContentNotAvailableException.class) + public void invalidId() throws Exception { + final PlaylistExtractor extractor = + YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID"); + extractor.fetchPage(); + } + } + public static class TimelessPopHits implements BasePlaylistExtractorTest { private static YoutubePlaylistExtractor extractor; From 5686a6f562619c50bed53904eaafbc2aa85c8dbe Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 22:00:33 -0300 Subject: [PATCH 132/149] [YouTube] Detect when a stream is deleted or doesn't exist Added a test case as well. --- .../extractors/YoutubeStreamExtractor.java | 9 ++++++++ .../newpipe/extractor/utils/JsonUtils.java | 1 + .../YoutubeStreamExtractorDefaultTest.java | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 6aad8cf2..13f88110 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -32,6 +33,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -619,6 +621,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerResponse = getPlayerResponse(); + final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY); + final String status = playabilityStatus.getString("status"); + if (status != null && status.toLowerCase().equals("error")) { + final String reason = playabilityStatus.getString("reason"); + throw new ContentNotAvailableException("Got error: \"" + reason + "\""); + } + if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ab916379..25bb3f6c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -11,6 +11,7 @@ import java.util.Arrays; import java.util.List; public class JsonUtils { + public static final JsonObject DEFAULT_EMPTY = new JsonObject(); private JsonUtils() { } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index d6cf3815..0d36fd9a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; @@ -56,6 +57,27 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; */ public class YoutubeStreamExtractorDefaultTest { + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final StreamExtractor extractor = + YouTube.getStreamExtractor("https://www.youtube.com/watch?v=don-t-exist"); + extractor.fetchPage(); + } + + @Test(expected = ParsingException.class) + public void invalidId() throws Exception { + final StreamExtractor extractor = + YouTube.getStreamExtractor("https://www.youtube.com/watch?v=INVALID_ID_INVALID_ID"); + extractor.fetchPage(); + } + } + /** * Test for {@link StreamExtractor} */ From 35252235b0f88fcc44eb048be0228f94bcc36360 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 1 Mar 2020 12:52:37 +0100 Subject: [PATCH 133/149] Go through all badges when checking if a StreamInfoItem is a live stream --- .../extractors/YoutubeStreamInfoItemExtractor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index be2aaf97..8725e554 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -55,9 +55,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public StreamType getStreamType() { try { - if (videoInfo.getArray("badges").getObject(0).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { - return StreamType.LIVE_STREAM; + JsonArray badges = videoInfo.getArray("badges"); + for (Object badge : badges) { + if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { + return StreamType.LIVE_STREAM; + } } + } catch (Exception ignored) {} return StreamType.VIDEO_STREAM; } From e590417cc442041941d23005d74496c0e48a539d Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Thu, 27 Feb 2020 00:08:23 -0300 Subject: [PATCH 134/149] Test if services recognizes their own items urls --- .../extractor/services/DefaultTests.java | 58 +++++++++++++++---- .../PeertubeChannelExtractorTest.java | 10 ++-- .../SoundcloudChannelExtractorTest.java | 10 ++-- .../SoundcloudPlaylistExtractorTest.java | 18 +++--- .../youtube/YoutubeChannelExtractorTest.java | 27 +++++---- .../YoutubeChannelLocalizationTest.java | 2 +- .../youtube/YoutubeCommentsExtractorTest.java | 2 +- .../youtube/YoutubeFeedExtractorTest.java | 8 +-- .../youtube/YoutubePlaylistExtractorTest.java | 15 +++-- 9 files changed, 91 insertions(+), 59 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index 8cf27094..6c2dd9ef 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -2,20 +2,29 @@ package org.schabi.newpipe.extractor.services; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.Calendar; import java.util.List; +import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.*; +import static org.schabi.newpipe.extractor.StreamingService.*; public final class DefaultTests { - public static void defaultTestListOfItems(int expectedServiceId, List itemsList, List errors) { - assertTrue("List of items is empty", !itemsList.isEmpty()); + public static void defaultTestListOfItems(StreamingService expectedService, List itemsList, List errors) throws ParsingException { + assertFalse("List of items is empty", itemsList.isEmpty()); assertFalse("List of items contains a null element", itemsList.contains(null)); - assertEmptyErrors("Errors during stream list extraction", errors); + assertEmptyErrors("Errors during extraction", errors); for (InfoItem item : itemsList) { assertIsSecureUrl(item.getUrl()); @@ -23,12 +32,17 @@ public final class DefaultTests { assertIsSecureUrl(item.getThumbnailUrl()); } assertNotNull("InfoItem type not set: " + item, item.getInfoType()); - assertEquals("Service id doesn't match: " + item, expectedServiceId, item.getServiceId()); + assertEquals("Unexpected item service id", expectedService.getServiceId(), item.getServiceId()); + assertNotEmpty("Item name not set: " + item, item.getName()); if (item instanceof StreamInfoItem) { StreamInfoItem streamInfoItem = (StreamInfoItem) item; assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName()); assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); + assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + + assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM); + assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL); final String textualUploadDate = streamInfoItem.getTextualUploadDate(); if (textualUploadDate != null && !textualUploadDate.isEmpty()) { @@ -37,34 +51,56 @@ public final class DefaultTests { assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance())); } + } else if (item instanceof ChannelInfoItem) { + final ChannelInfoItem channelInfoItem = (ChannelInfoItem) item; + assertExpectedLinkType(expectedService, channelInfoItem.getUrl(), LinkType.CHANNEL); + + } else if (item instanceof PlaylistInfoItem) { + final PlaylistInfoItem playlistInfoItem = (PlaylistInfoItem) item; + assertExpectedLinkType(expectedService, playlistInfoItem.getUrl(), LinkType.PLAYLIST); } } } - public static ListExtractor.InfoItemsPage defaultTestRelatedItems(ListExtractor extractor, int expectedServiceId) throws Exception { + private static void assertExpectedLinkType(StreamingService expectedService, String url, LinkType expectedLinkType) throws ParsingException { + final LinkType linkTypeByUrl = expectedService.getLinkTypeByUrl(url); + + assertNotEquals("Url is not recognized by its own service: \"" + url + "\"", + LinkType.NONE, linkTypeByUrl); + assertEquals("Service returned wrong link type for: \"" + url + "\"", + expectedLinkType, linkTypeByUrl); + } + + public static void assertNoMoreItems(ListExtractor extractor) throws Exception { + assertFalse("More items available when it shouldn't", extractor.hasNextPage()); + final String nextPageUrl = extractor.getNextPageUrl(); + assertTrue("Next page is not empty or null", nextPageUrl == null || nextPageUrl.isEmpty()); + } + + public static ListExtractor.InfoItemsPage defaultTestRelatedItems(ListExtractor extractor) throws Exception { final ListExtractor.InfoItemsPage page = extractor.getInitialPage(); final List itemsList = page.getItems(); List errors = page.getErrors(); - defaultTestListOfItems(expectedServiceId, itemsList, errors); + defaultTestListOfItems(extractor.getService(), itemsList, errors); return page; } - public static ListExtractor.InfoItemsPage defaultTestMoreItems(ListExtractor extractor, int expectedServiceId) throws Exception { + public static ListExtractor.InfoItemsPage defaultTestMoreItems(ListExtractor extractor) throws Exception { assertTrue("Doesn't have more items", extractor.hasNextPage()); ListExtractor.InfoItemsPage nextPage = extractor.getPage(extractor.getNextPageUrl()); final List items = nextPage.getItems(); - assertTrue("Next page is empty", !items.isEmpty()); + assertFalse("Next page is empty", items.isEmpty()); assertEmptyErrors("Next page have errors", nextPage.getErrors()); - defaultTestListOfItems(expectedServiceId, nextPage.getItems(), nextPage.getErrors()); + defaultTestListOfItems(extractor.getService(), nextPage.getItems(), nextPage.getErrors()); return nextPage; } - public static void defaultTestGetPageInNewExtractor(ListExtractor extractor, ListExtractor newExtractor, int expectedServiceId) throws Exception { + public static void defaultTestGetPageInNewExtractor(ListExtractor extractor, ListExtractor newExtractor) throws Exception { final String nextPageUrl = extractor.getNextPageUrl(); final ListExtractor.InfoItemsPage page = newExtractor.getPage(nextPageUrl); - defaultTestListOfItems(expectedServiceId, page.getItems(), page.getErrors()); + defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index a1d1f383..7c7a9b80 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -68,12 +68,12 @@ public class PeertubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, PeerTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -127,7 +127,7 @@ public class PeertubeChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -165,12 +165,12 @@ public class PeertubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, PeerTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 0de8e847..485462fb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -64,12 +64,12 @@ public class SoundcloudChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -120,7 +120,7 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -158,12 +158,12 @@ public class SoundcloudChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index c27dfdf8..b93c2cfb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -7,7 +7,6 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -67,13 +66,13 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() { try { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } catch (Throwable ignored) { return; } @@ -165,12 +164,12 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, SoundCloud.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -226,11 +225,10 @@ public class SoundcloudPlaylistExtractorTest { // Additional Testing //////////////////////////////////////////////////////////////////////////*/ - @Ignore @Test public void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -269,18 +267,18 @@ public class SoundcloudPlaylistExtractorTest { @Ignore @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, SoundCloud.getServiceId()); + defaultTestRelatedItems(extractor); } //TODO: FUCK THIS: This triggers a 500 at sever @Ignore @Test public void testMoreRelatedItems() throws Exception { - ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId()); + ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); // Test for 2 more levels for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); - defaultTestListOfItems(SoundCloud.getServiceId(), currentPage.getItems(), currentPage.getErrors()); + defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 4d69a6c8..dfcaee40 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -4,7 +4,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -88,12 +87,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -178,12 +177,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -241,7 +240,7 @@ public class YoutubeChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -280,12 +279,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -371,12 +370,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -461,12 +460,12 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -553,13 +552,13 @@ public class YoutubeChannelExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() { try { - defaultTestMoreItems(extractor, YouTube.getServiceId()); + defaultTestMoreItems(extractor); } catch (Throwable ignored) { return; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java index 29f3d032..d5a629b0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java @@ -49,7 +49,7 @@ public class YoutubeChannelLocalizationTest { final ChannelExtractor extractor = YouTube.getChannelExtractor(channelUrl); extractor.forceLocalization(currentLocalization); extractor.fetchPage(); - itemsPage = defaultTestRelatedItems(extractor, YouTube.getServiceId()); + itemsPage = defaultTestRelatedItems(extractor); } catch (Throwable e) { System.out.println("[!] " + currentLocalization + " → failed"); throw e; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 3345ae3a..4349727b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -84,7 +84,7 @@ public class YoutubeCommentsExtractorTest { public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractorYT.getInitialPage(); - DefaultTests.defaultTestListOfItems(YouTube.getServiceId(), comments.getItems(), comments.getErrors()); + DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); for (CommentsInfoItem c : comments.getItems()) { assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); assertFalse(StringUtil.isBlank(c.getAuthorName())); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java index c7f1b1c1..a6a10ac5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtra import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; public class YoutubeFeedExtractorTest { @@ -60,13 +61,12 @@ public class YoutubeFeedExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test - public void testMoreRelatedItems() { - assertFalse(extractor.hasNextPage()); - assertNull(extractor.getNextPageUrl()); + public void testMoreRelatedItems() throws Exception { + assertNoMoreItems(extractor); } } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 67584195..65bb7f99 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -6,7 +6,7 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.ServiceList; + import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; @@ -93,12 +93,12 @@ public class YoutubePlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + defaultTestMoreItems(extractor); } /*////////////////////////////////////////////////////////////////////////// @@ -161,7 +161,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl()); - defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); + defaultTestGetPageInNewExtractor(extractor, newExtractor); } /*////////////////////////////////////////////////////////////////////////// @@ -200,18 +200,17 @@ public class YoutubePlaylistExtractorTest { @Test public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor, YouTube.getServiceId()); + defaultTestRelatedItems(extractor); } @Test public void testMoreRelatedItems() throws Exception { - ListExtractor.InfoItemsPage currentPage - = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); // test for 2 more levels for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); - defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors()); + defaultTestListOfItems(YouTube, currentPage.getItems(), currentPage.getErrors()); } } From 90ae5fbea2d610075a33b16199cc867d5e1190cb Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Thu, 27 Feb 2020 00:09:09 -0300 Subject: [PATCH 135/149] Improve kiosk tests across services --- .../PeertubeTrendingExtractorTest.java | 123 ++++++------- .../SoundcloudChartsExtractorTest.java | 170 +++++++++++------- .../youtube/YoutubeKioskExtractorTest.java | 70 ++++++++ .../youtube/YoutubeTrendingExtractorTest.java | 101 ----------- 4 files changed, 229 insertions(+), 235 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index 5d2d8478..e7334fca 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -5,90 +5,75 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudChartsExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.List; import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.ServiceList.*; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; -/** - * Test for {@link PeertubeTrendingExtractor} - */ public class PeertubeTrendingExtractorTest { + public static class Trending implements BaseListExtractorTest { + private static PeertubeTrendingExtractor extractor; - static KioskExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = PeerTube - .getKioskList() - .getExtractorById("Trending", null); - extractor.fetchPage(); - } - - @Test - public void testGetDownloader() throws Exception { - assertNotNull(NewPipe.getDownloader()); - } - - @Test - public void testGetName() throws Exception { - assertEquals(extractor.getName(), "Trending"); - } - - @Test - public void testId() { - assertEquals(extractor.getId(), "Trending"); - } - - @Test - public void testGetStreams() throws Exception { - ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if (!page.getErrors().isEmpty()) { - System.err.println("----------"); - List errors = page.getErrors(); - for (Throwable e : errors) { - e.printStackTrace(); - System.err.println("----------"); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeTrendingExtractor) PeerTube.getKioskList() + .getExtractorById("Trending", null); + extractor.fetchPage(); } - assertTrue("no streams are received", - !page.getItems().isEmpty() - && page.getErrors().isEmpty()); - } - @Test - public void testGetStreamsErrors() throws Exception { - assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); - } + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ - @Test - public void testHasMoreStreams() throws Exception { - // Setup the streams - extractor.getInitialPage(); - assertTrue("has more streams", extractor.hasNextPage()); - } + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } - @Test - public void testGetNextPageUrl() throws Exception { - assertTrue(extractor.hasNextPage()); - } + @Test + public void testName() throws Exception { + assertEquals("Trending", extractor.getName()); + } - @Test - public void testGetNextPage() throws Exception { - extractor.getInitialPage().getItems(); - assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); - } + @Test + public void testId() throws Exception { + assertEquals("Trending", extractor.getId()); + } - @Test - public void testGetCleanUrl() throws Exception { - assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending"); + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index dea1d170..189be417 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -5,87 +5,127 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.util.List; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.*; -/** - * Test for {@link SoundcloudChartsLinkHandlerFactory} - */ public class SoundcloudChartsExtractorTest { + public static class NewAndHot implements BaseListExtractorTest { + private static SoundcloudChartsExtractor extractor; - static KioskExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = SoundCloud - .getKioskList() - .getExtractorById("Top 50", null); - extractor.fetchPage(); - } - - @Test - public void testGetDownloader() throws Exception { - assertNotNull(NewPipe.getDownloader()); - } - - @Test - public void testGetName() throws Exception { - assertEquals(extractor.getName(), "Top 50"); - } - - @Test - public void testId() { - assertEquals(extractor.getId(), "Top 50"); - } - - @Test - public void testGetStreams() throws Exception { - ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - if (!page.getErrors().isEmpty()) { - System.err.println("----------"); - List errors = page.getErrors(); - for (Throwable e : errors) { - e.printStackTrace(); - System.err.println("----------"); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudChartsExtractor) SoundCloud.getKioskList() + .getExtractorById("New & hot", null); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(SoundCloud.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() { + assertEquals("New & hot", extractor.getName()); + } + + @Test + public void testId() { + assertEquals("New & hot", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/new", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/new", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); } - assertTrue("no streams are received", - !page.getItems().isEmpty() - && page.getErrors().isEmpty()); } - @Test - public void testGetStreamsErrors() throws Exception { - assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); - } + public static class Top50Charts implements BaseListExtractorTest { + private static SoundcloudChartsExtractor extractor; - @Test - public void testHasMoreStreams() throws Exception { - // Setup the streams - extractor.getInitialPage(); - assertTrue("has more streams", extractor.hasNextPage()); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudChartsExtractor) SoundCloud.getKioskList() + .getExtractorById("Top 50", null); + extractor.fetchPage(); + } - @Test - public void testGetNextPageUrl() throws Exception { - assertTrue(extractor.hasNextPage()); - } + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ - @Test - public void testGetNextPage() throws Exception { - extractor.getInitialPage().getItems(); - assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); - } + @Test + public void testServiceId() { + assertEquals(SoundCloud.getServiceId(), extractor.getServiceId()); + } - @Test - public void testGetCleanUrl() throws Exception { - assertEquals(extractor.getUrl(), "https://soundcloud.com/charts/top"); + @Test + public void testName() { + assertEquals("Top 50", extractor.getName()); + } + + @Test + public void testId() { + assertEquals("Top 50", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/top", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://soundcloud.com/charts/top", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java new file mode 100644 index 00000000..bbe8a76c --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java @@ -0,0 +1,70 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseListExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + +public class YoutubeKioskExtractorTest { + public static class Trending implements BaseListExtractorTest { + private static YoutubeTrendingExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeTrendingExtractor) YouTube.getKioskList().getDefaultKioskExtractor(); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("Trending", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("Trending", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/feed/trending", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/feed/trending", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + assertNoMoreItems(extractor); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java deleted file mode 100644 index 4931fc50..00000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -/* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2017 - * YoutubeTrendingExtractorTest.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.localization.ContentCountry; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.utils.Utils; - -import static junit.framework.TestCase.assertFalse; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - - -/** - * Test for {@link YoutubeTrendingLinkHandlerFactory} - */ -public class YoutubeTrendingExtractorTest { - - static YoutubeTrendingExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeTrendingExtractor) YouTube - .getKioskList() - .getExtractorById("Trending", null); - extractor.forceContentCountry(new ContentCountry("de")); - extractor.fetchPage(); - } - - @Test - public void testGetDownloader() throws Exception { - assertNotNull(NewPipe.getDownloader()); - } - - @Test - public void testGetName() throws Exception { - assertFalse(extractor.getName().isEmpty()); - } - - @Test - public void testId() throws Exception { - assertEquals(extractor.getId(), "Trending"); - } - - @Test - public void testGetStreamsQuantity() throws Exception { - ListExtractor.InfoItemsPage page = extractor.getInitialPage(); - Utils.printErrors(page.getErrors()); - assertTrue("no streams are received", page.getItems().size() >= 20); - } - - @Test - public void testGetStreamsErrors() throws Exception { - assertEmptyErrors("errors during stream list extraction", extractor.getInitialPage().getErrors()); - } - - @Test - public void testHasMoreStreams() throws Exception { - // Setup the streams - extractor.getInitialPage(); - assertFalse("has more streams", extractor.hasNextPage()); - } - - @Test - public void testGetNextPage() { - assertTrue("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); - } - - @Test - public void testGetUrl() throws Exception { - assertEquals(extractor.getUrl(), extractor.getUrl(), "https://www.youtube.com/feed/trending"); - } -} From cfc278317ddf529b572632bd1f8d3214c5eeab98 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sun, 1 Mar 2020 10:13:00 -0300 Subject: [PATCH 136/149] [YouTube] Fail-fast if status exist and is anything other than "OK" --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 13f88110..8ee50201 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -623,7 +623,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.DEFAULT_EMPTY); final String status = playabilityStatus.getString("status"); - if (status != null && status.toLowerCase().equals("error")) { + // If status exist, and is not "OK", throw a ContentNotAvailableException with the reason. + if (status != null && !status.toLowerCase().equals("ok")) { final String reason = playabilityStatus.getString("reason"); throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } From 6aa4d59b91009fe5786612f5b8018df556dbf845 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sun, 1 Mar 2020 10:22:49 -0300 Subject: [PATCH 137/149] [PeerTube] Update video used in age limit test (the previous was 404) --- .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 23adf353..e103e1a8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -145,7 +145,7 @@ public class PeertubeStreamExtractorDefaultTest { @Test public void testGetAgeLimit() throws ExtractionException, IOException { assertEquals(0, extractor.getAgeLimit()); - PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7"); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/3c0da7fb-e4d9-442e-84e3-a8c47004ee28"); ageLimit.fetchPage(); assertEquals(18, ageLimit.getAgeLimit()); } From cc8fb486ee396d907465c0509864546ac27305c1 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 15 Feb 2020 09:30:05 +0100 Subject: [PATCH 138/149] Handle youtube stream urls in "vnd.youtube:videoId" format. The official YouTube app accept intents in this format, which causes other developers to use it. --- .../YoutubeCommentsLinkHandlerFactory.java | 12 ++++++++++++ .../youtube/linkHandler/YoutubeParsingHelper.java | 7 ++++++- .../linkHandler/YoutubeStreamLinkHandlerFactory.java | 12 ++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java index 421fc13f..43baf6a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java @@ -1,7 +1,10 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; + import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import java.util.List; @@ -14,6 +17,15 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { return instance; } + @Override + public ListLinkHandler fromUrl(String url) throws ParsingException { + if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){ + return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL); + } else { + return super.fromUrl(url); + } + } + @Override public String getUrl(String id) { return "https://m.youtube.com/watch?v=" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index d2b0a7da..96532555 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -53,10 +53,15 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + /** + * The official youtube app supports intents in this format, where after the ':' is the videoId. + * Accordingly there are other apps sharing streams in this format. + */ + public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; - private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index e3ac4d52..c0728133 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -1,7 +1,10 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; + import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -49,6 +52,15 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return id; } + @Override + public LinkHandler fromUrl(String url) throws ParsingException { + if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){ + return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL); + } else { + return super.fromUrl(url); + } + } + @Override public String getUrl(String id) { return "https://www.youtube.com/watch?v=" + id; From 1ef706f5672ac7790601ee407362b574c7e35e03 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 7 Mar 2020 16:42:00 -0300 Subject: [PATCH 139/149] [YouTube] Detect LIVE videos in the trending page --- .../extractors/YoutubeStreamInfoItemExtractor.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 8725e554..00aa8ae2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -63,6 +63,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } } catch (Exception ignored) {} + + try { + final String style = videoInfo.getArray("thumbnailOverlays").getObject(0) + .getObject("thumbnailOverlayTimeStatusRenderer").getString("style"); + if (style.equalsIgnoreCase("LIVE")) { + return StreamType.LIVE_STREAM; + } + } catch (Exception ignored) {} + return StreamType.VIDEO_STREAM; } From 70abd5785286f062140faf17ab6d43b20dd75397 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 7 Mar 2020 16:46:00 -0300 Subject: [PATCH 140/149] [YouTube] Cache commonly used stream type result --- .../extractors/YoutubeStreamInfoItemExtractor.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 00aa8ae2..2330130e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -40,6 +40,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private JsonObject videoInfo; private final TimeAgoParser timeAgoParser; + private StreamType cachedStreamType; /** * Creates an extractor of StreamInfoItems from a YouTube page. @@ -54,11 +55,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public StreamType getStreamType() { + if (cachedStreamType != null) { + return cachedStreamType; + } + try { JsonArray badges = videoInfo.getArray("badges"); for (Object badge : badges) { if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) { - return StreamType.LIVE_STREAM; + return cachedStreamType = StreamType.LIVE_STREAM; } } @@ -68,11 +73,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { final String style = videoInfo.getArray("thumbnailOverlays").getObject(0) .getObject("thumbnailOverlayTimeStatusRenderer").getString("style"); if (style.equalsIgnoreCase("LIVE")) { - return StreamType.LIVE_STREAM; + return cachedStreamType = StreamType.LIVE_STREAM; } } catch (Exception ignored) {} - return StreamType.VIDEO_STREAM; + return cachedStreamType = StreamType.VIDEO_STREAM; } @Override From e8e535b815e6a99dee08d1f3c2536868c8c03de0 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 12 Feb 2020 00:25:34 +0100 Subject: [PATCH 141/149] mediaccc: update linkhandlers & refactor tests --- .../extractors/MediaCCCStreamExtractor.java | 5 +- .../MediaCCCConferenceLinkHandlerFactory.java | 46 +++- .../MediaCCCStreamLinkHandlerFactory.java | 41 +++- .../MediaCCCConferenceExtractorTest.java | 91 +++++--- .../MediaCCCStreamExtractorTest.java | 204 ++++++++++++------ 5 files changed, 279 insertions(+), 108 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index c499758d..424e8fdd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -170,15 +170,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor { return null; } - @Nonnull @Override public List getSubtitlesDefault() throws IOException, ExtractionException { return null; } - @Nonnull @Override - public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { + public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { return null; } @@ -212,7 +210,6 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } catch (JsonParserException jpe) { throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe); } - } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index 3c181767..4d77b072 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -1,9 +1,13 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; +import java.net.MalformedURLException; +import java.net.URL; import java.util.List; public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { @@ -14,19 +18,43 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory } @Override - public String getId(String url) throws ParsingException { - if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { - return url.replace("https://api.media.ccc.de/public/conferences/", ""); - } else if (url.startsWith("https://media.ccc.de/c/")) { - return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); - } else { - throw new ParsingException("Could not get id from url: " + url); + public String getId(String urlString) throws ParsingException { + if (urlString.startsWith("https://api.media.ccc.de/public/conferences/")) { + return urlString.replace("https://api.media.ccc.de/public/conferences/", ""); + } else if (urlString.startsWith("https://media.ccc.de/c/")) { + return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", urlString); } + + URL url; + try { + url = Utils.stringToURL(urlString); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("The given URL is not valid"); + } + + String path = url.getPath(); + // remove leading "/" of URL-path if URL-path is given + if (!path.isEmpty()) { + path = path.substring(1); + } + + if (path.contains("b/")) { + return path.substring(2); + } + + throw new ParsingException("Could not get id from url: " + url); + } @Override public boolean onAcceptUrl(String url) throws ParsingException { - return url.startsWith("https://api.media.ccc.de/public/conferences/") - || url.startsWith("https://media.ccc.de/c/"); + try { + getId(url); + return true; + } catch (FoundAdException fe) { + throw fe; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index bf291a60..f7262f0c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -1,16 +1,39 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.MalformedURLException; +import java.net.URL; public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override - public String getId(String url) throws ParsingException { - if (url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q=")) { - return url.replace("https://api.media.ccc.de/public/events/", ""); + public String getId(String urlString) throws ParsingException { + if (urlString.startsWith("https://api.media.ccc.de/public/events/") && + !urlString.contains("?q=")) { + return urlString.replace("https://api.media.ccc.de/public/events/", ""); } + + URL url; + try { + url = Utils.stringToURL(urlString); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("The given URL is not valid"); + } + + String path = url.getPath(); + // remove leading "/" of URL-path if URL-path is given + if (!path.isEmpty()) { + path = path.substring(1); + } + + if (path.contains("v/")) { + return path.substring(2); + } + throw new ParsingException("Could not get id from url: " + url); } @@ -21,7 +44,13 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public boolean onAcceptUrl(String url) throws ParsingException { - return url.startsWith("https://api.media.ccc.de/public/events/") && - !url.contains("?q="); + try { + getId(url); + return true; + } catch (FoundAdException fe) { + throw fe; + } catch (ParsingException e) { + return false; + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index 15a68a73..a6e78885 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -4,47 +4,86 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test {@link MediaCCCConferenceExtractor} */ public class MediaCCCConferenceExtractorTest { - private static ChannelExtractor extractor; - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017"); - extractor.fetchPage(); + public static class FrOSCon2017 { + private static MediaCCCConferenceExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/froscon2017"); + extractor.fetchPage(); + } + + @Test + public void testName() throws Exception { + assertEquals("FrOSCon 2017", extractor.getName()); + } + + @Test + public void testGetUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); + } + + @Test + public void testGetOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); + } + + @Test + public void testGetThumbnailUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); + } + + @Test + public void testGetInitalPage() throws Exception { + assertEquals(97, extractor.getInitialPage().getItems().size()); + } } - @Test - public void testName() throws Exception { - assertEquals("FrOSCon 2017", extractor.getName()); - } + public static class Oscal2019 { + private static MediaCCCConferenceExtractor extractor; - @Test - public void testGetUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); - } + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCConferenceExtractor) MediaCCC.getChannelExtractor("https://media.ccc.de/c/oscal19"); + extractor.fetchPage(); + } - @Test - public void testGetOriginalUrl() throws Exception { - assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); - } + @Test + public void testName() throws Exception { + assertEquals("Open Source Conference Albania 2019", extractor.getName()); + } - @Test - public void testGetThumbnailUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); - } + @Test + public void testGetUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/oscal19", extractor.getUrl()); + } - @Test - public void testGetInitalPage() throws Exception { - assertEquals(97, extractor.getInitialPage().getItems().size()); + @Test + public void testGetOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/c/oscal19", extractor.getOriginalUrl()); + } + + @Test + public void testGetThumbnailUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/oscal/2019/oscal-19.png", extractor.getAvatarUrl()); + } + + @Test + public void testGetInitalPage() throws Exception { + assertTrue(extractor.getInitialPage().getItems().size() >= 21); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index ed9dcee7..195656b1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -1,101 +1,179 @@ package org.schabi.newpipe.extractor.services.media_ccc; +import com.grack.nanojson.JsonObject; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.BaseExtractorTest; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamExtractor; +import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; +import static org.junit.Assert.assertFalse; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test {@link MediaCCCStreamExtractor} */ -public class MediaCCCStreamExtractorTest implements BaseExtractorTest { - private static StreamExtractor extractor; +public class MediaCCCStreamExtractorTest { - @BeforeClass - public static void setUpClass() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); + public static class Gpn18Tmux { + private static MediaCCCStreamExtractor extractor; - extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); - extractor.fetchPage(); + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + + extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht"); + extractor.fetchPage(); + } + + @Test + public void testServiceId() throws Exception { + assertEquals(2, extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId()); + } + + @Test + public void testUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl()); + } + + @Test + public void testThumbnail() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); + } + + @Test + public void testUploaderName() throws Exception { + assertEquals("gpn18", extractor.getUploaderName()); + } + + @Test + public void testUploaderUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); + } + + @Test + public void testUploaderAvatarUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); + } + + @Test + public void testVideoStreams() throws Exception { + assertEquals(4, extractor.getVideoStreams().size()); + } + + @Test + public void testAudioStreams() throws Exception { + assertEquals(2, extractor.getAudioStreams().size()); + } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } } - @Override - public void testServiceId() throws Exception { - assertEquals(2, extractor.getServiceId()); - } + public static class _36c3PrivacyMessaging { + private static MediaCCCStreamExtractor extractor; - @Override - public void testName() throws Exception { - assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); - } + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging"); + extractor.fetchPage(); + } - @Override - public void testId() throws Exception { - assertEquals("", extractor.getId()); - } + @Test + public void testName() throws Exception { + assertEquals("What's left for private messaging?", extractor.getName()); + } - @Override - public void testUrl() throws Exception { - assertEquals("", extractor.getUrl()); - } + @Test + public void testId() throws Exception { + assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId()); + } - @Override - public void testOriginalUrl() throws Exception { - assertEquals("", extractor.getOriginalUrl()); - } + @Test + public void testUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); + } - @Test - public void testThumbnail() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); - } + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl()); + } - @Test - public void testUploaderName() throws Exception { - assertEquals("gpn18", extractor.getUploaderName()); - } + @Test + public void testThumbnail() throws Exception { + assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); + } - @Test - public void testUploaderUrl() throws Exception { - assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); - } + @Test + public void testUploaderName() throws Exception { + assertEquals("36c3", extractor.getUploaderName()); + } - @Test - public void testUploaderAvatarUrl() throws Exception { - assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); - } + @Test + public void testUploaderUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); + } - @Test - public void testVideoStreams() throws Exception { - assertEquals(4, extractor.getVideoStreams().size()); - } + @Test + public void testUploaderAvatarUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); + } - @Test - public void testAudioStreams() throws Exception { - assertEquals(2, extractor.getAudioStreams().size()); - } + @Test + public void testVideoStreams() throws Exception { + assertEquals(8, extractor.getVideoStreams().size()); + } - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); - } + @Test + public void testAudioStreams() throws Exception { + assertEquals(2, extractor.getAudioStreams().size()); + } - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + } } -} +} \ No newline at end of file From 9701c7d800ae91fda88791dc21e02ee80888c030 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 12 Feb 2020 18:59:46 +0100 Subject: [PATCH 142/149] invidious shortened links --- .../linkHandler/YoutubeStreamLinkHandlerFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index e3ac4d52..15e99894 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -190,7 +190,12 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return assertIsID(id); } - break; + String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsID(viewQueryValue); + } + + return assertIsID(path); } } From f742a6bd3ea68a1d243aeceae5092703e92c3cc9 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 12 Feb 2020 21:24:50 +0100 Subject: [PATCH 143/149] code optimization --- .../MediaCCCConferenceLinkHandlerFactory.java | 37 ++++--------------- .../MediaCCCStreamLinkHandlerFactory.java | 4 +- .../MediaCCCConferenceExtractorTest.java | 2 +- .../MediaCCCStreamExtractorTest.java | 4 -- 4 files changed, 9 insertions(+), 38 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index 4d77b072..f4074c0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -1,13 +1,9 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Parser; -import org.schabi.newpipe.extractor.utils.Utils; -import java.net.MalformedURLException; -import java.net.URL; import java.util.List; public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { @@ -18,32 +14,15 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory } @Override - public String getId(String urlString) throws ParsingException { - if (urlString.startsWith("https://api.media.ccc.de/public/conferences/")) { - return urlString.replace("https://api.media.ccc.de/public/conferences/", ""); - } else if (urlString.startsWith("https://media.ccc.de/c/")) { - return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", urlString); + public String getId(String url) throws ParsingException { + if (url.startsWith("https://api.media.ccc.de/public/conferences/")) { + return url.replace("https://api.media.ccc.de/public/conferences/", ""); + } else if (url.startsWith("https://media.ccc.de/c/")) { + return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); + } else if (url.startsWith("https://media.ccc.de/b/")) { + return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url); } - - URL url; - try { - url = Utils.stringToURL(urlString); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("The given URL is not valid"); - } - - String path = url.getPath(); - // remove leading "/" of URL-path if URL-path is given - if (!path.isEmpty()) { - path = path.substring(1); - } - - if (path.contains("b/")) { - return path.substring(2); - } - throw new ParsingException("Could not get id from url: " + url); - } @Override @@ -51,8 +30,6 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory try { getId(url); return true; - } catch (FoundAdException fe) { - throw fe; } catch (ParsingException e) { return false; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index f7262f0c..76e5df89 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -30,7 +30,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { path = path.substring(1); } - if (path.contains("v/")) { + if (path.startsWith("v/")) { return path.substring(2); } @@ -47,8 +47,6 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { try { getId(url); return true; - } catch (FoundAdException fe) { - throw fe; } catch (ParsingException e) { return false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index a6e78885..258a55b0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 195656b1..9cb85662 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -1,23 +1,19 @@ package org.schabi.newpipe.extractor.services.media_ccc; -import com.grack.nanojson.JsonObject; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; -import static org.junit.Assert.assertFalse; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** From 0cd5e05b7b927a3d53048e5347a4f37fd8920efa Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 2 Mar 2020 22:38:44 +0100 Subject: [PATCH 144/149] MediaCCCLH: use substring instead of replace; improve a bit tests, return Collections.emptyList(); instead of null where it's annotated @NonNull --- .../extractors/MediaCCCStreamExtractor.java | 11 +++++++---- .../MediaCCCStreamLinkHandlerFactory.java | 2 +- .../media_ccc/MediaCCCStreamExtractorTest.java | 12 ++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 424e8fdd..59bfe1f4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.*; import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; @@ -104,13 +105,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public String getDashMpdUrl() throws ParsingException { - return null; + return ""; } @Nonnull @Override public String getHlsUrl() throws ParsingException { - return null; + return ""; } @Override @@ -170,14 +171,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor { return null; } + @Nonnull @Override public List getSubtitlesDefault() throws IOException, ExtractionException { - return null; + return Collections.emptyList(); } + @Nonnull @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { - return null; + return Collections.emptyList(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index 76e5df89..2c91d056 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -14,7 +14,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { public String getId(String urlString) throws ParsingException { if (urlString.startsWith("https://api.media.ccc.de/public/events/") && !urlString.contains("?q=")) { - return urlString.replace("https://api.media.ccc.de/public/events/", ""); + return urlString.substring(39); //remove api…/public/events part } URL url; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 9cb85662..c0f51437 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; +import org.schabi.newpipe.extractor.utils.UtilsTest; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -14,6 +15,7 @@ import java.util.Calendar; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** @@ -49,16 +51,19 @@ public class MediaCCCStreamExtractorTest { @Test public void testUrl() throws Exception { + assertIsSecureUrl(extractor.getUrl()); assertEquals("https://api.media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); } @Test public void testOriginalUrl() throws Exception { + assertIsSecureUrl(extractor.getOriginalUrl()); assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl()); } @Test public void testThumbnail() throws Exception { + assertIsSecureUrl(extractor.getThumbnailUrl()); assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); } @@ -69,11 +74,13 @@ public class MediaCCCStreamExtractorTest { @Test public void testUploaderUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderUrl()); assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); } @Test public void testUploaderAvatarUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); } @@ -122,16 +129,19 @@ public class MediaCCCStreamExtractorTest { @Test public void testUrl() throws Exception { + assertIsSecureUrl(extractor.getUrl()); assertEquals("https://api.media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); } @Test public void testOriginalUrl() throws Exception { + assertIsSecureUrl(extractor.getOriginalUrl()); assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl()); } @Test public void testThumbnail() throws Exception { + assertIsSecureUrl(extractor.getThumbnailUrl()); assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); } @@ -142,11 +152,13 @@ public class MediaCCCStreamExtractorTest { @Test public void testUploaderUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderUrl()); assertEquals("https://api.media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); } @Test public void testUploaderAvatarUrl() throws Exception { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); } From 66518ec44452f2fd3916805a07d3959409b0e16c Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 13 Mar 2020 20:07:36 +0100 Subject: [PATCH 145/149] check wether mediaccc streams url are secure --- .../MediaCCCStreamExtractorTest.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index c0f51437..8120e07a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -7,11 +7,14 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.UtilsTest; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.List; import static java.util.Objects.requireNonNull; import static junit.framework.TestCase.assertEquals; @@ -86,12 +89,20 @@ public class MediaCCCStreamExtractorTest { @Test public void testVideoStreams() throws Exception { - assertEquals(4, extractor.getVideoStreams().size()); + List videoStreamList = extractor.getVideoStreams(); + assertEquals(4, videoStreamList.size()); + for (VideoStream stream : videoStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test public void testAudioStreams() throws Exception { - assertEquals(2, extractor.getAudioStreams().size()); + List audioStreamList = extractor.getAudioStreams(); + assertEquals(2, audioStreamList.size()); + for (AudioStream stream : audioStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test @@ -164,12 +175,20 @@ public class MediaCCCStreamExtractorTest { @Test public void testVideoStreams() throws Exception { - assertEquals(8, extractor.getVideoStreams().size()); + List videoStreamList = extractor.getVideoStreams(); + assertEquals(8, videoStreamList.size()); + for (VideoStream stream : videoStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test public void testAudioStreams() throws Exception { - assertEquals(2, extractor.getAudioStreams().size()); + List audioStreamList = extractor.getAudioStreams(); + assertEquals(2, audioStreamList.size()); + for (AudioStream stream : audioStreamList) { + assertIsSecureUrl(stream.getUrl()); + } } @Test From aea0962cfe87a2581321c14d9536ecfbdcfaba8a Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 14 Mar 2020 12:41:44 +0100 Subject: [PATCH 146/149] [SoundCloud] Update hardcoded client id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index ed50ea54..b8f6af24 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 + private static final String HARDCODED_CLIENT_ID = "t0h1jzYMsaZXy6ggnZO71gHK3Ms6CFwE"; // Updated on 14/03/20 private static String clientId; private SoundcloudParsingHelper() { From 325a51f73ab68bf52dfefe640a89472f6093abb9 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 14 Mar 2020 14:02:48 +0100 Subject: [PATCH 147/149] [YouTube] Do not crash if there is no next stream, but return null Like every other extractor does if there is no next stream --- .../extractors/YoutubeStreamExtractor.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8ee50201..04818de0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -523,10 +523,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (getAgeLimit() != NO_AGE_LIMIT) return null; try { - final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("secondaryResults").getObject("secondaryResults").getArray("results") - .getObject(0).getObject("compactAutoplayRenderer").getArray("contents") - .getObject(0).getObject("compactVideoRenderer"); + final JsonObject firstWatchNextItem = initialData.getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("secondaryResults") + .getObject("secondaryResults").getArray("results").getObject(0); + + if (!firstWatchNextItem.has("compactAutoplayRenderer")) { + // there is no "next" stream + return null; + } + + final JsonObject videoInfo = firstWatchNextItem.getObject("compactAutoplayRenderer") + .getArray("contents").getObject(0).getObject("compactVideoRenderer"); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); From 957b78ed9b8078e385eab6c1a5fe263606faf27d Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 14 Mar 2020 14:04:01 +0100 Subject: [PATCH 148/149] [YouTube] Add test for an unlisted video, which has no next stream --- .../YoutubeStreamExtractorUnlistedTest.java | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java new file mode 100644 index 00000000..5dcd73ec --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java @@ -0,0 +1,166 @@ +package org.schabi.newpipe.extractor.services.youtube.stream; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.List; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeStreamExtractorUnlistedTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=udsB8KnIJTg"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetTitle() throws ParsingException { + assertFalse(extractor.getName().isEmpty()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().getContent().isEmpty()); + } + + @Test + public void testGetFullLinksInDescription() throws ParsingException { + assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/user/Roccowschiptune")); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertNotNull(extractor.getUploaderName()); + assertFalse(extractor.getUploaderName().isEmpty()); + } + + + @Test + public void testGetLength() throws ParsingException { + assertEquals(2488, extractor.getLength()); + } + + @Test + public void testGetViewCount() throws ParsingException { + long count = extractor.getViewCount(); + assertTrue(Long.toString(count), count >= /* specific to that video */ 1225); + } + + @Test + public void testGetTextualUploadDate() throws ParsingException { + Assert.assertEquals("2017-09-22", extractor.getTextualUploadDate()); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-09-22")); + assertNotNull(extractor.getUploadDate()); + assertEquals(instance, extractor.getUploadDate().date()); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetAudioStreams() throws ExtractionException { + List audioStreams = extractor.getAudioStreams(); + assertFalse(audioStreams.isEmpty()); + for (AudioStream s : audioStreams) { + assertIsSecureUrl(s.url); + assertTrue(Integer.toString(s.getFormatId()), + 0x100 <= s.getFormatId() && s.getFormatId() < 0x1000); + } + } + + @Test + public void testGetVideoStreams() throws ExtractionException { + for (VideoStream s : extractor.getVideoStreams()) { + assertIsSecureUrl(s.url); + assertTrue(s.resolution.length() > 0); + assertTrue(Integer.toString(s.getFormatId()), + 0 <= s.getFormatId() && s.getFormatId() < 0x100); + } + } + + @Test + public void testStreamType() throws ParsingException { + assertSame(StreamType.VIDEO_STREAM, extractor.getStreamType()); + } + + @Test + public void testGetNextVideo() throws ExtractionException { + assertNull(extractor.getNextStream()); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); + Utils.printErrors(relatedVideos.getErrors()); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() { + assertFalse(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() { + assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + } + + @Test + public void testGetLikeCount() throws ParsingException { + long likeCount = extractor.getLikeCount(); + assertTrue("" + likeCount, likeCount >= 96); + } + + @Test + public void testGetDislikeCount() throws ParsingException { + long dislikeCount = extractor.getDislikeCount(); + assertTrue("" + dislikeCount, dislikeCount >= 0); + } +} From 40b3bc138f830802991f6e2ae1f1e203383ee1e3 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 14 Mar 2020 21:54:54 +0100 Subject: [PATCH 149/149] Remove hack to fix YoutubeStreamLinkHandlerFactoryTest of vnd.youtube videos --- .../services/youtube/YoutubeStreamLinkHandlerFactoryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index aebb6bf1..908642fd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -77,8 +77,8 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", "youtube.com").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI", "youtube.com").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); }