From b9fad4fcc81003c526c44a1aa36edd3e40b4bc94 Mon Sep 17 00:00:00 2001 From: FireMasterK <20838718+FireMasterK@users.noreply.github.com> Date: Thu, 2 Sep 2021 00:10:00 +0530 Subject: [PATCH] Extract Uploader's Avatar on YouTube. --- .../BandcampRadioInfoItemExtractor.java | 6 + ...campDiscographStreamInfoItemExtractor.java | 8 + ...ndcampPlaylistStreamInfoItemExtractor.java | 7 + ...BandcampSearchStreamInfoItemExtractor.java | 8 + .../MediaCCCLiveStreamKioskExtractor.java | 6 + .../MediaCCCRecentKioskExtractor.java | 6 + .../MediaCCCStreamInfoItemExtractor.java | 6 + .../PeertubeStreamInfoItemExtractor.java | 8 + .../SoundcloudStreamInfoItemExtractor.java | 8 + .../YoutubeFeedInfoItemExtractor.java | 6 + .../YoutubeStreamInfoItemExtractor.java | 14 +- .../extractor/stream/StreamInfoItem.java | 10 + .../stream/StreamInfoItemExtractor.java | 9 + .../stream/StreamInfoItemsCollector.java | 5 + .../search/YoutubeSearchExtractorTest.java | 53 +- .../generated_mock_0.json | 217 +++++ .../generated_mock_1.json | 246 ++++++ .../generated_mock_2.json | 775 ++++++++++++++++++ 18 files changed, 1382 insertions(+), 16 deletions(-) create mode 100644 extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/search/video_uploader_avatar/generated_mock_0.json create mode 100644 extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/search/video_uploader_avatar/generated_mock_1.json create mode 100644 extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/search/video_uploader_avatar/generated_mock_2.json diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index 7ecc3184..5f670177 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -79,6 +79,12 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { return ""; } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 15b04498..e6c8d4fa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -4,6 +4,8 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; +import javax.annotation.Nullable; + public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { private final JsonObject discograph; @@ -18,6 +20,12 @@ public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInf return discograph.getString("band_name"); } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public String getName() { return discograph.getString("title"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index e90a12aa..ac21794c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import javax.annotation.Nullable; import java.io.IOException; @@ -53,6 +54,12 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI return ""; } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + /** * Each track can have its own cover art. Therefore, unless a substitute is provided, * the thumbnail is extracted using a stream extractor. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index adeea051..abe57d20 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nullable; + public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor { private final Element resultInfo, searchResult; @@ -24,6 +26,12 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte } } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public String getName() throws ParsingException { return resultInfo.getElementsByClass("heading").text(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java index 504dd3b0..111ac9cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java @@ -73,6 +73,12 @@ public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor return "https://media.ccc.de/c/" + conferenceInfo.getString("slug"); } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java index 2ff86a4f..72bb10fa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java @@ -68,6 +68,12 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor { .getUrl(); // web URL } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 02316643..cc9f3d74 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -46,6 +46,12 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor return event.getString("conference_url"); } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index df6060bd..8a5a229c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -9,6 +9,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; +import javax.annotation.Nullable; + public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { protected final JsonObject item; @@ -54,6 +56,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor .fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 90f6efd8..e0686776 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -7,6 +7,8 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import javax.annotation.Nullable; + import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; @@ -43,6 +45,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url")); } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return itemObject.getObject("user").getBoolean("verified"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index a1883621..a0bd8917 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -50,6 +50,12 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { return entryElement.select("author > uri").first().text(); } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index f7cd38d9..344a01a4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; @@ -40,7 +41,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; */ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - private JsonObject videoInfo; + private final JsonObject videoInfo; private final TimeAgoParser timeAgoParser; private StreamType cachedStreamType; @@ -162,6 +163,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return url; } + @Nullable + @Override + public String getUploaderAvatarUrl() throws ParsingException { + + if(videoInfo.has("channelThumbnailSupportedRenderers")) { + return JsonUtils.getArray(videoInfo, "channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails").getObject(0).getString("url"); + } + + return null; + } + @Override public boolean isUploaderVerified() throws ParsingException { return YoutubeParsingHelper.isVerified(videoInfo.getArray("ownerBadges")); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java index 63c57415..67431884 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java @@ -39,6 +39,7 @@ public class StreamInfoItem extends InfoItem { private long duration = -1; private String uploaderUrl = null; + private String uploaderAvatarUrl = null; private boolean uploaderVerified = false; public StreamInfoItem(int serviceId, String url, String name, StreamType streamType) { @@ -82,6 +83,15 @@ public class StreamInfoItem extends InfoItem { this.uploaderUrl = uploaderUrl; } + @Nullable + public String getUploaderAvatarUrl() { + return uploaderAvatarUrl; + } + + public void setUploaderAvatarUrl(String uploaderAvatarUrl) { + this.uploaderAvatarUrl = uploaderAvatarUrl; + } + @Nullable public String getTextualUploadDate() { return textualUploadDate; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index c9e45170..6d61ee44 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -71,6 +71,15 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { String getUploaderUrl() throws ParsingException; + /** + * Get the uploader's avatar + * + * @return The uploader's avatar url or {@code null} if not provided by the service. + * @throws ParsingException if there is an error in the extraction + */ + @Nullable + String getUploaderAvatarUrl() throws ParsingException; + /** * Whether the uploader has been verified by the service's provider. * If there is no verification implemented, return false. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 464653a9..ae00a1b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -91,6 +91,11 @@ public class StreamInfoItemsCollector extends InfoItemsCollector