diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 8f2fe79a..1ec47498 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -28,6 +28,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; public abstract class ChannelExtractor extends ListExtractor { + public static final long UNKNOWN_SUBSCRIBER_COUNT = -1; + public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d7ba0712..acc1fa11 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -89,9 +89,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { // we couldn't get information about the channel associated with this URL, if there is one. if (!channelId[0].equals("channel")) { final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - getExtractorLocalization(), getExtractorContentCountry()) - .value("url", "https://www.youtube.com/" + channelPath) - .done()) + getExtractorLocalization(), getExtractorContentCountry()) + .value("url", "https://www.youtube.com/" + channelPath) + .done()) .getBytes(UTF_8); final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url", @@ -104,8 +104,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ContentNotAvailableException("This channel doesn't exist."); } else { throw new ContentNotAvailableException("Got error:\"" - + errorJsonObject.getString("status") + "\": " - + errorJsonObject.getString("message")); + + errorJsonObject.getString("status") + "\": " + + errorJsonObject.getString("message")); } } @@ -136,10 +136,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { int level = 0; while (level < 3) { final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - getExtractorLocalization(), getExtractorContentCountry()) - .value("browseId", id) - .value("params", "EgZ2aWRlb3M%3D") // Equal to videos - .done()) + getExtractorLocalization(), getExtractorContentCountry()) + .value("browseId", id) + .value("params", "EgZ2aWRlb3M%3D") // Equal to videos + .done()) .getBytes(UTF_8); final JsonObject jsonResponse = getJsonPostResponse("browse", body, @@ -273,15 +273,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public long getSubscriberCount() throws ParsingException { final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header") .getObject("c4TabbedHeaderRenderer"); - if (c4TabbedHeaderRenderer.has("subscriberCountText")) { - try { - return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer - .getObject("subscriberCountText"))); - } catch (final NumberFormatException e) { - throw new ParsingException("Could not get subscriber count", e); - } - } else { - return ITEM_COUNT_UNKNOWN; + if (!c4TabbedHeaderRenderer.has("subscriberCountText")) { + return UNKNOWN_SUBSCRIBER_COUNT; + } + try { + return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer + .getObject("subscriberCountText"))); + } catch (final NumberFormatException e) { + throw new ParsingException("Could not get subscriber count", e); } } @@ -385,9 +384,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getString("token"); final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), - getExtractorContentCountry()) - .value("continuation", continuation) - .done()) + getExtractorContentCountry()) + .value("continuation", continuation) + .done()) .getBytes(UTF_8); return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 6c7ac66b..7c728af6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -180,7 +180,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { // TODO: this parses English formatted dates only, we need a better approach to parse // the textual date final LocalDate localDate = LocalDate.parse(getTextFromObject( - getVideoPrimaryInfoRenderer().getObject("dateText")), + getVideoPrimaryInfoRenderer().getObject("dateText")), DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH)); return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); } catch (final Exception ignored) { @@ -437,6 +437,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { return fixThumbnailUrl(url); } + @Override + public long getUploaderSubscriberCount() throws ParsingException { + final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, "owner.videoOwnerRenderer"); + if (!videoOwnerRenderer.has("subscriberCountText")) { + return UNKNOWN_SUBSCRIBER_COUNT; + } + try { + return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText"))); + } catch (final NumberFormatException e) { + throw new ParsingException("Could not get uploader subscriber count", e); + } + } + @Nonnull @Override public String getDashMpdUrl() throws ParsingException { @@ -666,9 +679,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { final Localization localization = getExtractorLocalization(); final ContentCountry contentCountry = getExtractorContentCountry(); final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - localization, contentCountry) - .value("videoId", videoId) - .done()) + localization, contentCountry) + .value("videoId", videoId) + .done()) .getBytes(UTF_8); // Put the sts string if we already know it so we don't have to fetch again the player @@ -717,8 +730,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (ageRestricted) { final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder( - localization, contentCountry, videoId) - .done()) + localization, contentCountry, videoId) + .done()) .getBytes(UTF_8); nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization); } else { @@ -800,7 +813,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String videoId) throws IOException, ExtractionException { final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder( - localization, contentCountry) + localization, contentCountry) .value("videoId", videoId) .done()) .getBytes(UTF_8); @@ -873,8 +886,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String videoId) throws IOException, ExtractionException { final byte[] androidMobileEmbedBody = JsonWriter.string( - prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId) - .done()) + prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId) + .done()) .getBytes(UTF_8); final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player", androidMobileEmbedBody, contentCountry, localization); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index b78f8664..939b4f5a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -47,6 +47,7 @@ import java.util.Locale; public abstract class StreamExtractor extends Extractor { public static final int NO_AGE_LIMIT = 0; + public static final long UNKNOWN_SUBSCRIBER_COUNT = -1; public StreamExtractor(StreamingService service, LinkHandler linkHandler) { super(service, linkHandler); @@ -202,6 +203,17 @@ public abstract class StreamExtractor extends Extractor { return false; } + /** + * The subscriber count of the uploader. + * If the subscriber count is not implemented, or is unavailable, return -1. + * + * @return the subscriber count of the uploader or {@value UNKNOWN_SUBSCRIBER_COUNT} if not available + * @throws ParsingException + */ + public long getUploaderSubscriberCount() throws ParsingException { + return UNKNOWN_SUBSCRIBER_COUNT; + } + /** * The url to the image file/profile picture/avatar of the creator/uploader of the stream. * If the url is not available you can return an empty String. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 455f0dc4..42ca495c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -235,6 +235,11 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setUploaderSubscriberCount(extractor.getUploaderSubscriberCount()); + } catch (Exception e) { + streamInfo.addError(e); + } try { streamInfo.setSubChannelName(extractor.getSubChannelName()); @@ -367,6 +372,7 @@ public class StreamInfo extends Info { private String uploaderUrl = ""; private String uploaderAvatarUrl = ""; private boolean uploaderVerified = false; + private long uploaderSubscriberCount = -1; private String subChannelName = ""; private String subChannelUrl = ""; @@ -540,6 +546,14 @@ public class StreamInfo extends Info { this.uploaderVerified = uploaderVerified; } + public long getUploaderSubscriberCount() { + return uploaderSubscriberCount; + } + + public void setUploaderSubscriberCount(long uploaderSubscriberCount) { + this.uploaderSubscriberCount = uploaderSubscriberCount; + } + public String getSubChannelName() { return subChannelName; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java index f32a6090..da418660 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -5,6 +5,7 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { void testUploaderName() throws Exception; void testUploaderUrl() throws Exception; void testUploaderAvatarUrl() throws Exception; + void testSubscriberCount() throws Exception; void testSubChannelName() throws Exception; void testSubChannelUrl() throws Exception; void testSubChannelAvatarUrl() throws Exception; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 1563abac..f8fb6e93 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -34,6 +34,7 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderInd import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; +import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT; /** * Test for {@link StreamExtractor} @@ -45,6 +46,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedDescriptionContains() { return Collections.emptyList(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index b7994036..73d7648a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -49,6 +49,7 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Amazing Atheist"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 977_000; } @Override public List expectedDescriptionContains() { return Arrays.asList("http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html", "freedom"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 6b3f1c71..67e80c29 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -138,6 +138,7 @@ public class YoutubeStreamExtractorDefaultTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "PewDiePie"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 110_000_000; } @Override public List expectedDescriptionContains() { return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA", "https://www.handcraftpictures.com/"); @@ -182,6 +183,7 @@ public class YoutubeStreamExtractorDefaultTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Unbox Therapy"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 18_000_000; } @Override public List expectedDescriptionContains() { return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", "https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", @@ -274,6 +276,7 @@ public class YoutubeStreamExtractorDefaultTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "tagesschau"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC5NOEUbkLheQcaaRldYW5GA"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_000_000; } @Override public boolean expectedUploaderVerified() { return true; } @Override public List expectedDescriptionContains() { return Arrays.asList("Themen der Sendung", "07:15", "Wetter", "Sendung nachträglich bearbeitet"); @@ -336,6 +339,7 @@ public class YoutubeStreamExtractorDefaultTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "maiLab"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_400_000; } @Override public List expectedDescriptionContains() {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");} @Override public boolean expectedUploaderVerified() { return true; } @Override public long expectedLength() { return 1010; } @@ -405,6 +409,7 @@ public class YoutubeStreamExtractorDefaultTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_500_000; } @Override public List expectedDescriptionContains() { return Arrays.asList("Lasst uns abtauchen!", "Angebot von funk", "Dinge"); } @Override public long expectedLength() { return 631; } @Override public long expectedTimestamp() { return TIMESTAMP; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index e21bb446..3a3e7f0b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -52,6 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor @Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; } @Override public String expectedUploaderName() { return "Lofi Girl"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 9_800_000; } @Override public List expectedDescriptionContains() { return Arrays.asList("Lofi Girl merch", "Thank you for listening, I hope you will have a good time here"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java index 5db8958c..a3f2b0b2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java @@ -45,6 +45,7 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Hooked"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw"; } + @Override public long expectedUploaderSubscriberCountAtLeast() { return 24_300; } @Override public List expectedDescriptionContains() { return Arrays.asList("https://www.youtube.com/user/Roccowschiptune", "https://www.facebook.com/HookedMagazinDE");