Merge pull request #796 from FireMasterK/streaminfo-subscriber-count
Add support for extracting channel subscriber count in StreamInfo
This commit is contained in:
commit
2aa5f98c26
12 changed files with 92 additions and 30 deletions
|
@ -28,6 +28,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
|||
|
||||
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
|
||||
|
||||
public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
|
||||
|
||||
public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
|
||||
super(service, linkHandler);
|
||||
}
|
||||
|
|
|
@ -89,9 +89,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
// we couldn't get information about the channel associated with this URL, if there is one.
|
||||
if (!channelId[0].equals("channel")) {
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("url", "https://www.youtube.com/" + channelPath)
|
||||
.done())
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("url", "https://www.youtube.com/" + channelPath)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
|
||||
|
@ -104,8 +104,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
throw new ContentNotAvailableException("This channel doesn't exist.");
|
||||
} else {
|
||||
throw new ContentNotAvailableException("Got error:\""
|
||||
+ errorJsonObject.getString("status") + "\": "
|
||||
+ errorJsonObject.getString("message"));
|
||||
+ errorJsonObject.getString("status") + "\": "
|
||||
+ errorJsonObject.getString("message"));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -136,10 +136,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
int level = 0;
|
||||
while (level < 3) {
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("browseId", id)
|
||||
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
|
||||
.done())
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("browseId", id)
|
||||
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
|
||||
|
@ -273,15 +273,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
public long getSubscriberCount() throws ParsingException {
|
||||
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header")
|
||||
.getObject("c4TabbedHeaderRenderer");
|
||||
if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
|
||||
try {
|
||||
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
|
||||
.getObject("subscriberCountText")));
|
||||
} catch (final NumberFormatException e) {
|
||||
throw new ParsingException("Could not get subscriber count", e);
|
||||
}
|
||||
} else {
|
||||
return ITEM_COUNT_UNKNOWN;
|
||||
if (!c4TabbedHeaderRenderer.has("subscriberCountText")) {
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
try {
|
||||
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
|
||||
.getObject("subscriberCountText")));
|
||||
} catch (final NumberFormatException e) {
|
||||
throw new ParsingException("Could not get subscriber count", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -385,9 +384,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
.getString("token");
|
||||
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
|
||||
getExtractorContentCountry())
|
||||
.value("continuation", continuation)
|
||||
.done())
|
||||
getExtractorContentCountry())
|
||||
.value("continuation", continuation)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);
|
||||
|
|
|
@ -180,7 +180,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
// TODO: this parses English formatted dates only, we need a better approach to parse
|
||||
// the textual date
|
||||
final LocalDate localDate = LocalDate.parse(getTextFromObject(
|
||||
getVideoPrimaryInfoRenderer().getObject("dateText")),
|
||||
getVideoPrimaryInfoRenderer().getObject("dateText")),
|
||||
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
|
||||
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
|
||||
} catch (final Exception ignored) {
|
||||
|
@ -437,6 +437,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
return fixThumbnailUrl(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getUploaderSubscriberCount() throws ParsingException {
|
||||
final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, "owner.videoOwnerRenderer");
|
||||
if (!videoOwnerRenderer.has("subscriberCountText")) {
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
try {
|
||||
return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText")));
|
||||
} catch (final NumberFormatException e) {
|
||||
throw new ParsingException("Could not get uploader subscriber count", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getDashMpdUrl() throws ParsingException {
|
||||
|
@ -666,9 +679,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
final Localization localization = getExtractorLocalization();
|
||||
final ContentCountry contentCountry = getExtractorContentCountry();
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
localization, contentCountry)
|
||||
.value("videoId", videoId)
|
||||
.done())
|
||||
localization, contentCountry)
|
||||
.value("videoId", videoId)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
||||
// Put the sts string if we already know it so we don't have to fetch again the player
|
||||
|
@ -717,8 +730,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
if (ageRestricted) {
|
||||
final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
|
||||
localization, contentCountry, videoId)
|
||||
.done())
|
||||
localization, contentCountry, videoId)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization);
|
||||
} else {
|
||||
|
@ -800,7 +813,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
final String videoId)
|
||||
throws IOException, ExtractionException {
|
||||
final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
|
||||
localization, contentCountry)
|
||||
localization, contentCountry)
|
||||
.value("videoId", videoId)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
|
@ -873,8 +886,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
final String videoId)
|
||||
throws IOException, ExtractionException {
|
||||
final byte[] androidMobileEmbedBody = JsonWriter.string(
|
||||
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
|
||||
.done())
|
||||
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
|
||||
androidMobileEmbedBody, contentCountry, localization);
|
||||
|
|
|
@ -47,6 +47,7 @@ import java.util.Locale;
|
|||
public abstract class StreamExtractor extends Extractor {
|
||||
|
||||
public static final int NO_AGE_LIMIT = 0;
|
||||
public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
|
||||
|
||||
public StreamExtractor(StreamingService service, LinkHandler linkHandler) {
|
||||
super(service, linkHandler);
|
||||
|
@ -202,6 +203,17 @@ public abstract class StreamExtractor extends Extractor {
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* The subscriber count of the uploader.
|
||||
* If the subscriber count is not implemented, or is unavailable, return <code>-1</code>.
|
||||
*
|
||||
* @return the subscriber count of the uploader or {@value UNKNOWN_SUBSCRIBER_COUNT} if not available
|
||||
* @throws ParsingException
|
||||
*/
|
||||
public long getUploaderSubscriberCount() throws ParsingException {
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
|
||||
/**
|
||||
* The url to the image file/profile picture/avatar of the creator/uploader of the stream.
|
||||
* If the url is not available you can return an empty String.
|
||||
|
|
|
@ -235,6 +235,11 @@ public class StreamInfo extends Info {
|
|||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
try {
|
||||
streamInfo.setUploaderSubscriberCount(extractor.getUploaderSubscriberCount());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
|
||||
try {
|
||||
streamInfo.setSubChannelName(extractor.getSubChannelName());
|
||||
|
@ -367,6 +372,7 @@ public class StreamInfo extends Info {
|
|||
private String uploaderUrl = "";
|
||||
private String uploaderAvatarUrl = "";
|
||||
private boolean uploaderVerified = false;
|
||||
private long uploaderSubscriberCount = -1;
|
||||
|
||||
private String subChannelName = "";
|
||||
private String subChannelUrl = "";
|
||||
|
@ -540,6 +546,14 @@ public class StreamInfo extends Info {
|
|||
this.uploaderVerified = uploaderVerified;
|
||||
}
|
||||
|
||||
public long getUploaderSubscriberCount() {
|
||||
return uploaderSubscriberCount;
|
||||
}
|
||||
|
||||
public void setUploaderSubscriberCount(long uploaderSubscriberCount) {
|
||||
this.uploaderSubscriberCount = uploaderSubscriberCount;
|
||||
}
|
||||
|
||||
public String getSubChannelName() {
|
||||
return subChannelName;
|
||||
}
|
||||
|
|
|
@ -5,6 +5,7 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest {
|
|||
void testUploaderName() throws Exception;
|
||||
void testUploaderUrl() throws Exception;
|
||||
void testUploaderAvatarUrl() throws Exception;
|
||||
void testSubscriberCount() throws Exception;
|
||||
void testSubChannelName() throws Exception;
|
||||
void testSubChannelUrl() throws Exception;
|
||||
void testSubChannelAvatarUrl() throws Exception;
|
||||
|
|
|
@ -34,6 +34,7 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderInd
|
|||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
|
||||
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
|
||||
import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT;
|
||||
|
||||
/**
|
||||
* Test for {@link StreamExtractor}
|
||||
|
@ -45,6 +46,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
|||
public abstract String expectedUploaderName();
|
||||
public abstract String expectedUploaderUrl();
|
||||
public boolean expectedUploaderVerified() { return false; }
|
||||
public long expectedUploaderSubscriberCountAtLeast() { return UNKNOWN_SUBSCRIBER_COUNT; }
|
||||
public String expectedSubChannelName() { return ""; } // default: there is no subchannel
|
||||
public String expectedSubChannelUrl() { return ""; } // default: there is no subchannel
|
||||
public boolean expectedDescriptionIsEmpty() { return false; } // default: description is not empty
|
||||
|
@ -105,6 +107,16 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
|||
assertEquals(expectedUploaderVerified(), extractor().isUploaderVerified());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Override
|
||||
public void testSubscriberCount() throws Exception {
|
||||
if (expectedUploaderSubscriberCountAtLeast() == UNKNOWN_SUBSCRIBER_COUNT) {
|
||||
assertEquals(UNKNOWN_SUBSCRIBER_COUNT, extractor().getUploaderSubscriberCount());
|
||||
} else {
|
||||
assertGreaterOrEqual(expectedUploaderSubscriberCountAtLeast(), extractor().getUploaderSubscriberCount());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Override
|
||||
public void testSubChannelName() throws Exception {
|
||||
|
|
|
@ -45,6 +45,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "DAN TV"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCcQHIVL83g5BEQe2IJFb-6w"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 50; }
|
||||
@Override public boolean expectedUploaderVerified() { return false; }
|
||||
@Override public boolean expectedDescriptionIsEmpty() { return true; }
|
||||
@Override public List<String> expectedDescriptionContains() { return Collections.emptyList(); }
|
||||
|
|
|
@ -49,6 +49,7 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "Amazing Atheist"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 977_000; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html",
|
||||
"freedom");
|
||||
|
|
|
@ -138,6 +138,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "PewDiePie"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 110_000_000; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA",
|
||||
"https://www.handcraftpictures.com/");
|
||||
|
@ -182,6 +183,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "Unbox Therapy"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 18_000_000; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
"https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
|
||||
|
@ -274,6 +276,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "tagesschau"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC5NOEUbkLheQcaaRldYW5GA"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_000_000; }
|
||||
@Override public boolean expectedUploaderVerified() { return true; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("Themen der Sendung", "07:15", "Wetter", "Sendung nachträglich bearbeitet");
|
||||
|
@ -336,6 +339,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "maiLab"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_400_000; }
|
||||
@Override public List<String> expectedDescriptionContains() {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");}
|
||||
@Override public boolean expectedUploaderVerified() { return true; }
|
||||
@Override public long expectedLength() { return 1010; }
|
||||
|
@ -405,6 +409,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_500_000; }
|
||||
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("Lasst uns abtauchen!", "Angebot von funk", "Dinge"); }
|
||||
@Override public long expectedLength() { return 631; }
|
||||
@Override public long expectedTimestamp() { return TIMESTAMP; }
|
||||
|
|
|
@ -52,6 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
|
|||
@Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "Lofi Girl"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 9_800_000; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("Lofi Girl merch",
|
||||
"Thank you for listening, I hope you will have a good time here");
|
||||
|
|
|
@ -45,6 +45,7 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
|
|||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "Hooked"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw"; }
|
||||
@Override public long expectedUploaderSubscriberCountAtLeast() { return 24_300; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("https://www.youtube.com/user/Roccowschiptune",
|
||||
"https://www.facebook.com/HookedMagazinDE");
|
||||
|
|
Loading…
Reference in a new issue