CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[YouTube] Parse watching count in live streams items

Browse source
This commit is contained in:
Mauricio Colli 2019-04-28 17:03:17 -03:00
parent 3638f0e0ea
commit d8280ce0da
No known key found for this signature in database
GPG key ID: F200BFD6F29DDD85
3 changed files with 86 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file

@ -304,12 +304,65 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public long getViewCount() throws ParsingException {
assertPageFetched();
try {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return getLiveStreamWatchingCount();
}
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
} catch (Exception e) {//todo: find fallback method
throw new ParsingException("Could not get number of views", e);
}
}
private long getLiveStreamWatchingCount() throws ExtractionException, IOException, JsonParserException {
// https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=
String innerTubeKey = null, clientVersion = null;
if (playerArgs != null && !playerArgs.isEmpty()) {
innerTubeKey = playerArgs.getString("innertube_api_key");
clientVersion = playerArgs.getString("innertube_context_client_version");
} else if (!videoInfoPage.isEmpty()) {
innerTubeKey = videoInfoPage.get("innertube_api_key");
clientVersion = videoInfoPage.get("innertube_context_client_version");
}
if (innerTubeKey == null || innerTubeKey.isEmpty()) {
throw new ExtractionException("Couldn't get innerTube key");
}
if (clientVersion == null || clientVersion.isEmpty()) {
throw new ExtractionException("Couldn't get innerTube client version");
}
final String metadataUrl = "https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=" + innerTubeKey;
final byte[] dataBody = ("{\"context\":{\"client\":{\"clientName\":1,\"clientVersion\":\"" + clientVersion + "\"}}" +
",\"videoId\":\"" + getId() + "\"}").getBytes("UTF-8");
final Response response = getDownloader().execute(Request.newBuilder()
.post(metadataUrl, dataBody)
.addHeader("Content-Type", "application/json")
.build());
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
for (Object actionEntry : jsonObject.getArray("actions")) {
if (!(actionEntry instanceof JsonObject)) continue;
final JsonObject entry = (JsonObject) actionEntry;
final JsonObject updateViewershipAction = entry.getObject("updateViewershipAction", null);
if (updateViewershipAction == null) continue;
final JsonArray viewCountRuns = JsonUtils.getArray(updateViewershipAction, "viewership.videoViewCountRenderer.viewCount.runs");
if (viewCountRuns.isEmpty()) continue;
final JsonObject textObject = viewCountRuns.getObject(0);
if (!textObject.has("text")) {
throw new ExtractionException("Response don't have \"text\" element");
}
return Long.parseLong(Utils.removeNonDigitCharacters(textObject.getString("text")));
}
throw new ExtractionException("Could not find correct results in response");
}
@Override
public long getLikeCount() throws ParsingException {
assertPageFetched();

46
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View file

@ -3,10 +3,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
@ -141,6 +141,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getTextualUploadDate() throws ParsingException {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return null;
}
if (cachedUploadDate != null) {
return cachedUploadDate;
}
@ -160,9 +164,12 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public Calendar getUploadDate() throws ParsingException {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return null;
}
String textualUploadDate = getTextualUploadDate();
if (timeAgoParser != null
&& textualUploadDate != null && !"".equals(textualUploadDate)) {
if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) {
return timeAgoParser.parse(textualUploadDate);
} else {
return null;
@ -172,24 +179,35 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public long getViewCount() throws ParsingException {
String input;
try {
// TODO: Return the actual live stream's watcher count
// -1 for no view count
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
if (meta == null) return -1;
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
Element meta = item.select("ul[class=\"yt-lockup-meta-info\"]").first();
if (meta == null) return 0;
// This case can happen if google releases a special video
if(meta.select("li").size() < 2) return -1;
final Elements li = meta.select("li");
if (li.isEmpty()) return 0;
input = meta.select("li").get(1).text();
input = li.first().text();
} else {
try {
Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
if (meta == null) return -1;
} catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
// This case can happen if google releases a special video
if (meta.select("li").size() < 2) return -1;
input = meta.select("li").get(1).text();
} catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
}
}
if (input == null) {
throw new ParsingException("Input is null");
}
try {
return Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) {
// if this happens the video probably has no views

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View file

@ -68,7 +68,7 @@ public class YoutubeStreamExtractorLivestreamTest {
@Test
public void testGetViewCount() throws ParsingException {
long count = extractor.getViewCount();
assertTrue(Long.toString(count), count >= 7148995);
assertTrue(Long.toString(count), count > -1);
}
@Test