CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

added loadable comments in stream info

Browse source
This commit is contained in:
Ritvik Saraf 2018-09-03 02:24:03 +05:30
parent 9fb0622a24
commit 823551170d
2 changed files with 116 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

115
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View file

@ -48,7 +48,8 @@ public class StreamInfo extends Info {
}
}
public StreamInfo(int serviceId, String url, String originalUrl, StreamType streamType, String id, String name, int ageLimit) {
public StreamInfo(int serviceId, String url, String originalUrl, StreamType streamType, String id, String name,
int ageLimit) {
super(serviceId, id, url, originalUrl, name);
this.streamType = streamType;
this.ageLimit = ageLimit;
@ -70,9 +71,12 @@ public class StreamInfo extends Info {
streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor);
} catch (ExtractionException e) {
// Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor
// will assume that a video is age restricted while in reality it it blocked by country.
// Currently YouTube does not distinguish between age restricted videos and
// videos blocked
// by country. This means that during the initialisation of the extractor, the
// extractor
// will assume that a video is age restricted while in reality it it blocked by
// country.
//
// We will now detect whether the video is blocked by country or not.
String errorMsg = extractor.getErrorMessage();
@ -89,7 +93,8 @@ public class StreamInfo extends Info {
private static StreamInfo extractImportantData(StreamExtractor extractor) throws ExtractionException {
/* ---- important data, without the video can't be displayed goes here: ---- */
// if one of these is not available an exception is meant to be thrown directly into the frontend.
// if one of these is not available an exception is meant to be thrown directly
// into the frontend.
int serviceId = extractor.getServiceId();
String url = extractor.getUrl();
@ -99,18 +104,16 @@ public class StreamInfo extends Info {
String name = extractor.getName();
int ageLimit = extractor.getAgeLimit();
if ((streamType == StreamType.NONE)
|| (url == null || url.isEmpty())
|| (id == null || id.isEmpty())
|| (name == null /* streamInfo.title can be empty of course */)
|| (ageLimit == -1)) {
if ((streamType == StreamType.NONE) || (url == null || url.isEmpty()) || (id == null || id.isEmpty())
|| (name == null /* streamInfo.title can be empty of course */) || (ageLimit == -1)) {
throw new ExtractionException("Some important stream information was not given.");
}
return new StreamInfo(serviceId, url, originalUrl, streamType, id, name, ageLimit);
}
private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException {
/* ---- stream extraction goes here ---- */
// At least one type of stream has to be available,
// otherwise an exception will be thrown directly into the frontend.
@ -133,13 +136,13 @@ public class StreamInfo extends Info {
} catch (Exception e) {
streamInfo.addError(new ExtractionException("Couldn't get audio streams", e));
}
/* Extract video stream url*/
/* Extract video stream url */
try {
streamInfo.setVideoStreams(extractor.getVideoStreams());
} catch (Exception e) {
streamInfo.addError(new ExtractionException("Couldn't get video streams", e));
}
/* Extract video only stream url*/
/* Extract video only stream url */
try {
streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams());
} catch (Exception e) {
@ -147,9 +150,12 @@ public class StreamInfo extends Info {
}
// Lists can be null if a exception was thrown during extraction
if (streamInfo.getVideoStreams() == null) streamInfo.setVideoStreams(new ArrayList<VideoStream>());
if (streamInfo.getVideoOnlyStreams() == null) streamInfo.setVideoOnlyStreams(new ArrayList<VideoStream>());
if (streamInfo.getAudioStreams() == null) streamInfo.setAudioStreams(new ArrayList<AudioStream>());
if (streamInfo.getVideoStreams() == null)
streamInfo.setVideoStreams(new ArrayList<VideoStream>());
if (streamInfo.getVideoOnlyStreams() == null)
streamInfo.setVideoOnlyStreams(new ArrayList<VideoStream>());
if (streamInfo.getAudioStreams() == null)
streamInfo.setAudioStreams(new ArrayList<AudioStream>());
Exception dashMpdError = null;
if (streamInfo.getDashMpdUrl() != null && !streamInfo.getDashMpdUrl().isEmpty()) {
@ -159,19 +165,23 @@ public class StreamInfo extends Info {
streamInfo.getAudioStreams().addAll(result.getAudioStreams());
streamInfo.getVideoStreams().addAll(result.getVideoStreams());
} catch (Exception e) {
// Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl),
// just skip the exception (but store it somewhere), as we later check if we have streams anyway.
// Sometimes we receive 403 (forbidden) error when trying to download the
// manifest (similar to what happens with youtube-dl),
// just skip the exception (but store it somewhere), as we later check if we
// have streams anyway.
dashMpdError = e;
}
}
// Either audio or video has to be available, otherwise we didn't get a stream (since videoOnly are optional, they don't count).
if ((streamInfo.videoStreams.isEmpty())
&& (streamInfo.audioStreams.isEmpty())) {
// Either audio or video has to be available, otherwise we didn't get a stream
// (since videoOnly are optional, they don't count).
if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) {
if (dashMpdError != null) {
// If we don't have any video or audio and the dashMpd 'errored', add it to the error list
// (it's optional and it don't get added automatically, but it's good to have some additional error context)
// If we don't have any video or audio and the dashMpd 'errored', add it to the
// error list
// (it's optional and it don't get added automatically, but it's good to have
// some additional error context)
streamInfo.addError(dashMpdError);
}
@ -183,8 +193,10 @@ public class StreamInfo extends Info {
private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available.
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
// If one of these fails, the frontend needs to handle that they are not
// available.
// Exceptions are therefore not thrown into the frontend, but stored into the
// error List,
// so the frontend can afterwards check where errors happened.
try {
@ -258,18 +270,37 @@ public class StreamInfo extends Info {
CommentsExtractor commentsExtractor = null;
try {
commentsExtractor = NewPipe.getService(streamInfo.getServiceId()).getCommentsExtractor(streamInfo.getUrl());
} catch (ExtractionException e) {
streamInfo.setCommentsExtractor(commentsExtractor);
} catch (Exception e) {
streamInfo.addError(e);
}
if(null != commentsExtractor) {
InfoItemsPage<CommentsInfoItem> initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(streamInfo, commentsExtractor);
streamInfo.setComments(initialCommentsPage.getItems());
if (null != commentsExtractor) {
InfoItemsPage<CommentsInfoItem> initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(streamInfo,
commentsExtractor);
streamInfo.setComments(new ArrayList<>());
streamInfo.getComments().addAll(initialCommentsPage.getItems());
streamInfo.setHasMoreComments(initialCommentsPage.hasNextPage());
streamInfo.setNextCommentsPageUrl(initialCommentsPage.getNextPageUrl());
}
return streamInfo;
}
public static void loadMoreComments(StreamInfo streamInfo) {
if (streamInfo.hasMoreComments() && null != streamInfo.getCommentsExtractor()) {
try {
InfoItemsPage<CommentsInfoItem> commentsPage = streamInfo.getCommentsExtractor()
.getPage(streamInfo.getNextCommentsPageUrl());
streamInfo.getComments().addAll(commentsPage.getItems());
streamInfo.setHasMoreComments(commentsPage.hasNextPage());
streamInfo.setNextCommentsPageUrl(commentsPage.getNextPageUrl());
} catch (IOException | ExtractionException e) {
streamInfo.addError(e);
}
}
}
private StreamType streamType;
private String thumbnailUrl;
private String uploadDate;
@ -293,7 +324,11 @@ public class StreamInfo extends Info {
private String hlsUrl;
private StreamInfoItem nextVideo;
private List<InfoItem> relatedStreams;
private CommentsExtractor commentsExtractor;
private List<CommentsInfoItem> comments;
private boolean hasMoreComments;
private String nextCommentsPageUrl;
private long startPosition = 0;
private List<Subtitles> subtitles;
@ -499,6 +534,28 @@ public class StreamInfo extends Info {
this.comments = comments;
}
public boolean hasMoreComments() {
return hasMoreComments;
}
public void setHasMoreComments(boolean hasMoreComments) {
this.hasMoreComments = hasMoreComments;
}
public CommentsExtractor getCommentsExtractor() {
return commentsExtractor;
}
public void setCommentsExtractor(CommentsExtractor commentsExtractor) {
this.commentsExtractor = commentsExtractor;
}
public String getNextCommentsPageUrl() {
return nextCommentsPageUrl;
}
public void setNextCommentsPageUrl(String nextCommentsPageUrl) {
this.nextCommentsPageUrl = nextCommentsPageUrl;
}
}

25
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View file

@ -4,6 +4,7 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import java.io.IOException;
import java.util.List;
import org.junit.BeforeClass;
import org.junit.Test;
@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
public class YoutubeCommentsExtractorTest {
@ -31,7 +33,7 @@ public class YoutubeCommentsExtractorTest {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "i should really be in the top comment.lol");
while (comments.hasNextPage()) {
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "i should really be in the top comment.lol");
}
@ -39,7 +41,26 @@ public class YoutubeCommentsExtractorTest {
assertTrue(result);
}
@Test
public void testGetCommentsFromStreamInfo() throws IOException, ExtractionException {
boolean result = false;
StreamInfo streamInfo = StreamInfo.getInfo("https://www.youtube.com/watch?v=rrgFN3AxGfs");
result = findInComments(streamInfo.getComments(), "i should really be in the top comment.lol");
while (streamInfo.hasMoreComments() && !result) {
StreamInfo.loadMoreComments(streamInfo);
result = findInComments(streamInfo.getComments(), "i should really be in the top comment.lol");
}
assertTrue(result);
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return comments.getItems().stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
return comments.stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
}
}