added loadable comments in stream info

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -48,7 +48,8 @@ public class StreamInfo extends Info {
         }
     }
 
-    public StreamInfo(int serviceId, String url, String originalUrl, StreamType streamType, String id, String name, int ageLimit) {
+    public StreamInfo(int serviceId, String url, String originalUrl, StreamType streamType, String id, String name,
+            int ageLimit) {
         super(serviceId, id, url, originalUrl, name);
         this.streamType = streamType;
         this.ageLimit = ageLimit;
@@ -70,9 +71,12 @@ public class StreamInfo extends Info {
             streamInfo = extractStreams(streamInfo, extractor);
             streamInfo = extractOptionalData(streamInfo, extractor);
         } catch (ExtractionException e) {
-            // Currently YouTube does not distinguish between age restricted videos and videos blocked
+            // Currently YouTube does not distinguish between age restricted videos and
-            // by country.  This means that during the initialisation of the extractor, the extractor
+            // videos blocked
-            // will assume that a video is age restricted while in reality it it blocked by country.
+            // by country. This means that during the initialisation of the extractor, the
+            // extractor
+            // will assume that a video is age restricted while in reality it it blocked by
+            // country.
             //
             // We will now detect whether the video is blocked by country or not.
             String errorMsg = extractor.getErrorMessage();
@@ -89,7 +93,8 @@ public class StreamInfo extends Info {
 
     private static StreamInfo extractImportantData(StreamExtractor extractor) throws ExtractionException {
         /* ---- important data, without the video can't be displayed goes here: ---- */
-        // if one of these is not available an exception is meant to be thrown directly into the frontend.
+        // if one of these is not available an exception is meant to be thrown directly
+        // into the frontend.
 
         int serviceId = extractor.getServiceId();
         String url = extractor.getUrl();
@@ -99,18 +104,16 @@ public class StreamInfo extends Info {
         String name = extractor.getName();
         int ageLimit = extractor.getAgeLimit();
 
-        if ((streamType == StreamType.NONE)
+        if ((streamType == StreamType.NONE) || (url == null || url.isEmpty()) || (id == null || id.isEmpty())
-                || (url == null || url.isEmpty())
+                || (name == null /* streamInfo.title can be empty of course */) || (ageLimit == -1)) {
-                || (id == null || id.isEmpty())
-                || (name == null /* streamInfo.title can be empty of course */)
-                || (ageLimit == -1)) {
             throw new ExtractionException("Some important stream information was not given.");
         }
 
         return new StreamInfo(serviceId, url, originalUrl, streamType, id, name, ageLimit);
     }
 
-    private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
+    private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor)
+            throws ExtractionException {
         /* ---- stream extraction goes here ---- */
         // At least one type of stream has to be available,
         // otherwise an exception will be thrown directly into the frontend.
@@ -133,13 +136,13 @@ public class StreamInfo extends Info {
         } catch (Exception e) {
             streamInfo.addError(new ExtractionException("Couldn't get audio streams", e));
         }
-        /* Extract video stream url*/
+        /* Extract video stream url */
         try {
             streamInfo.setVideoStreams(extractor.getVideoStreams());
         } catch (Exception e) {
             streamInfo.addError(new ExtractionException("Couldn't get video streams", e));
         }
-        /* Extract video only stream url*/
+        /* Extract video only stream url */
         try {
             streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams());
         } catch (Exception e) {
@@ -147,9 +150,12 @@ public class StreamInfo extends Info {
         }
 
         // Lists can be null if a exception was thrown during extraction
-        if (streamInfo.getVideoStreams() == null) streamInfo.setVideoStreams(new ArrayList<VideoStream>());
+        if (streamInfo.getVideoStreams() == null)
-        if (streamInfo.getVideoOnlyStreams() == null) streamInfo.setVideoOnlyStreams(new ArrayList<VideoStream>());
+            streamInfo.setVideoStreams(new ArrayList<VideoStream>());
-        if (streamInfo.getAudioStreams() == null) streamInfo.setAudioStreams(new ArrayList<AudioStream>());
+        if (streamInfo.getVideoOnlyStreams() == null)
+            streamInfo.setVideoOnlyStreams(new ArrayList<VideoStream>());
+        if (streamInfo.getAudioStreams() == null)
+            streamInfo.setAudioStreams(new ArrayList<AudioStream>());
 
         Exception dashMpdError = null;
         if (streamInfo.getDashMpdUrl() != null && !streamInfo.getDashMpdUrl().isEmpty()) {
@@ -159,19 +165,23 @@ public class StreamInfo extends Info {
                 streamInfo.getAudioStreams().addAll(result.getAudioStreams());
                 streamInfo.getVideoStreams().addAll(result.getVideoStreams());
             } catch (Exception e) {
-                // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl),
+                // Sometimes we receive 403 (forbidden) error when trying to download the
-                // just skip the exception (but store it somewhere), as we later check if we have streams anyway.
+                // manifest (similar to what happens with youtube-dl),
+                // just skip the exception (but store it somewhere), as we later check if we
+                // have streams anyway.
                 dashMpdError = e;
             }
         }
 
-        // Either audio or video has to be available, otherwise we didn't get a stream (since videoOnly are optional, they don't count).
+        // Either audio or video has to be available, otherwise we didn't get a stream
-        if ((streamInfo.videoStreams.isEmpty())
+        // (since videoOnly are optional, they don't count).
-                && (streamInfo.audioStreams.isEmpty())) {
+        if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) {
 
             if (dashMpdError != null) {
-                // If we don't have any video or audio and the dashMpd 'errored', add it to the error list
+                // If we don't have any video or audio and the dashMpd 'errored', add it to the
-                // (it's optional and it don't get added automatically, but it's good to have some additional error context)
+                // error list
+                // (it's optional and it don't get added automatically, but it's good to have
+                // some additional error context)
                 streamInfo.addError(dashMpdError);
             }
 
@@ -183,8 +193,10 @@ public class StreamInfo extends Info {
 
     private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
         /* ---- optional data goes here: ---- */
-        // If one of these fails, the frontend needs to handle that they are not available.
+        // If one of these fails, the frontend needs to handle that they are not
-        // Exceptions are therefore not thrown into the frontend, but stored into the error List,
+        // available.
+        // Exceptions are therefore not thrown into the frontend, but stored into the
+        // error List,
         // so the frontend can afterwards check where errors happened.
 
         try {
@@ -258,18 +270,37 @@ public class StreamInfo extends Info {
         CommentsExtractor commentsExtractor = null;
         try {
             commentsExtractor = NewPipe.getService(streamInfo.getServiceId()).getCommentsExtractor(streamInfo.getUrl());
-        } catch (ExtractionException e) {
+            streamInfo.setCommentsExtractor(commentsExtractor);
+        } catch (Exception e) {
             streamInfo.addError(e);
         }
 
-        if(null != commentsExtractor) {
+        if (null != commentsExtractor) {
-            InfoItemsPage<CommentsInfoItem> initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(streamInfo, commentsExtractor);
+            InfoItemsPage<CommentsInfoItem> initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(streamInfo,
-            streamInfo.setComments(initialCommentsPage.getItems());
+                    commentsExtractor);
+            streamInfo.setComments(new ArrayList<>());
+            streamInfo.getComments().addAll(initialCommentsPage.getItems());
+            streamInfo.setHasMoreComments(initialCommentsPage.hasNextPage());
+            streamInfo.setNextCommentsPageUrl(initialCommentsPage.getNextPageUrl());
         }
 
         return streamInfo;
     }
 
+    public static void loadMoreComments(StreamInfo streamInfo) {
+        if (streamInfo.hasMoreComments() && null != streamInfo.getCommentsExtractor()) {
+            try {
+                InfoItemsPage<CommentsInfoItem> commentsPage = streamInfo.getCommentsExtractor()
+                        .getPage(streamInfo.getNextCommentsPageUrl());
+                streamInfo.getComments().addAll(commentsPage.getItems());
+                streamInfo.setHasMoreComments(commentsPage.hasNextPage());
+                streamInfo.setNextCommentsPageUrl(commentsPage.getNextPageUrl());
+            } catch (IOException | ExtractionException e) {
+                streamInfo.addError(e);
+            }
+        }
+    }
+
     private StreamType streamType;
     private String thumbnailUrl;
     private String uploadDate;
@@ -293,7 +324,11 @@ public class StreamInfo extends Info {
     private String hlsUrl;
     private StreamInfoItem nextVideo;
     private List<InfoItem> relatedStreams;
+
+    private CommentsExtractor commentsExtractor;
     private List<CommentsInfoItem> comments;
+    private boolean hasMoreComments;
+    private String nextCommentsPageUrl;
 
     private long startPosition = 0;
     private List<Subtitles> subtitles;
@@ -499,6 +534,28 @@ public class StreamInfo extends Info {
         this.comments = comments;
     }
 
+    public boolean hasMoreComments() {
+        return hasMoreComments;
+    }
 
+    public void setHasMoreComments(boolean hasMoreComments) {
+        this.hasMoreComments = hasMoreComments;
+    }
+
+    public CommentsExtractor getCommentsExtractor() {
+        return commentsExtractor;
+    }
+
+    public void setCommentsExtractor(CommentsExtractor commentsExtractor) {
+        this.commentsExtractor = commentsExtractor;
+    }
+
+    public String getNextCommentsPageUrl() {
+        return nextCommentsPageUrl;
+    }
+
+    public void setNextCommentsPageUrl(String nextCommentsPageUrl) {
+        this.nextCommentsPageUrl = nextCommentsPageUrl;
+    }
 
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -4,6 +4,7 @@ import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfo;
 
 public class YoutubeCommentsExtractorTest {
 
@@ -31,7 +33,7 @@ public class YoutubeCommentsExtractorTest {
         InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
         result = findInComments(comments, "i should really be in the top comment.lol");
 
-        while (comments.hasNextPage()) {
+        while (comments.hasNextPage() && !result) {
             comments = extractor.getPage(comments.getNextPageUrl());
             result = findInComments(comments, "i should really be in the top comment.lol");
         }
@@ -39,7 +41,26 @@ public class YoutubeCommentsExtractorTest {
         assertTrue(result);
     }
 
+    @Test
+    public void testGetCommentsFromStreamInfo() throws IOException, ExtractionException {
+        boolean result = false;
+        StreamInfo streamInfo = StreamInfo.getInfo("https://www.youtube.com/watch?v=rrgFN3AxGfs");
+
+        result = findInComments(streamInfo.getComments(), "i should really be in the top comment.lol");
+
+        while (streamInfo.hasMoreComments() && !result) {
+            StreamInfo.loadMoreComments(streamInfo);
+            result = findInComments(streamInfo.getComments(), "i should really be in the top comment.lol");
+        }
+
+        assertTrue(result);
+    }
+
     private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
-        return comments.getItems().stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
+        return findInComments(comments.getItems(), comment);
+    }
+
+    private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
+        return comments.stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
     }
 }