CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #340 from wb9688/peertube-deleted-comments

Browse source 
Handle isDeleted for PeerTube comments
This commit is contained in:
Tobias Groza 2020-06-28 10:21:37 +02:00 committed by GitHub
commit c0ceb5cb27
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 96 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View file

@ -15,20 +15,19 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
public class PeertubeCommentsExtractor extends CommentsExtractor {
private InfoItemsPage<CommentsInfoItem> initPage;
private long total;
public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
public PeertubeCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@ -38,22 +37,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
return initPage;
}
private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract comments info", e);
}
private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject json) throws ParsingException {
final JsonArray contents = json.getArray("data");
for (Object c : contents) {
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
if (!item.getBoolean("isDeleted")) {
final PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
}
}
}
}
@Override
@ -63,8 +58,8 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
public InfoItemsPage<CommentsInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
final Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
@ -74,11 +69,11 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
}
}
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
if (json != null) {
Number number = JsonUtils.getNumber(json, "total");
final Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
collectCommentsFrom(collector, json);
} else {
throw new ExtractionException("Unable to get peertube comments info");
}

51
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View file

@ -18,7 +18,7 @@ import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest {
public static class Default {
private static PeertubeCommentsExtractor extractor;
@BeforeClass
@ -30,10 +30,10 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "@root A great documentary on a great guy.");
assertTrue(comments.getErrors().isEmpty());
boolean result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
@ -44,25 +44,24 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue(commentsInfo.getErrors().isEmpty());
assertEquals("Comments", commentsInfo.getName());
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
final InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
}
assertTrue(result);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (final CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
@ -72,16 +71,16 @@ public class PeertubeCommentsExtractorTest {
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
assertEquals(-1, c.getLikeCount());
}
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
private boolean findInComments(final InfoItemsPage<CommentsInfoItem> comments, final String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
private boolean findInComments(final List<CommentsInfoItem> comments, final String comment) {
for (final CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
@ -89,3 +88,27 @@ public class PeertubeCommentsExtractorTest {
return false;
}
}
public static class DeletedComments {
private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
assertTrue(commentsInfo.getErrors().isEmpty());
}
}
}