CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

removed generics

Browse source
This commit is contained in:
Ritvik Saraf 2018-09-26 04:21:58 +05:30
parent 8e27801183
commit ce76885553
5 changed files with 105 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View file

@ -61,17 +61,17 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
JsonArray arr; JsonArray arr;
try { try {
arr = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations"); arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations");
} catch (ParsingException e) { } catch (Exception e) {
return ""; return "";
} }
if(null == arr || arr.isEmpty()) { if(arr.isEmpty()) {
return ""; return "";
} }
String continuation; String continuation;
try { try {
continuation = JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation"); continuation = (String) JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation");
} catch (ParsingException e) { } catch (Exception e) {
return ""; return "";
} }
return getNextPageUrl(continuation); return getNextPageUrl(continuation);
@ -109,22 +109,34 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException { private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException {
JsonArray contents;
JsonArray contents = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents"); try {
contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents");
}catch(Exception e) {
throw new ParsingException("unable to get parse youtube comments", e);
}
fetchTitle(contents); fetchTitle(contents);
List<JsonObject> comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer"); List<Object> comments;
try {
comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
}catch(Exception e) {
throw new ParsingException("unable to get parse youtube comments", e);
}
for(JsonObject c: comments) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(c, pageUrl); for(Object c: comments) {
if(c instanceof JsonObject) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, pageUrl);
collector.commit(extractor); collector.commit(extractor);
} }
}
} }
private void fetchTitle(JsonArray contents) { private void fetchTitle(JsonArray contents) {
if(null == title) { if(null == title) {
try { try {
title = JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText"); title = (String) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText");
} catch (Exception e) { } catch (Exception e) {
title = "Youtube Comments"; title = "Youtube Comments";
} }

69
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View file

@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor{ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json; private final JsonObject json;
private final String url; private final String url;
@ -24,55 +24,92 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails"); try {
return JsonUtils.getValue(arr.getObject(2), "url"); JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return (String) JsonUtils.getValue(arr.getObject(2), "url");
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
} }
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
return JsonUtils.getValue(json, "authorText.simpleText"); try {
return (String) JsonUtils.getValue(json, "authorText.simpleText");
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
}
} }
@Override @Override
public String getPublishedTime() throws ParsingException { public String getPublishedTime() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "publishedTimeText.runs"); try {
return JsonUtils.getValue(arr.getObject(0), "text"); JsonArray arr = (JsonArray) JsonUtils.getValue(json, "publishedTimeText.runs");
return (String) JsonUtils.getValue(arr.getObject(0), "text");
} catch (Exception e) {
throw new ParsingException("Could not get publishedTimeText", e);
}
} }
@Override @Override
public Integer getLikeCount() throws ParsingException { public Integer getLikeCount() throws ParsingException {
return JsonUtils.getValue(json, "likeCount"); try {
return (Integer) JsonUtils.getValue(json, "likeCount");
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
}
} }
@Override @Override
public String getCommentText() throws ParsingException { public String getCommentText() throws ParsingException {
try { try {
return JsonUtils.getValue(json, "contentText.simpleText"); return (String) JsonUtils.getValue(json, "contentText.simpleText");
} catch (Exception e) { } catch (Exception e1) {
JsonArray arr = JsonUtils.getValue(json, "contentText.runs"); try {
return JsonUtils.getValue(arr.getObject(0), "text"); JsonArray arr = (JsonArray) JsonUtils.getValue(json, "contentText.runs");
return (String) JsonUtils.getValue(arr.getObject(0), "text");
} catch (Exception e2) {
throw new ParsingException("Could not get comment text", e2);
}
} }
} }
@Override @Override
public String getCommentId() throws ParsingException { public String getCommentId() throws ParsingException {
return JsonUtils.getValue(json, "commentId"); try {
return (String) JsonUtils.getValue(json, "commentId");
} catch (Exception e) {
throw new ParsingException("Could not get comment id", e);
}
} }
@Override @Override
public String getAuthorThumbnail() throws ParsingException { public String getAuthorThumbnail() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails"); try {
return JsonUtils.getValue(arr.getObject(2), "url"); JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return (String) JsonUtils.getValue(arr.getObject(2), "url");
} catch (Exception e) {
throw new ParsingException("Could not get author thumbnail", e);
}
} }
@Override @Override
public String getAuthorName() throws ParsingException { public String getAuthorName() throws ParsingException {
return JsonUtils.getValue(json, "authorText.simpleText"); try {
return (String) JsonUtils.getValue(json, "authorText.simpleText");
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
}
} }
@Override @Override
public String getAuthorEndpoint() throws ParsingException { public String getAuthorEndpoint() throws ParsingException {
return "https://youtube.com" + JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl"); try {
return "https://youtube.com"
+ (String) JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
} catch (Exception e) {
throw new ParsingException("Could not get author endpoint", e);
}
} }
} }

10
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View file

@ -18,24 +18,24 @@ public class JsonUtils {
} }
@Nonnull @Nonnull
public static <T> T getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
List<String> keys = Arrays.asList(path.split("\\.")); List<String> keys = Arrays.asList(path.split("\\."));
object = getObject(object, keys.subList(0, keys.size() - 1)); object = getObject(object, keys.subList(0, keys.size() - 1));
if (null == object) throw new ParsingException("Unable to get " + path); if (null == object) throw new ParsingException("Unable to get " + path);
T result = (T) object.get(keys.get(keys.size() - 1)); Object result = object.get(keys.get(keys.size() - 1));
if(null == result) throw new ParsingException("Unable to get " + path); if(null == result) throw new ParsingException("Unable to get " + path);
return result; return result;
} }
@Nonnull @Nonnull
public static <T> List<T> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
List<T> result = new ArrayList<>(); List<Object> result = new ArrayList<>();
for (int i = 0; i < array.size(); i++) { for (int i = 0; i < array.size(); i++) {
JsonObject obj = array.getObject(i); JsonObject obj = array.getObject(i);
result.add((T)getValue(obj, path)); result.add(getValue(obj, path));
} }
return result; return result;
} }

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View file

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -59,6 +60,23 @@ public class YoutubeCommentsExtractorTest {
assertTrue(result); assertTrue(result);
} }
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for(CommentsInfoItem c: comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
assertFalse(StringUtil.isBlank(c.getAuthorName()));
assertFalse(StringUtil.isBlank(c.getAuthorThumbnail()));
assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getPublishedTime()));
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() == null);
}
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) { private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment); return findInComments(comments.getItems(), comment);
} }

6
extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java
View file

@ -31,15 +31,15 @@ public class JsonUtilsTest {
@Test @Test
public void testGetArray() throws JsonParserException, ParsingException { public void testGetArray() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
JsonArray arr = JsonUtils.getValue(obj, "batters.batter"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "batters.batter");
assertTrue(!arr.isEmpty()); assertTrue(!arr.isEmpty());
} }
@Test @Test
public void testGetValues() throws JsonParserException, ParsingException { public void testGetValues() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
JsonArray arr = JsonUtils.getValue(obj, "topping"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping");
List<String> types = JsonUtils.getValues(arr, "type"); List<Object> types = JsonUtils.getValues(arr, "type");
assertTrue(types.contains("Chocolate with Sprinkles")); assertTrue(types.contains("Chocolate with Sprinkles"));
} }