CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

removed generics

Browse source
This commit is contained in:
Ritvik Saraf 2018-09-26 04:21:58 +05:30
parent 8e27801183
commit ce76885553
5 changed files with 105 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View file

@ -61,17 +61,17 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
JsonArray arr;
try {
arr = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations");
} catch (ParsingException e) {
arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations");
} catch (Exception e) {
return "";
}
if(null == arr || arr.isEmpty()) {
if(arr.isEmpty()) {
return "";
}
String continuation;
try {
continuation = JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation");
} catch (ParsingException e) {
continuation = (String) JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation");
} catch (Exception e) {
return "";
}
return getNextPageUrl(continuation);
@ -109,14 +109,26 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException {
JsonArray contents = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents");
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents");
}catch(Exception e) {
throw new ParsingException("unable to get parse youtube comments", e);
}
fetchTitle(contents);
List<JsonObject> comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
List<Object> comments;
try {
comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
}catch(Exception e) {
throw new ParsingException("unable to get parse youtube comments", e);
}
for(JsonObject c: comments) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(c, pageUrl);
collector.commit(extractor);
for(Object c: comments) {
if(c instanceof JsonObject) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, pageUrl);
collector.commit(extractor);
}
}
}
@ -124,7 +136,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private void fetchTitle(JsonArray contents) {
if(null == title) {
try {
title = JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText");
title = (String) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText");
} catch (Exception e) {
title = "Youtube Comments";
}

73
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View file

@ -7,11 +7,11 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor{
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json;
private final String url;
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) {
this.json = json;
this.url = url;
@ -24,55 +24,92 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return JsonUtils.getValue(arr.getObject(2), "url");
try {
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return (String) JsonUtils.getValue(arr.getObject(2), "url");
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getValue(json, "authorText.simpleText");
try {
return (String) JsonUtils.getValue(json, "authorText.simpleText");
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
}
}
@Override
public String getPublishedTime() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "publishedTimeText.runs");
return JsonUtils.getValue(arr.getObject(0), "text");
try {
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "publishedTimeText.runs");
return (String) JsonUtils.getValue(arr.getObject(0), "text");
} catch (Exception e) {
throw new ParsingException("Could not get publishedTimeText", e);
}
}
@Override
public Integer getLikeCount() throws ParsingException {
return JsonUtils.getValue(json, "likeCount");
try {
return (Integer) JsonUtils.getValue(json, "likeCount");
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
}
}
@Override
public String getCommentText() throws ParsingException {
try {
return JsonUtils.getValue(json, "contentText.simpleText");
} catch (Exception e) {
JsonArray arr = JsonUtils.getValue(json, "contentText.runs");
return JsonUtils.getValue(arr.getObject(0), "text");
return (String) JsonUtils.getValue(json, "contentText.simpleText");
} catch (Exception e1) {
try {
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "contentText.runs");
return (String) JsonUtils.getValue(arr.getObject(0), "text");
} catch (Exception e2) {
throw new ParsingException("Could not get comment text", e2);
}
}
}
@Override
public String getCommentId() throws ParsingException {
return JsonUtils.getValue(json, "commentId");
try {
return (String) JsonUtils.getValue(json, "commentId");
} catch (Exception e) {
throw new ParsingException("Could not get comment id", e);
}
}
@Override
public String getAuthorThumbnail() throws ParsingException {
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return JsonUtils.getValue(arr.getObject(2), "url");
try {
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails");
return (String) JsonUtils.getValue(arr.getObject(2), "url");
} catch (Exception e) {
throw new ParsingException("Could not get author thumbnail", e);
}
}
@Override
public String getAuthorName() throws ParsingException {
return JsonUtils.getValue(json, "authorText.simpleText");
try {
return (String) JsonUtils.getValue(json, "authorText.simpleText");
} catch (Exception e) {
throw new ParsingException("Could not get author name", e);
}
}
@Override
public String getAuthorEndpoint() throws ParsingException {
return "https://youtube.com" + JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
try {
return "https://youtube.com"
+ (String) JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
} catch (Exception e) {
throw new ParsingException("Could not get author endpoint", e);
}
}
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View file

@ -18,24 +18,24 @@ public class JsonUtils {
}
@Nonnull
public static <T> T getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
List<String> keys = Arrays.asList(path.split("\\."));
object = getObject(object, keys.subList(0, keys.size() - 1));
if (null == object) throw new ParsingException("Unable to get " + path);
T result = (T) object.get(keys.get(keys.size() - 1));
Object result = object.get(keys.get(keys.size() - 1));
if(null == result) throw new ParsingException("Unable to get " + path);
return result;
}
@Nonnull
public static <T> List<T> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
List<T> result = new ArrayList<>();
List<Object> result = new ArrayList<>();
for (int i = 0; i < array.size(); i++) {
JsonObject obj = array.getObject(i);
result.add((T)getValue(obj, path));
result.add(getValue(obj, path));
}
return result;
}

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View file

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -58,6 +59,23 @@ public class YoutubeCommentsExtractorTest {
assertTrue(result);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for(CommentsInfoItem c: comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
assertFalse(StringUtil.isBlank(c.getAuthorName()));
assertFalse(StringUtil.isBlank(c.getAuthorThumbnail()));
assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getPublishedTime()));
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() == null);
}
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);

6
extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java
View file

@ -31,15 +31,15 @@ public class JsonUtilsTest {
@Test
public void testGetArray() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
JsonArray arr = JsonUtils.getValue(obj, "batters.batter");
JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "batters.batter");
assertTrue(!arr.isEmpty());
}
@Test
public void testGetValues() throws JsonParserException, ParsingException {
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
JsonArray arr = JsonUtils.getValue(obj, "topping");
List<String> types = JsonUtils.getValues(arr, "type");
JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping");
List<Object> types = JsonUtils.getValues(arr, "type");
assertTrue(types.contains("Chocolate with Sprinkles"));
}