CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Upgrade jsoup

Browse source
This commit is contained in:
wb9688 2020-05-02 08:21:47 +02:00
parent 60e6df7e6b
commit 036cc7e26e
12 changed files with 84 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
extractor/build.gradle
View file

@ -2,7 +2,7 @@ dependencies {
implementation project(':timeago-parser')
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.jsoup:jsoup:1.13.1'
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java
View file

@ -3,13 +3,14 @@ package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -43,7 +44,7 @@ public class PeertubeInstance {
throw new Exception("unable to configure instance " + url, e);
}
if (response == null || StringUtil.isBlank(response.responseBody())) {
if (response == null || Utils.isBlank(response.responseBody())) {
throw new Exception("unable to configure instance " + url);
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View file

@ -1,10 +1,11 @@
package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonObject;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@ -24,7 +25,7 @@ public class PeertubeParsingHelper {
public static void validate(JsonObject json) throws ContentNotAvailableException {
String error = json.getString("error");
if (!StringUtil.isBlank(error)) {
if (!Utils.isBlank(error)) {
throw new ContentNotAvailableException(error);
}
}
@ -51,7 +52,7 @@ public class PeertubeParsingHelper {
} catch (Parser.RegexException e) {
return "";
}
if (StringUtil.isBlank(prevStart)) return "";
if (Utils.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.parseLong(prevStart) + ITEMS_PER_PAGE;

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java
View file

@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -119,7 +120,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !StringUtil.isBlank(response.responseBody())) {
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View file

@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -127,7 +128,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View file

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -65,7 +66,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
@ -88,5 +89,4 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
this.initPage = getPage(getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View file

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -84,7 +85,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
@ -104,5 +105,4 @@ public class PeertubeSearchExtractor extends SearchExtractor {
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
initPage = getPage(getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View file

@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@ -17,10 +17,18 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -29,6 +37,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
public class PeertubeStreamExtractor extends StreamExtractor {
@ -255,7 +265,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} else {
apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
}
if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
if (!Utils.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
return collector;
}
@ -292,7 +302,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !Utils.isBlank(response.responseBody())) {
try {
relatedVideosJson = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java
View file

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -69,7 +70,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {

21
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View file

@ -203,4 +203,23 @@ public class Utils {
public static boolean isNullOrEmpty(final Map map) {
return map == null || map.isEmpty();
}
}
public static boolean isWhitespace(final int c){
return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
}
public static boolean isBlank(final String string) {
if (string == null || string.isEmpty()) {
return true;
}
final int length = string.length();
for (int i = 0; i < length; i++) {
if (!isWhitespace(string.codePointAt(i))) {
return false;
}
}
return true;
}
}

22
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View file

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.jsoup.helper.StringUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
@ -10,6 +9,7 @@ import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
@ -50,7 +50,7 @@ public class PeertubeCommentsExtractorTest {
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl();
while (!StringUtil.isBlank(nextPage) && !result) {
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
@ -63,15 +63,15 @@ public class PeertubeCommentsExtractorTest {
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getUploaderUrl()));
assertFalse(StringUtil.isBlank(c.getUploaderName()));
assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl()));
assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getTextualUploadDate()));
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
}
}

27
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View file

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.helper.StringUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
@ -11,11 +10,15 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.DefaultTests;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
@ -71,7 +74,7 @@ public class YoutubeCommentsExtractorTest {
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
/* String nextPage = commentsInfo.getNextPageUrl();
while (!StringUtil.isBlank(nextPage) && !result) {
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "s1ck m3m3");
nextPage = moreItems.getNextPageUrl();
@ -85,16 +88,16 @@ public class YoutubeCommentsExtractorTest {
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getUploaderUrl()));
assertFalse(StringUtil.isBlank(c.getUploaderName()));
assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl()));
assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertNotNull(c.getUploadDate());
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0);
}
}