From 366f5c1632c38a2dc376dd472ba81f554eba51b8 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Thu, 28 Jul 2022 07:49:21 +0530 Subject: [PATCH 1/5] Use StandardCharsets.UTF_8. --- .../MediaCCCSearchQueryHandlerFactory.java | 5 ++--- .../extractors/PeertubeStreamExtractor.java | 4 ++-- .../PeertubeSearchQueryHandlerFactory.java | 6 +++--- .../soundcloud/SoundcloudParsingHelper.java | 9 ++++---- .../extractors/SoundcloudStreamExtractor.java | 4 ++-- .../SoundcloudSuggestionExtractor.java | 9 ++++---- .../SoundcloudSearchQueryHandlerFactory.java | 12 +++++------ .../youtube/YoutubeParsingHelper.java | 8 +++---- .../extractors/YoutubeChannelExtractor.java | 8 +++---- .../YoutubeMusicSearchExtractor.java | 6 +++--- .../extractors/YoutubeSearchExtractor.java | 8 +++---- .../YoutubeSuggestionExtractor.java | 7 ++++--- .../extractors/YoutubeTrendingExtractor.java | 14 ++++++------- .../YoutubeSearchQueryHandlerFactory.java | 21 ++++++++++++------- .../newpipe/extractor/utils/Parser.java | 5 ++++- .../schabi/newpipe/extractor/utils/Utils.java | 14 +++++-------- .../search/SoundcloudSearchExtractorTest.java | 19 ++++++++++------- .../YoutubeSubscriptionExtractorTest.java | 19 ++++++++++------- 18 files changed, 94 insertions(+), 84 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index 83930d1e..54fa0479 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -5,10 +5,9 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String ALL = "all"; public static final String CONFERENCES = "conferences"; @@ -33,7 +32,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory final String sortFilter) throws ParsingException { try { return "https://media.ccc.de/public/events/search?q=" - + URLEncoder.encode(query, UTF_8); + + URLEncoder.encode(query, StandardCharsets.UTF_8.name()); } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not create search string with query: " + query, e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index d70abf11..9b7d8382 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -37,6 +36,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -328,7 +328,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { params.append("start=0&count=8&sort=-createdAt"); for (final String tag : tags) { params.append("&tagsOneOf="); - params.append(URLEncoder.encode(tag, UTF_8)); + params.append(URLEncoder.encode(tag, StandardCharsets.UTF_8.name())); } return url + "?" + params; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 09c14df6..18013129 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -6,10 +6,9 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String VIDEOS = "videos"; @@ -43,7 +42,8 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF final String sortFilter, final String baseUrl) throws ParsingException { try { - return baseUrl + SEARCH_ENDPOINT + "?search=" + URLEncoder.encode(searchString, UTF_8); + return baseUrl + SEARCH_ENDPOINT + "?search=" + URLEncoder.encode(searchString, + StandardCharsets.UTF_8.name()); } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 592a77ad..1f653517 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; @@ -33,6 +32,7 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -109,7 +109,7 @@ public final class SoundcloudParsingHelper { public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) throws IOException, ExtractionException { final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" - + "?url=" + URLEncoder.encode(url, UTF_8) + + "?url=" + URLEncoder.encode(url, StandardCharsets.UTF_8.name()) + "&client_id=" + clientId(); try { @@ -131,7 +131,8 @@ public final class SoundcloudParsingHelper { ReCaptchaException { final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody(); + + URLEncoder.encode(apiUrl, StandardCharsets.UTF_8.name()), + SoundCloud.getLocalization()).responseBody(); return Jsoup.parse(response).select("link[rel=\"canonical\"]").first() .attr("abs:href"); @@ -162,7 +163,7 @@ public final class SoundcloudParsingHelper { try { final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url=" - + URLEncoder.encode(url.toString(), UTF_8) + + URLEncoder.encode(url.toString(), StandardCharsets.UTF_8.name()) + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); final String response = NewPipe.getDownloader().get(widgetUrl, SoundCloud.getLocalization()).responseBody(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 0d28ed95..b87ac5ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -4,7 +4,6 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsing import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -36,6 +35,7 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -320,7 +320,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { private static String urlEncode(final String value) { try { - return URLEncoder.encode(value, UTF_8); + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); } catch (final UnsupportedEncodingException e) { throw new IllegalStateException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java index edb3cc4a..1e6beafb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java @@ -1,9 +1,12 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -14,12 +17,10 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.io.IOException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - public class SoundcloudSuggestionExtractor extends SuggestionExtractor { public SoundcloudSuggestionExtractor(final StreamingService service) { @@ -32,7 +33,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor { final List suggestions = new ArrayList<>(); final Downloader dl = NewPipe.getDownloader(); final String url = SOUNDCLOUD_API_V2_URL + "search/queries" + "?q=" - + URLEncoder.encode(query, UTF_8) + "&client_id=" + + URLEncoder.encode(query, StandardCharsets.UTF_8.name()) + "&client_id=" + SoundcloudParsingHelper.clientId() + "&limit=10"; final String response = dl.get(url, getExtractorLocalization()).responseBody(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java index deb3d8b6..33df3a1b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; + import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -9,11 +11,9 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String TRACKS = "tracks"; @@ -48,9 +48,9 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto } } - return url + "?q=" + URLEncoder.encode(id, UTF_8) + "&client_id=" - + SoundcloudParsingHelper.clientId() + "&limit=" + ITEMS_PER_PAGE - + "&offset=0"; + return url + "?q=" + URLEncoder.encode(id, StandardCharsets.UTF_8.name()) + + "&client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=" + ITEMS_PER_PAGE + "&offset=0"; } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index be38fa9a..ab748b36 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -23,7 +23,6 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static java.util.Collections.singletonList; @@ -575,7 +574,7 @@ public final class YoutubeParsingHelper { .end() .value("fetchLiveState", true) .end() - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final Map> headers = new HashMap<>(); @@ -820,7 +819,7 @@ public final class YoutubeParsingHelper { .end() .end() .value("input", "") - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final Map> headers = new HashMap<>(); @@ -892,7 +891,8 @@ public final class YoutubeParsingHelper { for (final String param : params) { if (param.split("=")[0].equals("q")) { try { - return URLDecoder.decode(param.split("=")[1], UTF_8); + return URLDecoder.decode(param.split("=")[1], + StandardCharsets.UTF_8.name()); } catch (final UnsupportedEncodingException e) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 1622fbe1..86a16409 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -9,7 +9,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -35,6 +34,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -98,7 +98,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { getExtractorLocalization(), getExtractorContentCountry()) .value("url", "https://www.youtube.com/" + channelPath) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url", body, getExtractorLocalization()); @@ -146,7 +146,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .value("browseId", id) .value("params", "EgZ2aWRlb3M%3D") // Equal to videos .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); final JsonObject jsonResponse = getJsonPostResponse("browse", body, getExtractorLocalization()); @@ -397,7 +397,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { getExtractorContentCountry()) .value("continuation", continuation) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index e93c171f..b7bcb5b6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -10,7 +10,6 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -38,6 +37,7 @@ import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -113,7 +113,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .end() .value("query", getSearchString()) .value("params", params) - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final Map> headers = new HashMap<>(); @@ -248,7 +248,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .value("enableSafetyMode", false) .end() .end() - .end().done().getBytes(UTF_8); + .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final Map> headers = new HashMap<>(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index de937cb0..1489963e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -8,7 +8,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -20,6 +19,7 @@ import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -28,12 +28,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; @@ -90,7 +90,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { jsonBody.value("params", params); } - final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8); + final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8); initialData = getJsonPostResponse("search", body, localization); } @@ -184,7 +184,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { getExtractorContentCountry()) .value("continuation", page.getId()) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); // @formatter:on final String responseBody = getValidJsonResponseBody(getDownloader().post( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index f25fea05..0a014a29 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getCookieHeader; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; @@ -16,6 +15,7 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import java.io.IOException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -54,8 +54,9 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml + "&jsonp=" + "JP" + "&ds=" + "yt" - + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8) - + "&q=" + URLEncoder.encode(query, UTF_8); + + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), + StandardCharsets.UTF_8.name()) + + "&q=" + URLEncoder.encode(query, StandardCharsets.UTF_8.name()); String response = dl.get(url, getCookieHeader(), getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index ea53d936..6804909d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -18,7 +18,10 @@ * along with NewPipe Extractor. If not, see . */ -package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; @@ -35,15 +38,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import java.io.IOException; +import java.nio.charset.StandardCharsets; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData; @@ -61,7 +59,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { getExtractorContentCountry()) .value("browseId", "FEtrending") .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); // @formatter:on initialData = getJsonPostResponse("browse", body, getExtractorLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 798a2347..0bbc76da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -1,15 +1,16 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; -import javax.annotation.Nonnull; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { @@ -44,24 +45,28 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa default: break; case VIDEOS: - return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + return SEARCH_URL + URLEncoder.encode(searchString, + StandardCharsets.UTF_8.name()) + "&sp=EgIQAQ%253D%253D"; case CHANNELS: - return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + return SEARCH_URL + URLEncoder.encode(searchString, + StandardCharsets.UTF_8.name()) + "&sp=EgIQAg%253D%253D"; case PLAYLISTS: - return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + return SEARCH_URL + URLEncoder.encode(searchString, + StandardCharsets.UTF_8.name()) + "&sp=EgIQAw%253D%253D"; case MUSIC_SONGS: case MUSIC_VIDEOS: case MUSIC_ALBUMS: case MUSIC_PLAYLISTS: case MUSIC_ARTISTS: - return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8); + return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, + StandardCharsets.UTF_8.name()); } } - return SEARCH_URL + URLEncoder.encode(searchString, UTF_8); + return SEARCH_URL + URLEncoder.encode(searchString, StandardCharsets.UTF_8.name()); } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 64707cd3..03e2c67c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -26,6 +26,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -99,7 +102,7 @@ public final class Parser { for (final String arg : input.split("&")) { final String[] splitArg = arg.split("="); if (splitArg.length > 1) { - map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8)); + map.put(splitArg[0], URLDecoder.decode(splitArg[1], StandardCharsets.UTF_8.name())); } else { map.put(splitArg[0], ""); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 47b24cb3..a9cd7229 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -6,6 +6,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.Map; @@ -17,14 +18,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; public final class Utils { - public static final String HTTP = "http://"; public static final String HTTPS = "https://"; - /** - * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} - */ - @Deprecated - public static final String UTF_8 = "UTF-8"; private static final Pattern M_PATTERN = Pattern.compile("(https?)?://m\\."); private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?://www\\."); @@ -139,7 +134,7 @@ public final class Utils { String query; try { - query = URLDecoder.decode(params[0], UTF_8); + query = URLDecoder.decode(params[0], StandardCharsets.UTF_8.name()); } catch (final UnsupportedEncodingException e) { // Cannot decode string with UTF-8, using the string without decoding query = params[0]; @@ -147,7 +142,7 @@ public final class Utils { if (query.equals(parameterName)) { try { - return URLDecoder.decode(params[1], UTF_8); + return URLDecoder.decode(params[1], StandardCharsets.UTF_8.name()); } catch (final UnsupportedEncodingException e) { // Cannot decode string with UTF-8, using the string without decoding return params[1]; @@ -246,7 +241,8 @@ public final class Utils { try { final URL decoded = Utils.stringToURL(url); if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { - return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8); + return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), + StandardCharsets.UTF_8.name()); } } catch (final Exception ignored) { } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java index eeeb896d..bb53146c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -1,5 +1,13 @@ package org.schabi.newpipe.extractor.services.soundcloud.search; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; +import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.PLAYLISTS; +import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.TRACKS; +import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.USERS; +import static java.util.Collections.singletonList; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; @@ -12,18 +20,13 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; -import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; -import static java.util.Collections.singletonList; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; -import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; +import javax.annotation.Nullable; public class SoundcloudSearchExtractorTest { @@ -138,7 +141,7 @@ public class SoundcloudSearchExtractorTest { private static String urlEncode(String value) { try { - return URLEncoder.encode(value, UTF_8); + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index f5003383..8224f991 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -1,5 +1,11 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.schabi.newpipe.FileUtils.resolveTestResource; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; @@ -13,13 +19,10 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; -import static org.schabi.newpipe.FileUtils.resolveTestResource; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - /** * Test for {@link YoutubeSubscriptionExtractor} */ @@ -54,7 +57,7 @@ public class YoutubeSubscriptionExtractorTest { @Test public void testEmptySourceException() throws Exception { final List items = subscriptionExtractor.fromInputStream( - new ByteArrayInputStream("[]".getBytes(UTF_8))); + new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8))); assertTrue(items.isEmpty()); } @@ -62,7 +65,7 @@ public class YoutubeSubscriptionExtractorTest { public void testSubscriptionWithEmptyTitleInSource() throws Exception { final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]"; final List items = subscriptionExtractor.fromInputStream( - new ByteArrayInputStream(source.getBytes(UTF_8))); + new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); assertEquals(1, items.size()); assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); @@ -75,7 +78,7 @@ public class YoutubeSubscriptionExtractorTest { final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," + "{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]"; final List items = subscriptionExtractor.fromInputStream( - new ByteArrayInputStream(source.getBytes(UTF_8))); + new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); assertEquals(1, items.size()); assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); @@ -99,7 +102,7 @@ public class YoutubeSubscriptionExtractorTest { for (String invalidContent : invalidList) { try { - byte[] bytes = invalidContent.getBytes(UTF_8); + byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8); subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); } catch (final Exception e) { From ddbce3b83d5475824251c77e24cc89947e9e7b34 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Tue, 9 Aug 2022 07:33:29 +0530 Subject: [PATCH 2/5] Add Utils methods for URL encoding/decoding using UTF-8. --- .../BandcampSuggestionExtractor.java | 5 +-- .../BandcampSearchQueryHandlerFactory.java | 18 ++++----- .../MediaCCCSearchQueryHandlerFactory.java | 6 +-- .../extractors/PeertubeStreamExtractor.java | 5 +-- .../PeertubeSearchQueryHandlerFactory.java | 6 +-- .../soundcloud/SoundcloudParsingHelper.java | 9 ++--- .../extractors/SoundcloudStreamExtractor.java | 5 +-- .../SoundcloudSuggestionExtractor.java | 9 ++--- .../SoundcloudSearchQueryHandlerFactory.java | 5 +-- .../youtube/YoutubeParsingHelper.java | 4 +- .../YoutubeSuggestionExtractor.java | 8 ++-- .../extractors/YoutubeTrendingExtractor.java | 40 ++++++++++--------- .../YoutubeSearchQueryHandlerFactory.java | 20 +++------- .../newpipe/extractor/utils/Parser.java | 8 +--- .../schabi/newpipe/extractor/utils/Utils.java | 33 ++++++++++++--- .../search/SoundcloudSearchExtractorTest.java | 5 +-- 16 files changed, 88 insertions(+), 98 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java index 6cc43385..123de430 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java @@ -13,9 +13,9 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.net.URLEncoder; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -33,7 +33,7 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { try { final JsonObject fuzzyResults = JsonParser.object().from(downloader - .get(AUTOCOMPLETE_URL + URLEncoder.encode(query, "UTF-8")).responseBody()); + .get(AUTOCOMPLETE_URL + Utils.encodeUrlUtf8(query)).responseBody()); return fuzzyResults.getObject("auto").getArray("results").stream() .filter(JsonObject.class::isInstance) @@ -44,6 +44,5 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor { } catch (final JsonParserException e) { return Collections.emptyList(); } - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java index 15cf3670..f1eeab4a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java @@ -2,24 +2,22 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; - import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.UnsupportedEncodingException; +import java.util.List; + public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory { - - @Override public String getUrl(final String query, final List contentFilter, final String sortFilter) throws ParsingException { try { - return BASE_URL + "/search?q=" + URLEncoder.encode(query, "UTF-8") + "&page=1"; + return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1"; } catch (final UnsupportedEncodingException e) { throw new ParsingException("query \"" + query + "\" could not be encoded", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index 54fa0479..5a395dae 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -2,10 +2,9 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.List; public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { @@ -31,8 +30,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory public String getUrl(final String query, final List contentFilter, final String sortFilter) throws ParsingException { try { - return "https://media.ccc.de/public/events/search?q=" - + URLEncoder.encode(query, StandardCharsets.UTF_8.name()); + return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query); } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not create search string with query: " + query, e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 9b7d8382..18606377 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -35,8 +35,6 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -327,8 +325,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { final StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); for (final String tag : tags) { - params.append("&tagsOneOf="); - params.append(URLEncoder.encode(tag, StandardCharsets.UTF_8.name())); + params.append("&tagsOneOf=").append(Utils.encodeUrlUtf8(tag)); } return url + "?" + params; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java index 18013129..42d386bc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -3,10 +3,9 @@ package org.schabi.newpipe.extractor.services.peertube.linkHandler; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.List; public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { @@ -42,8 +41,7 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF final String sortFilter, final String baseUrl) throws ParsingException { try { - return baseUrl + SEARCH_ENDPOINT + "?search=" + URLEncoder.encode(searchString, - StandardCharsets.UTF_8.name()); + return baseUrl + SEARCH_ENDPOINT + "?search=" + Utils.encodeUrlUtf8(searchString); } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 1f653517..57deb64a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,8 +31,6 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -109,7 +107,7 @@ public final class SoundcloudParsingHelper { public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) throws IOException, ExtractionException { final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" - + "?url=" + URLEncoder.encode(url, StandardCharsets.UTF_8.name()) + + "?url=" + Utils.encodeUrlUtf8(url) + "&client_id=" + clientId(); try { @@ -131,8 +129,7 @@ public final class SoundcloudParsingHelper { ReCaptchaException { final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + URLEncoder.encode(apiUrl, StandardCharsets.UTF_8.name()), - SoundCloud.getLocalization()).responseBody(); + + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody(); return Jsoup.parse(response).select("link[rel=\"canonical\"]").first() .attr("abs:href"); @@ -163,7 +160,7 @@ public final class SoundcloudParsingHelper { try { final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url=" - + URLEncoder.encode(url.toString(), StandardCharsets.UTF_8.name()) + + Utils.encodeUrlUtf8(url.toString()) + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); final String response = NewPipe.getDownloader().get(widgetUrl, SoundCloud.getLocalization()).responseBody(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index b87ac5ec..150aaf8e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -31,11 +31,10 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -320,7 +319,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { private static String urlEncode(final String value) { try { - return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + return Utils.encodeUrlUtf8(value); } catch (final UnsupportedEncodingException e) { throw new IllegalStateException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java index 1e6beafb..59614b6e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java @@ -14,10 +14,9 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -32,9 +31,9 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor { ExtractionException { final List suggestions = new ArrayList<>(); final Downloader dl = NewPipe.getDownloader(); - final String url = SOUNDCLOUD_API_V2_URL + "search/queries" + "?q=" - + URLEncoder.encode(query, StandardCharsets.UTF_8.name()) + "&client_id=" - + SoundcloudParsingHelper.clientId() + "&limit=10"; + final String url = SOUNDCLOUD_API_V2_URL + "search/queries?q=" + + Utils.encodeUrlUtf8(query) + "&client_id=" + SoundcloudParsingHelper.clientId() + + "&limit=10"; final String response = dl.get(url, getExtractorLocalization()).responseBody(); try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java index 33df3a1b..5d07e1c9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java @@ -7,11 +7,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.List; public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory { @@ -48,7 +47,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto } } - return url + "?q=" + URLEncoder.encode(id, StandardCharsets.UTF_8.name()) + return url + "?q=" + Utils.encodeUrlUtf8(id) + "&client_id=" + SoundcloudParsingHelper.clientId() + "&limit=" + ITEMS_PER_PAGE + "&offset=0"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ab748b36..5c910373 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -54,7 +54,6 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; -import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; import java.time.LocalDate; @@ -891,8 +890,7 @@ public final class YoutubeParsingHelper { for (final String param : params) { if (param.split("=")[0].equals("q")) { try { - return URLDecoder.decode(param.split("=")[1], - StandardCharsets.UTF_8.name()); + return Utils.decodeUrlUtf8(param.split("=")[1]); } catch (final UnsupportedEncodingException e) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index 0a014a29..4be05c5e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -12,10 +12,9 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -54,9 +53,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml + "&jsonp=" + "JP" + "&ds=" + "yt" - + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), - StandardCharsets.UTF_8.name()) - + "&q=" + URLEncoder.encode(query, StandardCharsets.UTF_8.name()); + + "&gl=" + Utils.encodeUrlUtf8(getExtractorContentCountry().getCountryCode()) + + "&q=" + Utils.encodeUrlUtf8(query); String response = dl.get(url, getCookieHeader(), getExtractorLocalization()).responseBody(); // trim JSONP part "JP(...)" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 6804909d..9323c335 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -1,22 +1,4 @@ -/* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeTrendingExtractor.java is part of NewPipe Extractor. - * - * NewPipe Extractor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe Extractor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe Extractor. If not, see . - */ +package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; @@ -42,6 +24,26 @@ import java.nio.charset.StandardCharsets; import javax.annotation.Nonnull; +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeTrendingExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 0bbc76da..3c2bc707 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -1,13 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import static org.schabi.newpipe.extractor.utils.Utils.encodeUrlUtf8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.List; import javax.annotation.Nonnull; @@ -45,28 +44,21 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa default: break; case VIDEOS: - return SEARCH_URL + URLEncoder.encode(searchString, - StandardCharsets.UTF_8.name()) - + "&sp=EgIQAQ%253D%253D"; + return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D"; case CHANNELS: - return SEARCH_URL + URLEncoder.encode(searchString, - StandardCharsets.UTF_8.name()) - + "&sp=EgIQAg%253D%253D"; + return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAg%253D%253D"; case PLAYLISTS: - return SEARCH_URL + URLEncoder.encode(searchString, - StandardCharsets.UTF_8.name()) - + "&sp=EgIQAw%253D%253D"; + return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAw%253D%253D"; case MUSIC_SONGS: case MUSIC_VIDEOS: case MUSIC_ALBUMS: case MUSIC_PLAYLISTS: case MUSIC_ARTISTS: - return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, - StandardCharsets.UTF_8.name()); + return MUSIC_SEARCH_URL + encodeUrlUtf8(searchString); } } - return SEARCH_URL + URLEncoder.encode(searchString, StandardCharsets.UTF_8.name()); + return SEARCH_URL + encodeUrlUtf8(searchString); } catch (final UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 03e2c67c..66c47398 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -20,15 +20,9 @@ package org.schabi.newpipe.extractor.utils; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -102,7 +96,7 @@ public final class Parser { for (final String arg : input.split("&")) { final String[] splitArg = arg.split("="); if (splitArg.length > 1) { - map.put(splitArg[0], URLDecoder.decode(splitArg[1], StandardCharsets.UTF_8.name())); + map.put(splitArg[0], Utils.decodeUrlUtf8(splitArg[1])); } else { map.put(splitArg[0], ""); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index a9cd7229..320a4996 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -6,6 +6,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; @@ -27,6 +28,29 @@ public final class Utils { // no instance } + /** + * Encodes a string to URL format using the UTF-8 character set. + * + * @param string The string to be encoded. + * @return The encoded URL. + * @throws UnsupportedEncodingException This shouldn't be thrown, as UTF-8 should be supported. + */ + public static String encodeUrlUtf8(final String string) throws UnsupportedEncodingException { + // TODO: Switch to URLEncoder.encode(String, Charset) in Java 10. + return URLEncoder.encode(string, StandardCharsets.UTF_8.name()); + } + + /** + * Decodes a URL using the UTF-8 character set. + * @param url The URL to be decoded. + * @return The decoded URL. + * @throws UnsupportedEncodingException This shouldn't be thrown, as UTF-8 should be supported. + */ + public static String decodeUrlUtf8(final String url) throws UnsupportedEncodingException { + // TODO: Switch to URLDecoder.decode(String, Charset) in Java 10. + return URLDecoder.decode(url, StandardCharsets.UTF_8.name()); + } + /** * Remove all non-digit characters from a string. * @@ -134,7 +158,7 @@ public final class Utils { String query; try { - query = URLDecoder.decode(params[0], StandardCharsets.UTF_8.name()); + query = decodeUrlUtf8(params[0]); } catch (final UnsupportedEncodingException e) { // Cannot decode string with UTF-8, using the string without decoding query = params[0]; @@ -142,7 +166,7 @@ public final class Utils { if (query.equals(parameterName)) { try { - return URLDecoder.decode(params[1], StandardCharsets.UTF_8.name()); + return decodeUrlUtf8(params[1]); } catch (final UnsupportedEncodingException e) { // Cannot decode string with UTF-8, using the string without decoding return params[1]; @@ -239,10 +263,9 @@ public final class Utils { public static String followGoogleRedirectIfNeeded(final String url) { // If the url is a redirect from a Google search, extract the actual URL try { - final URL decoded = Utils.stringToURL(url); + final URL decoded = stringToURL(url); if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { - return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), - StandardCharsets.UTF_8.name()); + return decodeUrlUtf8(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url)); } } catch (final Exception ignored) { } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java index bb53146c..c79a4505 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -19,11 +19,10 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.List; import javax.annotation.Nullable; @@ -141,7 +140,7 @@ public class SoundcloudSearchExtractorTest { private static String urlEncode(String value) { try { - return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + return Utils.encodeUrlUtf8(value); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } From ff5f223d3f415c55e2e87d4a203aecab496f1f83 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 1 Oct 2022 11:12:53 +0530 Subject: [PATCH 3/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 51abdb14..3c6cf90b 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. 2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:INSERT_VERSION_HERE'` to the `dependencies` in your `build.gradle`. Replace `INSERT_VERSION_HERE` with the [latest release](https://github.com/TeamNewPipe/NewPipeExtractor/releases/latest). -**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. +**Note:** To use NewPipe Extractor in projects with a `minSdk` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. If the `minSdk` is below 19, the `desugar_jdk_libs_nio` artifact is required, which requires Android Gradle Plugin (AGP) version 7.4.0. ### Testing changes From 416089146e297f9843504eec78bc5b3f1d6cfd89 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 4 Nov 2022 07:20:01 +0530 Subject: [PATCH 4/5] Fix failing tests. --- .../bandcamp/BandcampSearchExtractorTest.java | 26 ++++++++++--------- .../PeertubePlaylistExtractorTest.java | 20 +++++++------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java index 9840ef87..d60178bd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -2,10 +2,18 @@ package org.schabi.newpipe.extractor.services.bandcamp; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.search.SearchExtractor; @@ -13,12 +21,9 @@ import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import javax.annotation.Nullable; import java.io.IOException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; +import javax.annotation.Nullable; /** * Test for {@link BandcampSearchExtractor} @@ -73,20 +78,17 @@ public class BandcampSearchExtractorTest { @Test void testAlbumSearch() throws ExtractionException, IOException { final SearchExtractor extractor = Bandcamp.getSearchExtractor("minecraft volume alpha"); - InfoItem minecraft = extractor.getInitialPage() - .getItems().get(0); + final InfoItem minecraft = extractor.getInitialPage().getItems().get(0); // Minecraft volume alpha should be the first result, no? - assertEquals("Minecraft: Volume Alpha (cover)", minecraft.getName()); + assertEquals("Minecraft - Volume Alpha", minecraft.getName()); assertTrue(minecraft.getThumbnailUrl().endsWith(".jpg")); assertTrue(minecraft.getThumbnailUrl().contains("f4.bcbits.com/img/")); - assertEquals( - "https://chromacat248.bandcamp.com/album/minecraft-volume-alpha-cover", + assertEquals("https://c418.bandcamp.com/album/minecraft-volume-alpha", minecraft.getUrl()); // Verify that playlist tracks counts get extracted correctly - assertEquals(3, ((PlaylistInfoItem) minecraft).getStreamCount()); - + assertEquals(24, ((PlaylistInfoItem) minecraft).getStreamCount()); } /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java index 29e23fac..5bd47de4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java @@ -1,5 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -9,9 +12,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - public class PeertubePlaylistExtractorTest { public static class Shocking { @@ -39,41 +39,41 @@ public class PeertubePlaylistExtractorTest { } @Test - void testGetUploaderUrl() throws ParsingException { + void testGetUploaderUrl() { assertEquals("https://skeptikon.fr/accounts/metadechoc", extractor.getUploaderUrl()); } @Test void testGetUploaderAvatarUrl() throws ParsingException { assertEquals( - "https://framatube.org/lazy-static/avatars/cd0f781d-0287-4be2-94f1-24cd732337b2.jpg", + "https://framatube.org/lazy-static/avatars/c6801ff9-cb49-42e6-b2db-3db623248115.jpg", extractor.getUploaderAvatarUrl()); } @Test - void testGetUploaderName() throws ParsingException { + void testGetUploaderName() { assertEquals("Méta de Choc", extractor.getUploaderName()); } @Test - void testGetStreamCount() throws ParsingException { + void testGetStreamCount() { ExtractorAsserts.assertGreaterOrEqual(39, extractor.getStreamCount()); } @Test - void testGetSubChannelUrl() throws ParsingException { + void testGetSubChannelUrl() { assertEquals("https://skeptikon.fr/video-channels/metadechoc_channel", extractor.getSubChannelUrl()); } @Test - void testGetSubChannelName() throws ParsingException { + void testGetSubChannelName() { assertEquals("SHOCKING !", extractor.getSubChannelName()); } @Test void testGetSubChannelAvatarUrl() throws ParsingException { assertEquals( - "https://framatube.org/lazy-static/avatars/637753af-fcf2-4b61-88f9-b9857c953457.png", + "https://framatube.org/lazy-static/avatars/e801ccce-8694-4309-b0ab-e6f0e552ef77.png", extractor.getSubChannelAvatarUrl()); } } From e4d982c7eacd90f29cf1aa3bc70d3c7fb363f7cf Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Wed, 9 Nov 2022 09:04:16 +0530 Subject: [PATCH 5/5] Fix license. --- .../extractors/YoutubeTrendingExtractor.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 9323c335..fabb3815 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -1,3 +1,23 @@ +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeTrendingExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; @@ -24,26 +44,6 @@ import java.nio.charset.StandardCharsets; import javax.annotation.Nonnull; -/* - * Created by Christian Schabesberger on 12.08.17. - * - * Copyright (C) Christian Schabesberger 2018 - * YoutubeTrendingExtractor.java is part of NewPipe Extractor. - * - * NewPipe Extractor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe Extractor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe Extractor. If not, see . - */ - public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData;