Change Json dependency again

The other had some incompatibilities with android.
This commit is contained in:
Mauricio Colli 2017-08-15 23:40:03 -03:00
parent aff595c40e
commit a8a4eaf81b
29 changed files with 287 additions and 256 deletions

View File

@ -1,18 +1,20 @@
apply plugin: 'java-library'
allprojects {
sourceCompatibility = 1.7
targetCompatibility = 1.7
}
repositories {
jcenter()
}
dependencies {
implementation 'com.github.openjson:openjson:1.0.8'
implementation 'com.grack:nanojson:1.1'
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1'
testImplementation 'junit:junit:4.12'
sourceCompatibility = 1.7
targetCompatibility = 1.7
}
task sourcesJar(type: Jar, dependsOn: classes) {

View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.EnumSet;
/*
* Created by Christian Schabesberger on 10.08.15.
@ -27,7 +26,7 @@ import java.util.EnumSet;
public abstract class SearchEngine {
public enum Filter {
STREAM, CHANNEL, PLAYLIST
ANY, STREAM, CHANNEL, PLAYLIST
}
public static class NothingFoundException extends ExtractionException {
@ -46,8 +45,6 @@ public abstract class SearchEngine {
return collector;
}
//Result search(String query, int page);
public abstract InfoItemSearchCollector search(
String query, int page, String contentCountry, EnumSet<Filter> filter)
public abstract InfoItemSearchCollector search(String query, int page, String contentCountry, Filter filter)
throws IOException, ExtractionException;
}

View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
/*
@ -29,8 +28,7 @@ import java.util.List;
*/
public class SearchResult {
public static SearchResult getSearchResult(SearchEngine engine, String query,
int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter)
throws IOException, ExtractionException {
SearchResult result = engine

View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@ -14,7 +16,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
private JSONObject user;
private JsonObject user;
public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
@ -29,16 +31,16 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
user = new JSONObject(response);
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@Override
public String getCleanUrl() {
try {
return user.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl();
}
@Override
@ -53,16 +55,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() {
return user.optString("avatar_url");
return user.getString("avatar_url");
}
@Override
public String getBannerUrl() throws ParsingException {
try {
return user.getJSONObject("visuals").getJSONArray("visuals").getJSONObject(0).getString("visual_url");
} catch (Exception e) {
throw new ParsingException("Could not get Banner", e);
}
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", "");
}
@Override
@ -72,12 +70,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() {
return user.optLong("followers_count");
return user.getNumber("followers_count", 0).longValue();
}
@Override
public String getDescription() throws ParsingException {
return user.optString("description");
return user.getString("description", "");
}
@Override

View File

@ -1,12 +1,12 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private JSONObject searchResult;
private JsonObject searchResult;
public SoundcloudChannelInfoItemExtractor(JSONObject searchResult) {
public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
}
@ -22,21 +22,21 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getThumbnailUrl() {
return searchResult.optString("avatar_url");
return searchResult.getString("avatar_url", "");
}
@Override
public long getSubscriberCount() {
return searchResult.optLong("followers_count");
return searchResult.getNumber("followers_count", 0).longValue();
}
@Override
public long getStreamCount() {
return searchResult.optLong("track_count");
return searchResult.getNumber("track_count", 0).longValue();
}
@Override
public String getDescription() {
return searchResult.optString("description");
return searchResult.getString("description", "");
}
}

View File

@ -1,7 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -65,12 +67,16 @@ public class SoundcloudParsingHelper {
* Call the endpoint "/resolve" of the api.<br/>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId();
return new JSONObject(NewPipe.getDownloader().download(apiUrl));
try {
return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl));
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
/**
@ -123,11 +129,16 @@ public class SoundcloudParsingHelper {
*/
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) {
if (o instanceof JsonObject) collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) o));
}
String nextStreamsUrl;

View File

@ -1,10 +1,13 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@ -13,7 +16,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JSONObject playlist;
private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
@ -29,16 +32,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
"&representation=compact";
String response = dl.download(apiUrl);
playlist = new JSONObject(response);
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@Override
public String getCleanUrl() {
try {
return playlist.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl();
}
@Override
@ -48,12 +51,12 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() {
return playlist.optString("title");
return playlist.getString("title");
}
@Override
public String getThumbnailUrl() {
return playlist.optString("artwork_url");
return playlist.getString("artwork_url");
}
@Override
@ -63,22 +66,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() {
return playlist.getJSONObject("user").getString("permalink_url");
return playlist.getObject("user").getString("permalink_url", "");
}
@Override
public String getUploaderName() {
return playlist.getJSONObject("user").getString("username");
return playlist.getObject("user").getString("username", "");
}
@Override
public String getUploaderAvatarUrl() {
return playlist.getJSONObject("user").getString("avatar_url");
return playlist.getObject("user", new JsonObject()).getString("avatar_url", "");
}
@Override
public long getStreamCount() {
return playlist.getLong("track_count");
return playlist.getNumber("track_count", 0).longValue();
}
@Override

View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
@ -9,60 +8,54 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
private static final String USER_KEY = "user";
private static final String AVATAR_URL_KEY = "avatar_url";
private static final String ARTWORK_URL_KEY = "artwork_url";
private static final String NULL_VALUE = "null";
private JSONObject searchResult;
private JsonObject searchResult;
public SoundcloudPlaylistInfoItemExtractor(JSONObject searchResult) {
public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
}
@Override
public String getName() throws ParsingException {
try {
return searchResult.getString("title");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist name", e);
}
return searchResult.getString("title");
}
@Override
public String getUrl() throws ParsingException {
try {
return searchResult.getString("permalink_url");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist name", e);
}
return searchResult.getString("permalink_url");
}
@Override
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
try {
final String artworkUrl = searchResult.optString(ARTWORK_URL_KEY);
if (!artworkUrl.isEmpty() && !artworkUrl.equals(NULL_VALUE)) return artworkUrl;
if (searchResult.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, "");
if (!artworkUrl.isEmpty()) return artworkUrl;
}
try {
// Look for artwork url inside the track list
final JSONArray tracks = searchResult.optJSONArray("tracks");
if (tracks == null) return null;
for (int i = 0; i < tracks.length(); i++) {
if (tracks.isNull(i)) continue;
final JSONObject track = tracks.optJSONObject(i);
if (track == null) continue;
for (Object track : searchResult.getArray("tracks")) {
final JsonObject trackObject = (JsonObject) track;
// First look for track artwork url
final String url = track.optString(ARTWORK_URL_KEY);
if (!url.isEmpty() && !url.equals(NULL_VALUE)) return url;
if (trackObject.isString(ARTWORK_URL_KEY)) {
final String url = trackObject.getString(ARTWORK_URL_KEY, "");
if (!url.isEmpty()) return url;
}
// Then look for track creator avatar url
final JSONObject creator = track.getJSONObject(USER_KEY);
final String creatorAvatar = creator.optString(AVATAR_URL_KEY);
if (!creatorAvatar.isEmpty() && !creatorAvatar.equals(NULL_VALUE)) return creatorAvatar;
final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject());
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, "");
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
// Try other method
}
try {
// Last resort, use user avatar url. If still not found, then throw exception.
final JSONObject user = searchResult.getJSONObject(USER_KEY);
return user.getString(AVATAR_URL_KEY);
return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
@ -71,8 +64,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
@Override
public String getUploaderName() throws ParsingException {
try {
final JSONObject user = searchResult.getJSONObject(USER_KEY);
return user.optString("username");
return searchResult.getObject(USER_KEY).getString("username");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
@ -80,10 +72,6 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
@Override
public long getStreamCount() throws ParsingException {
try {
return Long.parseLong(searchResult.optString("track_count"));
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist stream count", e);
}
return searchResult.getNumber("track_count", 0).longValue();
}
}

View File

@ -1,16 +1,18 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
public class SoundcloudSearchEngine extends SearchEngine {
public static final String CHARSET_UTF_8 = "UTF-8";
@ -20,19 +22,27 @@ public class SoundcloudSearchEngine extends SearchEngine {
}
@Override
public InfoItemSearchCollector search(String query, int page, String languageCode, EnumSet<Filter> filter) throws IOException, ExtractionException {
public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader();
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search";
if (filter.contains(Filter.STREAM) && filter.size() == 1) {
url += "/tracks";
} else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
url += "/users";
} else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
url += "/playlists";
switch (filter) {
case STREAM:
url += "/tracks";
break;
case CHANNEL:
url += "/users";
break;
case PLAYLIST:
url += "/playlists";
break;
case ANY:
// Don't append any parameter to search for everything
default:
break;
}
url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
@ -40,23 +50,32 @@ public class SoundcloudSearchEngine extends SearchEngine {
+ "&limit=10"
+ "&offset=" + Integer.toString(page * 10);
String searchJson = downloader.download(url);
JSONObject search = new JSONObject(searchJson);
JSONArray searchCollection = search.getJSONArray("collection");
JsonArray searchCollection;
try {
searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
if (searchCollection.length() == 0) {
if (searchCollection.size() == 0) {
throw new NothingFoundException("Nothing found");
}
for (int i = 0; i < searchCollection.length(); i++) {
JSONObject searchResult = searchCollection.getJSONObject(i);
String kind = searchResult.getString("kind");
if (kind.equals("user")) {
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
} else if (kind.equals("track")) {
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
} else if (kind.equals("playlist")) {
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
for (Object result : searchCollection) {
if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions
JsonObject searchResult = (JsonObject) result;
String kind = searchResult.getString("kind", "");
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
break;
case "track":
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
break;
case "playlist":
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
break;
}
}

View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
@ -16,7 +18,7 @@ import java.util.ArrayList;
import java.util.List;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JSONObject track;
private JsonObject track;
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url);
@ -26,7 +28,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public void fetchPage() throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
String policy = track.getString("policy");
String policy = track.getString("policy", "");
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + policy);
}
@ -34,11 +36,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public String getCleanUrl() {
try {
return track.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
return track.isString("permalink_url") ? track.getString("permalink_url") : getOriginalUrl();
}
@Override
@ -48,7 +46,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public String getName() {
return track.optString("title");
return track.getString("title");
}
@Override
@ -58,12 +56,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public String getThumbnailUrl() {
return track.optString("artwork_url");
return track.getString("artwork_url", "");
}
@Override
public String getDescription() {
return track.optString("description");
return track.getString("description");
}
@Override
@ -73,7 +71,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public long getLength() {
return track.getLong("duration") / 1000L;
return track.getNumber("duration", 0).longValue() / 1000L;
}
@Override
@ -125,12 +123,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public long getViewCount() {
return track.getLong("playback_count");
return track.getNumber("playback_count", 0).longValue();
}
@Override
public long getLikeCount() {
return track.getLong("likes_count");
return track.getNumber("likes_count", 0).longValue();
}
@Override
@ -140,17 +138,17 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public String getUploaderUrl() {
return track.getJSONObject("user").getString("permalink_url");
return track.getObject("user").getString("permalink_url", "");
}
@Override
public String getUploaderName() {
return track.getJSONObject("user").getString("username");
return track.getObject("user").getString("username", "");
}
@Override
public String getUploaderAvatarUrl() {
return track.getJSONObject("user").optString("avatar_url");
return track.getObject("user", new JsonObject()).getString("avatar_url", "");
}
@Override
@ -167,10 +165,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
AudioStream audioStream = new AudioStream(responseObject.getString("http_mp3_128_url"), MediaFormat.MP3.id, 128);
audioStreams.add(audioStream);
String mp3Url = responseObject.getString("http_mp3_128_url");
if (mp3Url != null && !mp3Url.isEmpty()) {
audioStreams.add(new AudioStream(mp3Url, MediaFormat.MP3.id, 128));
} else {
throw new ExtractionException("Could not get SoundCloud's track audio url");
}
return audioStreams;
}

View File

@ -1,15 +1,15 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
private final JSONObject searchResult;
private final JsonObject searchResult;
public SoundcloudStreamInfoItemExtractor(JSONObject searchResult) {
public SoundcloudStreamInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
}
@ -25,12 +25,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public long getDuration() {
return searchResult.getLong("duration") / 1000L;
return searchResult.getNumber("duration", 0).longValue() / 1000L;
}
@Override
public String getUploaderName() {
return searchResult.getJSONObject("user").getString("username");
return searchResult.getObject("user").getString("username");
}
@Override
@ -40,12 +40,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public long getViewCount() {
return searchResult.getLong("playback_count");
return searchResult.getNumber("playback_count", 0).longValue();
}
@Override
public String getThumbnailUrl() {
return searchResult.optString("artwork_url");
return searchResult.getString("artwork_url");
}
@Override

View File

@ -1,12 +1,14 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.IOException;
import java.net.URLEncoder;
@ -22,7 +24,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
}
@Override
public List<String> suggestionList(String query, String contentCountry) throws RegexException, ReCaptchaException, IOException {
public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
List<String> suggestions = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
@ -33,14 +35,15 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
+ "&limit=10";
String response = dl.download(url);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
try {
JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (Object suggestion : collection) {
if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query"));
}
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject suggestion = responseCollection.getJSONObject(i);
suggestions.add(suggestion.getString("query"));
return suggestions;
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
return suggestions;
}
}

View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONException;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -159,7 +160,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl);
try {
JSONObject ajaxData = new JSONObject(ajaxDataRaw);
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html");
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl);
@ -170,7 +171,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} else {
nextStreamsUrl = "";
}
} catch (JSONException e) {
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}

View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONException;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -163,7 +164,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl);
try {
JSONObject ajaxData = new JSONObject(ajaxDataRaw);
JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl);
@ -174,7 +175,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} else {
nextStreamsUrl = "";
}
} catch (JSONException e) {
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}

View File

@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
/*
@ -44,26 +43,29 @@ public class YoutubeSearchEngine extends SearchEngine {
}
@Override
public InfoItemSearchCollector search(String query,
int page,
String languageCode,
EnumSet<Filter> filter)
public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter)
throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader();
String url = "https://www.youtube.com/results"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&page=" + Integer.toString(page + 1);
if (filter.contains(Filter.STREAM) && filter.size() == 1) {
url += "&sp=EgIQAVAU";
} else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
} else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
url += "&sp=EgIQA1AU"; //EgIQA( one )AU
switch (filter) {
case STREAM:
url += "&sp=EgIQAVAU";
break;
case CHANNEL:
url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
break;
case PLAYLIST:
url += "&sp=EgIQA1AU"; //EgIQA( one )AU
break;
case ANY:
// Don't append any parameter to search for everything
default:
break;
}
String site;

View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -74,7 +75,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/
private Document doc;
private JSONObject playerArgs;
private JsonObject playerArgs;
private Map<String, String> videoInfoPage;
private boolean isAgeRestricted;
@ -137,7 +138,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
try {
return playerArgs.getString("thumbnail_url");
if (playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
} catch (Exception ignored) {
// Try other method...
}
@ -174,7 +175,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getLength() throws ParsingException {
try {
return playerArgs.getLong("length_seconds");
long returnValue = playerArgs.getNumber("length_seconds", -1).longValue();
if (returnValue >= 0) return returnValue;
} catch (Exception ignored) {
// Try other method...
}
@ -342,8 +344,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String dashManifestUrl;
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs.get("dashmpd") != null) {
dashManifestUrl = playerArgs.optString("dashmpd");
} else if (playerArgs.isString("dashmpd")) {
dashManifestUrl = playerArgs.getString("dashmpd", "");
} else {
return "";
}
@ -513,7 +515,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerUrl = getPlayerUrlFromRestrictedVideo();
isAgeRestricted = true;
} else {
JSONObject ytPlayerConfig = getPlayerConfig(pageContent);
JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
@ -524,11 +526,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
private JsonObject getPlayerConfig(String pageContent) throws ParsingException {
try {
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return new JSONObject(ytPlayerConfigRaw);
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return JsonParser.object().from(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch (errorReason) {
@ -544,13 +545,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
JsonObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
playerArgs = playerConfig.getObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
@ -567,14 +568,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
private String getPlayerUrl(JsonObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl;
JSONObject ytAssets = playerConfig.getJSONObject("assets");
JsonObject ytAssets = playerConfig.getObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
@ -582,8 +583,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
return playerUrl;
} catch (Exception e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
throw new ParsingException("Could not load decryption code for the Youtube service.", e);
}
}
@ -684,8 +684,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String encodedUrlMap = "";
if (videoInfoPage != null && videoInfoPage.containsKey(encodedUrlMapKey)) {
encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
} else if (playerArgs != null && playerArgs.get(encodedUrlMapKey) != null) {
encodedUrlMap = playerArgs.optString(encodedUrlMapKey);
} else if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
}
for (String url_data_str : encodedUrlMap.split(",")) {

View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONArray;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
@ -53,14 +55,12 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
String response = dl.download(url);
try {
JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1);
for (int i = 0; i < suggestionsArray.length(); i++) {
suggestions.add(suggestionsArray.get(i).toString());
}
} catch (Exception e) {
throw new ParsingException("Could not parse suggestions response.", e);
}
JsonArray collection = JsonParser.array().from(response).getArray(1);
for (Object suggestion : collection) suggestions.add(suggestion.toString());
return suggestions;
return suggestions;
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
}

View File

@ -4,12 +4,10 @@ import org.junit.Test;
import java.util.HashSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
public class NewPipeTest {
@Test

View File

@ -0,0 +1,28 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
public class SoundcloudParsingHelperTest {
@BeforeClass
public static void setUp() {
NewPipe.init(Downloader.getInstance());
}
@Test
public void resolveUrlWithEmbedPlayerTest() throws Exception {
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));
Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159"));
}
@Test
public void resolveIdWithEmbedPlayerTest() throws Exception {
Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity"));
Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds"));
}
}

View File

@ -8,8 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -27,9 +25,8 @@ public class SoundcloudSearchEngineAllTest {
// SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert"
// keep in mind that the suggestions can NOT change by country (the parameter "de")
result = engine.search("lill uzi vert", 0, "de",
EnumSet.of(SearchEngine.Filter.CHANNEL,
SearchEngine.Filter.STREAM)).getSearchResult();
result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.ANY)
.getSearchResult();
}
@Test

View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -27,8 +25,8 @@ public class SoundcloudSearchEngineChannelTest {
// SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert"
// keep in mind that the suggestions can NOT change by country (the parameter "de")
result = engine.search("lill uzi vert", 0, "de",
EnumSet.of(SearchEngine.Filter.CHANNEL)).getSearchResult();
result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.CHANNEL)
.getSearchResult();
}
@Test

View File

@ -9,11 +9,7 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -49,7 +45,7 @@ public class SoundcloudSearchEnginePlaylistTest {
SearchEngine engine = SoundCloud.getService().getSearchEngine();
// Search by country not yet implemented
result = engine.search("parkmemme", 0, "", EnumSet.of(SearchEngine.Filter.PLAYLIST))
result = engine.search("parkmemme", 0, "", SearchEngine.Filter.PLAYLIST)
.getSearchResult();
}

View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -27,8 +25,8 @@ public class SoundcloudSearchEngineStreamTest {
// SoundCloud will suggest "lil uzi vert" instead of "lil uzi vert",
// keep in mind that the suggestions can NOT change by country (the parameter "de")
result = engine.search("lill uzi vert", 0, "de",
EnumSet.of(SearchEngine.Filter.STREAM)).getSearchResult();
result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.STREAM)
.getSearchResult();
}
@Test

View File

@ -8,8 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,10 +46,8 @@ public class YoutubeSearchEngineAllTest {
// Youtube will suggest "asdf" instead of "asdgff"
// keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("asdgff", 0, "de",
EnumSet.of(SearchEngine.Filter.CHANNEL,
SearchEngine.Filter.STREAM,
SearchEngine.Filter.PLAYLIST)).getSearchResult();
result = engine.search("asdgff", 0, "de", SearchEngine.Filter.ANY)
.getSearchResult();
}
@Test

View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,8 +46,8 @@ public class YoutubeSearchEngineChannelTest {
// Youtube will suggest "gronkh" instead of "grrunkh"
// keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("grrunkh", 0, "de",
EnumSet.of(SearchEngine.Filter.CHANNEL)).getSearchResult();
result = engine.search("grrunkh", 0, "de", SearchEngine.Filter.CHANNEL)
.getSearchResult();
}
@Test

View File

@ -9,11 +9,7 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -50,8 +46,8 @@ public class YoutubeSearchEnginePlaylistTest {
// Youtube will suggest "gronkh" instead of "grrunkh"
// keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("grrunkh", 0, "de",
EnumSet.of(SearchEngine.Filter.PLAYLIST)).getSearchResult();
result = engine.search("grrunkh", 0, "de", SearchEngine.Filter.PLAYLIST)
.getSearchResult();
}
@Test

View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,8 +46,8 @@ public class YoutubeSearchEngineStreamTest {
// Youtube will suggest "results" instead of "rsults",
// keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("rsults", 0, "de",
EnumSet.of(SearchEngine.Filter.STREAM)).getSearchResult();
result = engine.search("rsults", 0, "de", SearchEngine.Filter.STREAM)
.getSearchResult();
}
@Test

View File

@ -13,9 +13,7 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/*

View File

@ -67,7 +67,7 @@ public class YoutubeStreamExtractorRestrictedTest {
@Test
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
assertTrue(extractor.getViewCount() > 0);
}
@Test