From 29afd0c29c0329c5ab9094d6dbc5f7a3b8a6c83d Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sun, 4 Mar 2018 17:30:31 -0300 Subject: [PATCH] Fix some bugs discovered by the new tests --- .../SoundcloudChannelExtractor.java | 16 +++++----- .../SoundcloudChannelInfoItemExtractor.java | 4 ++- .../SoundcloudChannelUrlIdHandler.java | 4 ++- .../soundcloud/SoundcloudParsingHelper.java | 12 ++------ .../SoundcloudPlaylistExtractor.java | 25 ++++++++++++++-- .../SoundcloudPlaylistInfoItemExtractor.java | 30 ++++++++++--------- .../SoundcloudPlaylistUrlIdHandler.java | 4 ++- .../soundcloud/SoundcloudStreamExtractor.java | 8 +++-- .../SoundcloudStreamInfoItemExtractor.java | 24 ++++++++------- .../SoundcloudStreamUrlIdHandler.java | 4 ++- .../youtube/YoutubeChannelExtractor.java | 2 +- .../schabi/newpipe/extractor/utils/Utils.java | 13 ++++++++ 12 files changed, 95 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index d806c44e..b71d2672 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; @@ -14,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + @SuppressWarnings("WeakerAccess") public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; @@ -30,7 +33,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { userId = getUrlIdHandler().getId(getOriginalUrl()); - String apiUrl = "https://api.soundcloud.com/users/" + userId + + String apiUrl = "https://api-v2.soundcloud.com/users/" + userId + "?client_id=" + SoundcloudParsingHelper.clientId(); String response = downloader.download(apiUrl); @@ -44,7 +47,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Nonnull @Override public String getCleanUrl() { - return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl(); + return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl(); } @Nonnull @@ -66,11 +69,10 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() { - try { - return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", ""); - } catch (NullPointerException e) { - return null; - } + return user.getObject("visuals", new JsonObject()) + .getArray("visuals", new JsonArray()) + .getObject(0, new JsonObject()) + .getString("visual_url"); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java index ef1799f6..dcc6c20c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java @@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud; import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { private final JsonObject itemObject; @@ -17,7 +19,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac @Override public String getUrl() { - return itemObject.getString("permalink_url"); + return replaceHttpWithHttps(itemObject.getString("permalink_url")); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java index c0f92693..04902eb3 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java @@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + @@ -45,7 +47,7 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) .select("meta[property=og:url]").first(); - return ogElement.attr("content"); + return replaceHttpWithHttps(ogElement.attr("content")); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index bdb58673..1a4584a6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -23,11 +23,10 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudParsingHelper { private static String clientId; - - private static final String HTTP = "http://"; - private static final String HTTPS = "https://"; private SoundcloudParsingHelper() { } @@ -215,13 +214,6 @@ public class SoundcloudParsingHelper { return getStreamsFromApi(collector, apiUrl, false); } - private static String replaceHttpWithHttps(final String url) { - if(!url.isEmpty() && url.startsWith(HTTP)) { - return HTTPS + url.substring(HTTP.length()); - } - return url; - } - @Nonnull static String getUploaderUrl(JsonObject object) { String url = object.getObject("user").getString("permalink_url", ""); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 83f3b08f..7072d8fa 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -14,6 +14,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private String playlistId; @@ -45,7 +47,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override public String getCleanUrl() { - return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl(); + return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl(); } @Nonnull @@ -62,7 +64,26 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() { - return playlist.getString("artwork_url"); + String artworkUrl = playlist.getString("artwork_url"); + + if (artworkUrl == null) { + // If the thumbnail is null, traverse the items list and get a valid one, + // if it also fails, return null + try { + final StreamInfoItemsCollector infoItems = getInfoItems(); + if (infoItems.getItemList().isEmpty()) return null; + + for (StreamInfoItem item : infoItems.getItemList()) { + final String thumbnailUrl = item.getThumbnailUrl(); + if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue; + + return thumbnailUrl; + } + } catch (Exception ignored) { + } + } + + return artworkUrl; } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java index 276f2c6a..4a6b4f1e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java @@ -4,38 +4,40 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private static final String USER_KEY = "user"; private static final String AVATAR_URL_KEY = "avatar_url"; private static final String ARTWORK_URL_KEY = "artwork_url"; - private final JsonObject searchResult; + private final JsonObject itemObject; - public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) { - this.searchResult = searchResult; + public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) { + this.itemObject = itemObject; } @Override - public String getName() throws ParsingException { - return searchResult.getString("title"); + public String getName() { + return itemObject.getString("title"); } @Override - public String getUrl() throws ParsingException { - return searchResult.getString("permalink_url"); + public String getUrl() { + return replaceHttpWithHttps(itemObject.getString("permalink_url")); } @Override public String getThumbnailUrl() throws ParsingException { // Over-engineering at its finest - if (searchResult.isString(ARTWORK_URL_KEY)) { - final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, ""); + if (itemObject.isString(ARTWORK_URL_KEY)) { + final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, ""); if (!artworkUrl.isEmpty()) return artworkUrl; } try { // Look for artwork url inside the track list - for (Object track : searchResult.getArray("tracks")) { + for (Object track : itemObject.getArray("tracks")) { final JsonObject trackObject = (JsonObject) track; // First look for track artwork url @@ -55,7 +57,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr try { // Last resort, use user avatar url. If still not found, then throw exception. - return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, ""); + return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, ""); } catch (Exception e) { throw new ParsingException("Failed to extract playlist thumbnail url", e); } @@ -64,14 +66,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr @Override public String getUploaderName() throws ParsingException { try { - return searchResult.getObject(USER_KEY).getString("username"); + return itemObject.getObject(USER_KEY).getString("username"); } catch (Exception e) { throw new ParsingException("Failed to extract playlist uploader", e); } } @Override - public long getStreamCount() throws ParsingException { - return searchResult.getNumber("track_count", 0).longValue(); + public long getStreamCount() { + return itemObject.getNumber("track_count", 0).longValue(); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java index ea883c43..9518f1f9 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java @@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + @@ -45,7 +47,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) .select("meta[property=og:url]").first(); - return ogElement.attr("content"); + return replaceHttpWithHttps(ogElement.attr("content")); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 5819342a..dddc4d5c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -13,7 +13,11 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -35,7 +39,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public String getCleanUrl() { - return track.isString("permalink_url") ? track.getString("permalink_url") : getOriginalUrl(); + return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl(); } @Nonnull diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java index cfea37b9..358f32da 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java @@ -5,52 +5,54 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { - protected final JsonObject searchResult; + protected final JsonObject itemObject; - public SoundcloudStreamInfoItemExtractor(JsonObject searchResult) { - this.searchResult = searchResult; + public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) { + this.itemObject = itemObject; } @Override public String getUrl() { - return searchResult.getString("permalink_url"); + return replaceHttpWithHttps(itemObject.getString("permalink_url")); } @Override public String getName() { - return searchResult.getString("title"); + return itemObject.getString("title"); } @Override public long getDuration() { - return searchResult.getNumber("duration", 0).longValue() / 1000L; + return itemObject.getNumber("duration", 0).longValue() / 1000L; } @Override public String getUploaderName() { - return searchResult.getObject("user").getString("username"); + return itemObject.getObject("user").getString("username"); } @Override public String getUploaderUrl() { - return searchResult.getObject("user").getString("permalink_url"); + return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url")); } @Override public String getUploadDate() throws ParsingException { - return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at")); + return SoundcloudParsingHelper.toDateString(itemObject.getString("created_at")); } @Override public long getViewCount() { - return searchResult.getNumber("playback_count", 0).longValue(); + return itemObject.getNumber("playback_count", 0).longValue(); } @Override public String getThumbnailUrl() { - return searchResult.getString("artwork_url"); + return itemObject.getString("artwork_url"); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java index 2857c1ba..b38dac03 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java @@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + @@ -48,7 +50,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) .select("meta[property=og:url]").first(); - return ogElement.attr("content"); + return replaceHttpWithHttps(ogElement.attr("content")); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 593a4ef7..4305af14 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -175,7 +175,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ParsingException("Could not parse json data for next streams", pe); } - final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html")); + final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl); collectStreamsFrom(collector, ajaxHtml.select("body").first()); return new InfoItemPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); diff --git a/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index be52a5ab..cc49d78f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.util.List; public class Utils { + private Utils() { //no instance } @@ -46,5 +47,17 @@ public class Utils { System.err.println("----------------"); } } + + private static final String HTTP = "http://"; + private static final String HTTPS = "https://"; + + public static String replaceHttpWithHttps(final String url) { + if (url == null) return null; + + if(!url.isEmpty() && url.startsWith(HTTP)) { + return HTTPS + url.substring(HTTP.length()); + } + return url; + } }