Fix some bugs discovered by the new tests
This commit is contained in:
parent
d9cf1fd631
commit
29afd0c29c
12 changed files with 95 additions and 51 deletions
|
@ -1,5 +1,6 @@
|
|||
package org.schabi.newpipe.extractor.services.soundcloud;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
|
@ -14,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public class SoundcloudChannelExtractor extends ChannelExtractor {
|
||||
private String userId;
|
||||
|
@ -30,7 +33,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
|
||||
userId = getUrlIdHandler().getId(getOriginalUrl());
|
||||
String apiUrl = "https://api.soundcloud.com/users/" + userId +
|
||||
String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
|
||||
"?client_id=" + SoundcloudParsingHelper.clientId();
|
||||
|
||||
String response = downloader.download(apiUrl);
|
||||
|
@ -44,7 +47,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getCleanUrl() {
|
||||
return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl();
|
||||
return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl();
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
@ -66,11 +69,10 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
|
|||
|
||||
@Override
|
||||
public String getBannerUrl() {
|
||||
try {
|
||||
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", "");
|
||||
} catch (NullPointerException e) {
|
||||
return null;
|
||||
}
|
||||
return user.getObject("visuals", new JsonObject())
|
||||
.getArray("visuals", new JsonArray())
|
||||
.getObject(0, new JsonObject())
|
||||
.getString("visual_url");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud;
|
|||
import com.grack.nanojson.JsonObject;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
|
||||
private final JsonObject itemObject;
|
||||
|
||||
|
@ -17,7 +19,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
|
|||
|
||||
@Override
|
||||
public String getUrl() {
|
||||
return itemObject.getString("permalink_url");
|
||||
return replaceHttpWithHttps(itemObject.getString("permalink_url"));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
|
||||
private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
|
||||
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
|
||||
|
@ -45,7 +47,7 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
|
|||
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
|
||||
.select("meta[property=og:url]").first();
|
||||
|
||||
return ogElement.attr("content");
|
||||
return replaceHttpWithHttps(ogElement.attr("content"));
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
|
@ -23,11 +23,10 @@ import java.text.ParseException;
|
|||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudParsingHelper {
|
||||
private static String clientId;
|
||||
|
||||
private static final String HTTP = "http://";
|
||||
private static final String HTTPS = "https://";
|
||||
|
||||
private SoundcloudParsingHelper() {
|
||||
}
|
||||
|
@ -215,13 +214,6 @@ public class SoundcloudParsingHelper {
|
|||
return getStreamsFromApi(collector, apiUrl, false);
|
||||
}
|
||||
|
||||
private static String replaceHttpWithHttps(final String url) {
|
||||
if(!url.isEmpty() && url.startsWith(HTTP)) {
|
||||
return HTTPS + url.substring(HTTP.length());
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
static String getUploaderUrl(JsonObject object) {
|
||||
String url = object.getObject("user").getString("permalink_url", "");
|
||||
|
|
|
@ -14,6 +14,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
||||
private String playlistId;
|
||||
|
@ -45,7 +47,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getCleanUrl() {
|
||||
return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl();
|
||||
return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl();
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
@ -62,7 +64,26 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
@Override
|
||||
public String getThumbnailUrl() {
|
||||
return playlist.getString("artwork_url");
|
||||
String artworkUrl = playlist.getString("artwork_url");
|
||||
|
||||
if (artworkUrl == null) {
|
||||
// If the thumbnail is null, traverse the items list and get a valid one,
|
||||
// if it also fails, return null
|
||||
try {
|
||||
final StreamInfoItemsCollector infoItems = getInfoItems();
|
||||
if (infoItems.getItemList().isEmpty()) return null;
|
||||
|
||||
for (StreamInfoItem item : infoItems.getItemList()) {
|
||||
final String thumbnailUrl = item.getThumbnailUrl();
|
||||
if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
|
||||
|
||||
return thumbnailUrl;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
return artworkUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -4,38 +4,40 @@ import com.grack.nanojson.JsonObject;
|
|||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
|
||||
private static final String USER_KEY = "user";
|
||||
private static final String AVATAR_URL_KEY = "avatar_url";
|
||||
private static final String ARTWORK_URL_KEY = "artwork_url";
|
||||
|
||||
private final JsonObject searchResult;
|
||||
private final JsonObject itemObject;
|
||||
|
||||
public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) {
|
||||
this.searchResult = searchResult;
|
||||
public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) {
|
||||
this.itemObject = itemObject;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
return searchResult.getString("title");
|
||||
public String getName() {
|
||||
return itemObject.getString("title");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
return searchResult.getString("permalink_url");
|
||||
public String getUrl() {
|
||||
return replaceHttpWithHttps(itemObject.getString("permalink_url"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
// Over-engineering at its finest
|
||||
if (searchResult.isString(ARTWORK_URL_KEY)) {
|
||||
final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, "");
|
||||
if (itemObject.isString(ARTWORK_URL_KEY)) {
|
||||
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
|
||||
if (!artworkUrl.isEmpty()) return artworkUrl;
|
||||
}
|
||||
|
||||
try {
|
||||
// Look for artwork url inside the track list
|
||||
for (Object track : searchResult.getArray("tracks")) {
|
||||
for (Object track : itemObject.getArray("tracks")) {
|
||||
final JsonObject trackObject = (JsonObject) track;
|
||||
|
||||
// First look for track artwork url
|
||||
|
@ -55,7 +57,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
|
|||
|
||||
try {
|
||||
// Last resort, use user avatar url. If still not found, then throw exception.
|
||||
return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
|
||||
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Failed to extract playlist thumbnail url", e);
|
||||
}
|
||||
|
@ -64,14 +66,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
|
|||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
try {
|
||||
return searchResult.getObject(USER_KEY).getString("username");
|
||||
return itemObject.getObject(USER_KEY).getString("username");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Failed to extract playlist uploader", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getStreamCount() throws ParsingException {
|
||||
return searchResult.getNumber("track_count", 0).longValue();
|
||||
public long getStreamCount() {
|
||||
return itemObject.getNumber("track_count", 0).longValue();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
|
||||
private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
|
||||
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
|
||||
|
@ -45,7 +47,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
|
|||
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
|
||||
.select("meta[property=og:url]").first();
|
||||
|
||||
return ogElement.attr("content");
|
||||
return replaceHttpWithHttps(ogElement.attr("content"));
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
|
@ -13,7 +13,11 @@ import javax.annotation.Nonnull;
|
|||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudStreamExtractor extends StreamExtractor {
|
||||
private JsonObject track;
|
||||
|
@ -35,7 +39,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getCleanUrl() {
|
||||
return track.isString("permalink_url") ? track.getString("permalink_url") : getOriginalUrl();
|
||||
return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl();
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
|
@ -5,52 +5,54 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
|
||||
protected final JsonObject searchResult;
|
||||
protected final JsonObject itemObject;
|
||||
|
||||
public SoundcloudStreamInfoItemExtractor(JsonObject searchResult) {
|
||||
this.searchResult = searchResult;
|
||||
public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) {
|
||||
this.itemObject = itemObject;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() {
|
||||
return searchResult.getString("permalink_url");
|
||||
return replaceHttpWithHttps(itemObject.getString("permalink_url"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return searchResult.getString("title");
|
||||
return itemObject.getString("title");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getDuration() {
|
||||
return searchResult.getNumber("duration", 0).longValue() / 1000L;
|
||||
return itemObject.getNumber("duration", 0).longValue() / 1000L;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderName() {
|
||||
return searchResult.getObject("user").getString("username");
|
||||
return itemObject.getObject("user").getString("username");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() {
|
||||
return searchResult.getObject("user").getString("permalink_url");
|
||||
return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploadDate() throws ParsingException {
|
||||
return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
|
||||
return SoundcloudParsingHelper.toDateString(itemObject.getString("created_at"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() {
|
||||
return searchResult.getNumber("playback_count", 0).longValue();
|
||||
return itemObject.getNumber("playback_count", 0).longValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() {
|
||||
return searchResult.getString("artwork_url");
|
||||
return itemObject.getString("artwork_url");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|||
import org.schabi.newpipe.extractor.utils.Parser;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
||||
|
||||
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
|
||||
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
|
||||
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
|
||||
|
@ -48,7 +50,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
|
|||
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
|
||||
.select("meta[property=og:url]").first();
|
||||
|
||||
return ogElement.attr("content");
|
||||
return replaceHttpWithHttps(ogElement.attr("content"));
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
|
|
@ -175,7 +175,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
throw new ParsingException("Could not parse json data for next streams", pe);
|
||||
}
|
||||
|
||||
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
|
||||
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl);
|
||||
collectStreamsFrom(collector, ajaxHtml.select("body").first());
|
||||
|
||||
return new InfoItemPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
|
||||
|
|
|
@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|||
import java.util.List;
|
||||
|
||||
public class Utils {
|
||||
|
||||
private Utils() {
|
||||
//no instance
|
||||
}
|
||||
|
@ -46,5 +47,17 @@ public class Utils {
|
|||
System.err.println("----------------");
|
||||
}
|
||||
}
|
||||
|
||||
private static final String HTTP = "http://";
|
||||
private static final String HTTPS = "https://";
|
||||
|
||||
public static String replaceHttpWithHttps(final String url) {
|
||||
if (url == null) return null;
|
||||
|
||||
if(!url.isEmpty() && url.startsWith(HTTP)) {
|
||||
return HTTPS + url.substring(HTTP.length());
|
||||
}
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue