CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix some bugs discovered by the new tests

Browse source
This commit is contained in:
Mauricio Colli 2018-03-04 17:30:31 -03:00
parent d9cf1fd631
commit 29afd0c29c
12 changed files with 95 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View file

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
@ -14,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
@ -30,7 +33,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
userId = getUrlIdHandler().getId(getOriginalUrl());
String apiUrl = "https://api.soundcloud.com/users/" + userId +
String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = downloader.download(apiUrl);
@ -44,7 +47,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getCleanUrl() {
return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl();
return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl();
}
@Nonnull
@ -66,11 +69,10 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() {
try {
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", "");
} catch (NullPointerException e) {
return null;
}
return user.getObject("visuals", new JsonObject())
.getArray("visuals", new JsonArray())
.getObject(0, new JsonObject())
.getString("visual_url");
}
@Override

4
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java
View file

@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private final JsonObject itemObject;
@ -17,7 +19,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getUrl() {
return itemObject.getString("permalink_url");
return replaceHttpWithHttps(itemObject.getString("permalink_url"));
}
@Override

4
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java
View file

@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
@ -45,7 +47,7 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
return ogElement.attr("content");
return replaceHttpWithHttps(ogElement.attr("content"));
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}

12
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file

@ -23,11 +23,10 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
private static String clientId;
private static final String HTTP = "http://";
private static final String HTTPS = "https://";
private SoundcloudParsingHelper() {
}
@ -215,13 +214,6 @@ public class SoundcloudParsingHelper {
return getStreamsFromApi(collector, apiUrl, false);
}
private static String replaceHttpWithHttps(final String url) {
if(!url.isEmpty() && url.startsWith(HTTP)) {
return HTTPS + url.substring(HTTP.length());
}
return url;
}
@Nonnull
static String getUploaderUrl(JsonObject object) {
String url = object.getObject("user").getString("permalink_url", "");

25
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View file

@ -14,6 +14,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
@SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
@ -45,7 +47,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public String getCleanUrl() {
return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl();
return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl();
}
@Nonnull
@ -62,7 +64,26 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() {
return playlist.getString("artwork_url");
String artworkUrl = playlist.getString("artwork_url");
if (artworkUrl == null) {
// If the thumbnail is null, traverse the items list and get a valid one,
// if it also fails, return null
try {
final StreamInfoItemsCollector infoItems = getInfoItems();
if (infoItems.getItemList().isEmpty()) return null;
for (StreamInfoItem item : infoItems.getItemList()) {
final String thumbnailUrl = item.getThumbnailUrl();
if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
return thumbnailUrl;
}
} catch (Exception ignored) {
}
}
return artworkUrl;
}
@Override

30
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java
View file

@ -4,38 +4,40 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private static final String USER_KEY = "user";
private static final String AVATAR_URL_KEY = "avatar_url";
private static final String ARTWORK_URL_KEY = "artwork_url";
private final JsonObject searchResult;
private final JsonObject itemObject;
public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) {
this.itemObject = itemObject;
}
@Override
public String getName() throws ParsingException {
return searchResult.getString("title");
public String getName() {
return itemObject.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return searchResult.getString("permalink_url");
public String getUrl() {
return replaceHttpWithHttps(itemObject.getString("permalink_url"));
}
@Override
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
if (searchResult.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, "");
if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
if (!artworkUrl.isEmpty()) return artworkUrl;
}
try {
// Look for artwork url inside the track list
for (Object track : searchResult.getArray("tracks")) {
for (Object track : itemObject.getArray("tracks")) {
final JsonObject trackObject = (JsonObject) track;
// First look for track artwork url
@ -55,7 +57,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
try {
// Last resort, use user avatar url. If still not found, then throw exception.
return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
@ -64,14 +66,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
@Override
public String getUploaderName() throws ParsingException {
try {
return searchResult.getObject(USER_KEY).getString("username");
return itemObject.getObject(USER_KEY).getString("username");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
}
@Override
public long getStreamCount() throws ParsingException {
return searchResult.getNumber("track_count", 0).longValue();
public long getStreamCount() {
return itemObject.getNumber("track_count", 0).longValue();
}
}

4
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java
View file

@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
@ -45,7 +47,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
return ogElement.attr("content");
return replaceHttpWithHttps(ogElement.attr("content"));
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}

8
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View file

@ -13,7 +13,11 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -35,7 +39,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getCleanUrl() {
return track.isString("permalink_url") ? track.getString("permalink_url") : getOriginalUrl();
return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl();
}
@Nonnull

24
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java
View file

@ -5,52 +5,54 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
protected final JsonObject searchResult;
protected final JsonObject itemObject;
public SoundcloudStreamInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) {
this.itemObject = itemObject;
}
@Override
public String getUrl() {
return searchResult.getString("permalink_url");
return replaceHttpWithHttps(itemObject.getString("permalink_url"));
}
@Override
public String getName() {
return searchResult.getString("title");
return itemObject.getString("title");
}
@Override
public long getDuration() {
return searchResult.getNumber("duration", 0).longValue() / 1000L;
return itemObject.getNumber("duration", 0).longValue() / 1000L;
}
@Override
public String getUploaderName() {
return searchResult.getObject("user").getString("username");
return itemObject.getObject("user").getString("username");
}
@Override
public String getUploaderUrl() {
return searchResult.getObject("user").getString("permalink_url");
return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url"));
}
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
return SoundcloudParsingHelper.toDateString(itemObject.getString("created_at"));
}
@Override
public long getViewCount() {
return searchResult.getNumber("playback_count", 0).longValue();
return itemObject.getNumber("playback_count", 0).longValue();
}
@Override
public String getThumbnailUrl() {
return searchResult.getString("artwork_url");
return itemObject.getString("artwork_url");
}
@Override

4
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java
View file

@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
@ -48,7 +50,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
return ogElement.attr("content");
return replaceHttpWithHttps(ogElement.attr("content"));
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View file

@ -175,7 +175,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ParsingException("Could not parse json data for next streams", pe);
}
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl);
collectStreamsFrom(collector, ajaxHtml.select("body").first());
return new InfoItemPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));

13
src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View file

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.List;
public class Utils {
private Utils() {
//no instance
}
@ -46,5 +47,17 @@ public class Utils {
System.err.println("----------------");
}
}
private static final String HTTP = "http://";
private static final String HTTPS = "https://";
public static String replaceHttpWithHttps(final String url) {
if (url == null) return null;
if(!url.isEmpty() && url.startsWith(HTTP)) {
return HTTPS + url.substring(HTTP.length());
}
return url;
}
}