CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Format the SoundCloud package of the extractor and fix some warnings

Browse source 
Use final where possible in the package and format code to be in the 100 caracters per line limit.
Fix some warnings generated by Android Studio and do some code improvements
This commit is contained in:
TiA4f8R 2021-05-15 17:51:43 +02:00
parent 636e27333b
commit c5c190500c
No known key found for this signature in database
GPG key ID: E6D3E7F5949450DD
21 changed files with 387 additions and 291 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

180
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file

@ -41,7 +41,8 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "NcIaRZItQCNQp3Vq9Plvzf7tvjmVJnF6"; // Updated on 26/04/21
private static final String HARDCODED_CLIENT_ID =
"NcIaRZItQCNQp3Vq9Plvzf7tvjmVJnF6"; // Updated on 26/04/21
private static String clientId;
private SoundcloudParsingHelper() {
@ -50,7 +51,7 @@ public class SoundcloudParsingHelper {
public static synchronized String clientId() throws ExtractionException, IOException {
if (!isNullOrEmpty(clientId)) return clientId;
Downloader dl = NewPipe.getDownloader();
final Downloader dl = NewPipe.getDownloader();
clientId = HARDCODED_CLIENT_ID;
if (checkIfHardcodedClientIdIsValid()) {
return clientId;
@ -62,20 +63,22 @@ public class SoundcloudParsingHelper {
final String responseBody = download.responseBody();
final String clientIdPattern = ",client_id:\"(.*?)\"";
Document doc = Jsoup.parse(responseBody);
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
final Document doc = Jsoup.parse(responseBody);
final Elements possibleScripts = doc.select(
"script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
// The one containing the client id will likely be the last one
Collections.reverse(possibleScripts);
final HashMap<String, List<String>> headers = new HashMap<>();
headers.put("Range", singletonList("bytes=0-50000"));
for (Element element : possibleScripts) {
for (final Element element : possibleScripts) {
final String srcUrl = element.attr("src");
if (!isNullOrEmpty(srcUrl)) {
try {
return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers).responseBody());
} catch (RegexException ignored) {
return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers)
.responseBody());
} catch (final RegexException ignored) {
// Ignore it and proceed to try searching other script
}
}
@ -88,74 +91,84 @@ public class SoundcloudParsingHelper {
static boolean checkIfHardcodedClientIdIsValid() {
try {
SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud
.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
.getStreamExtractor(
"https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
e.fetchPage();
return !e.getAudioStreams().isEmpty();
} catch (Exception ignored) {
} catch (final Exception ignored) {
// No need to throw an exception here. If something went wrong, the client_id is wrong
return false;
}
}
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
public static OffsetDateTime parseDateFrom(final String textualUploadDate)
throws ParsingException {
try {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e1) {
} catch (final DateTimeParseException e1) {
try {
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter
.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (final DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\""
+ ", " + e1.getMessage(), e2);
}
}
}
/**
* Call the endpoint "/resolve" of the api.<p>
* Call the endpoint "/resolve" of the API.<p>
* <p>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api-v2.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, UTF_8)
+ "&client_id=" + clientId();
public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
throws IOException, ExtractionException {
final String apiUrl = "https://api-v2.soundcloud.com/resolve" + "?url="
+ URLEncoder.encode(url, UTF_8) + "&client_id=" + clientId();
try {
final String response = downloader.get(apiUrl, SoundCloud.getLocalization()).responseBody();
final String response = downloader.get(apiUrl, SoundCloud.getLocalization())
.responseBody();
return JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
/**
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url
* from the json API).
*
* @return the url resolved
*/
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException {
public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException,
ReCaptchaException {
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first()
.attr("abs:href");
}
/**
* Fetch the widget API with the url and return the id (like the id from the json api).
* Fetch the widget API with the url and return the id (like the id from the json API).
*
* @return the resolved id
*/
public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException {
public static String resolveIdWithWidgetApi(String urlString) throws IOException,
ParsingException {
// Remove the tailing slash from URLs due to issues with the SoundCloud API
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
// Make URL lower case and remove www. if it exists.
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0,
urlString.length() - 1);
// Make URL lower case and remove m. and www. if it exists.
// Without doing this, the widget API does not recognize the URL.
urlString = Utils.removeWWWFromUrl(urlString.toLowerCase());
urlString = Utils.removeMAndWWWFromUrl(urlString.toLowerCase());
final URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
@ -167,22 +180,27 @@ public class SoundcloudParsingHelper {
SoundCloud.getLocalization()).responseBody();
final JsonObject o = JsonParser.object().from(response);
return String.valueOf(JsonUtils.getValue(o, "id"));
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON response", e);
} catch (ExtractionException e) {
throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e);
} catch (final ExtractionException e) {
throw new ParsingException(
"Could not resolve id with embedded player. ClientId not extracted", e);
}
}
/**
* Fetch the users from the given api and commit each of them to the collector.
* Fetch the users from the given API and commit each of them to the collector.
* <p>
* This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense that they will always
* get MIN_ITEMS or more.
* This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense
* that they will always get MIN_ITEMS or more.
*
* @param minItems the method will return only when it have extracted that many items (equal or more)
* @param minItems the method will return only when it have extracted that many items
* (equal or more)
*/
public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
public static String getUsersFromApiMinItems(final int minItems,
final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@ -193,23 +211,27 @@ public class SoundcloudParsingHelper {
}
/**
* Fetch the user items from the given api and commit each of them to the collector.
* Fetch the user items from the given API and commit each of them to the collector.
*
* @return the next streams url, empty if don't have
*/
public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
JsonObject responseObject;
public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
final String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization())
.responseBody();
final JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) {
final JsonArray responseCollection = responseObject.getArray("collection");
for (final Object o : responseCollection) {
if (o instanceof JsonObject) {
JsonObject object = (JsonObject) o;
final JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudChannelInfoItemExtractor(object));
}
}
@ -217,8 +239,9 @@ public class SoundcloudParsingHelper {
String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) {
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id="
+ SoundcloudParsingHelper.clientId();
} catch (final Exception ignored) {
nextPageUrl = "";
}
@ -226,14 +249,18 @@ public class SoundcloudParsingHelper {
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
* Fetch the streams from the given API and commit each of them to the collector.
* <p>
* This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense that they will always
* get MIN_ITEMS or more items.
* This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense
* that they will always get MIN_ITEMS or more items.
*
* @param minItems the method will return only when it have extracted that many items (equal or more)
* @param minItems the method will return only when it have extracted that many items
* (equal or more)
*/
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
public static String getStreamsFromApiMinItems(final int minItems,
final StreamInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@ -244,59 +271,68 @@ public class SoundcloudParsingHelper {
}
/**
* Fetch the streams from the given api and commit each of them to the collector.
* Fetch the streams from the given API and commit each of them to the collector.
*
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl,
final boolean charts) throws IOException,
ReCaptchaException, ParsingException {
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud
.getLocalization());
if (response.responseCode() >= 400) {
throw new IOException("Could not get streams from API, HTTP " + response.responseCode());
throw new IOException("Could not get streams from API, HTTP " + response
.responseCode());
}
JsonObject responseObject;
final JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) {
final JsonArray responseCollection = responseObject.getArray("collection");
for (final Object o : responseCollection) {
if (o instanceof JsonObject) {
JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudStreamInfoItemExtractor(charts ? object.getObject("track") : object));
final JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudStreamInfoItemExtractor(charts
? object.getObject("track") : object));
}
}
String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) {
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id="
+ SoundcloudParsingHelper.clientId();
} catch (final Exception ignored) {
nextPageUrl = "";
}
return nextPageUrl;
}
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl) throws ReCaptchaException,
ParsingException, IOException {
return getStreamsFromApi(collector, apiUrl, false);
}
@Nonnull
public static String getUploaderUrl(JsonObject object) {
String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
public static String getUploaderUrl(final JsonObject object) {
final String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
@Nonnull
public static String getAvatarUrl(JsonObject object) {
String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
public static String getAvatarUrl(final JsonObject object) {
final String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
public static String getUploaderName(JsonObject object) {
public static String getUploaderName(final JsonObject object) {
return object.getObject("user").getString("username", EMPTY_STRING);
}
}

33
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View file

@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -23,7 +22,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) {
public SoundcloudService(final int id) {
super(id, "SoundCloud", asList(AUDIO, COMMENTS));
}
@ -54,29 +53,29 @@ public class SoundcloudService extends StreamingService {
@Override
public List<ContentCountry> getSupportedCountries() {
//Country selector here https://soundcloud.com/charts/top?genre=all-music
// Country selector here: https://soundcloud.com/charts/top?genre=all-music
return ContentCountry.listFrom(
"AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US"
);
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) {
return new SoundcloudStreamExtractor(this, LinkHandler);
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
return new SoundcloudStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler);
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) {
return new SoundcloudSearchExtractor(this, queryHandler);
}
@ -87,18 +86,11 @@ public class SoundcloudService extends StreamingService {
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id)
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
new SoundcloudChartsExtractor(SoundcloudService.this,
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
}
};
KioskList list = new KioskList(this);
final KioskList list = new KioskList(this);
// add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
@ -106,7 +98,7 @@ public class SoundcloudService extends StreamingService {
list.addKioskEntry(chartsFactory, h, "Top 50");
list.addKioskEntry(chartsFactory, h, "New & hot");
list.setDefaultKiosk("New & hot");
} catch (Exception e) {
} catch (final Exception e) {
throw new ExtractionException(e);
}
@ -124,9 +116,8 @@ public class SoundcloudService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {
return new SoundcloudCommentsExtractor(this, linkHandler);
}
}

28
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View file

@ -25,12 +25,14 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
private JsonObject user;
public SoundcloudChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
public SoundcloudChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
userId = getLinkHandler().getId();
final String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
@ -39,7 +41,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@ -63,7 +65,8 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() {
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url");
return user.getObject("visuals").getArray("visuals").getObject(0)
.getString("visual_url");
}
@Override
@ -105,29 +108,32 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
try {
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
final String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "?client_id=" + SoundcloudParsingHelper.clientId() + "&limit=20"
+ "&linked_partitioning=1";
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15,
streamInfoItemsCollector, apiUrl);
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
} catch (Exception e) {
} catch (final Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View file

@ -9,7 +9,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private final JsonObject itemObject;
public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) {
public SoundcloudChannelInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject;
}
@ -26,8 +26,8 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getThumbnailUrl() {
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
return avatarUrlBetterResolution;
// An avatar URL with a better resolution
return avatarUrl.replace("large.jpg", "crop.jpg");
}
@Override

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
View file

@ -18,14 +18,14 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
public SoundcloudChartsExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId) {
public SoundcloudChartsExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final String kioskId) {
super(service, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) {
public void onFetchPage(@Nonnull final Downloader downloader) {
}
@Nonnull
@ -35,13 +35,15 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true);
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
page.getUrl(), true);
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
@ -52,8 +54,8 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" +
"&client_id=" + SoundcloudParsingHelper.clientId();
"?genre=soundcloud:genres:all-music" + "&client_id="
+ SoundcloudParsingHelper.clientId();
if (getId().equals("Top 50")) {
apiUrl += "&kind=top";
@ -64,15 +66,18 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
final ContentCountry contentCountry = SoundCloud.getContentCountry();
String apiUrlWithRegion = null;
if (getService().getSupportedCountries().contains(contentCountry)) {
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
+ contentCountry.getCountryCode();
}
String nextPageUrl;
try {
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
} catch (IOException e) {
// Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
// we retry without the specified region.
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
} catch (final IOException e) {
// Request to other region may be geo-restricted.
// See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
// We retry without the specified region.
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View file

@ -24,24 +24,27 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudCommentsExtractor extends CommentsExtractor {
public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
public SoundcloudCommentsExtractor(final StreamingService service,
final ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Nonnull
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException, IOException {
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
IOException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(getUrl());
final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
@ -49,7 +52,8 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException, IOException {
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException,
IOException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@ -60,11 +64,12 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
@ -74,9 +79,10 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader) { }
private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException {
private void collectStreamsFrom(final CommentsInfoItemsCollector collector,
final JsonArray entries) throws ParsingException {
final String url = getUrl();
for (Object comment : entries) {
for (final Object comment : entries) {
collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url));
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java
View file

@ -10,10 +10,10 @@ import javax.annotation.Nullable;
import java.util.Objects;
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private JsonObject json;
private String url;
private final JsonObject json;
private final String url;
public SoundcloudCommentsInfoItemExtractor(JsonObject json, String url) {
public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final String url) {
this.json = json;
this.url = url;
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java
View file

@ -33,22 +33,23 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
public SoundcloudPlaylistExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
playlistId = getLinkHandler().getId();
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
final String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() + "&representation=compact";
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}
@ -76,11 +77,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
try {
final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage();
for (StreamInfoItem item : infoItems.getItems()) {
for (final StreamInfoItem item : infoItems.getItems()) {
artworkUrl = item.getThumbnailUrl();
if (!isNullOrEmpty(artworkUrl)) break;
}
} catch (Exception ignored) {
} catch (final Exception ignored) {
}
if (artworkUrl == null) {
@ -139,18 +140,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return "";
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
final List<String> ids = new ArrayList<>();
final JsonArray tracks = playlist.getArray("tracks");
for (Object o : tracks) {
for (final Object o : tracks) {
if (o instanceof JsonObject) {
final JsonObject track = (JsonObject) o;
if (track.has("title")) { // i.e. if full info is available
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
} else {
// %09d would be enough, but a 0 before the number does not create problems, so let's be sure
// %09d would be enough, but a 0 before the number does not create problems, so
// let's be sure
ids.add(String.format("%010d", track.getInt("id")));
}
}
@ -160,7 +165,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getIds())) {
throw new IllegalArgumentException("Page doesn't contain IDs");
}
@ -177,20 +183,20 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
}
final String currentPageUrl = "https://api-v2.soundcloud.com/tracks?client_id="
+ SoundcloudParsingHelper.clientId()
+ "&ids=" + Utils.join(",", currentIds);
+ SoundcloudParsingHelper.clientId() + "&ids=" + Utils.join(",", currentIds);
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody();
final String response = NewPipe.getDownloader().get(currentPageUrl,
getExtractorLocalization()).responseBody();
try {
final JsonArray tracks = JsonParser.array().from(response);
for (Object track : tracks) {
for (final Object track : tracks) {
if (track instanceof JsonObject) {
collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
}
}
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View file

@ -14,7 +14,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
private final JsonObject itemObject;
public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) {
public SoundcloudPlaylistInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject;
}
@ -34,22 +34,22 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
// An artwork URL with a better resolution
return artworkUrl.replace("large.jpg", "crop.jpg");
}
}
try {
// Look for artwork url inside the track list
for (Object track : itemObject.getArray("tracks")) {
for (final Object track : itemObject.getArray("tracks")) {
final JsonObject trackObject = (JsonObject) track;
// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
// An artwork URL with a better resolution
return artworkUrl.replace("large.jpg", "crop.jpg");
}
}
@ -58,14 +58,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING);
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
} catch (final Exception ignored) {
// Try other method
}
try {
// Last resort, use user avatar url. If still not found, then throw exception.
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
}
@ -74,7 +74,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
public String getUploaderName() throws ParsingException {
try {
return itemObject.getObject(USER_KEY).getString("username");
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
}

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View file

@ -28,7 +28,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray searchCollection;
public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
public SoundcloudSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@ -52,34 +53,39 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(getUrl()));
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(
getUrl()));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Downloader dl = getDownloader();
try {
final String response = dl.get(page.getUrl(), getExtractorLocalization()).responseBody();
final String response = dl.get(page.getUrl(), getExtractorLocalization())
.responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page.getUrl()));
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page
.getUrl()));
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final Downloader dl = getDownloader();
final String url = getUrl();
try {
final String response = dl.get(url, getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
@ -88,14 +94,14 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
}
}
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(JsonArray searchCollection) {
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(
final JsonArray searchCollection) {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
for (Object result : searchCollection) {
for (final Object result : searchCollection) {
if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions
JsonObject searchResult = (JsonObject) result;
String kind = searchResult.getString("kind", EMPTY_STRING);
final JsonObject searchResult = (JsonObject) result;
final String kind = searchResult.getString("kind", EMPTY_STRING);
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
@ -112,15 +118,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
return collector;
}
private Page getNextPageFromCurrentUrl(String currentUrl)
private Page getNextPageFromCurrentUrl(final String currentUrl)
throws MalformedURLException, UnsupportedEncodingException {
final int pageOffset = Integer.parseInt(
Parser.compatParseMap(
new URL(currentUrl)
.getQuery())
.get("offset"));
Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
return new Page(currentUrl.replace("&offset=" + pageOffset,
"&offset=" + (pageOffset + ITEMS_PER_PAGE)));
return new Page(currentUrl.replace("&offset=" + pageOffset, "&offset="
+ (pageOffset + ITEMS_PER_PAGE)));
}
}

46
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View file

@ -36,12 +36,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
private boolean isAvailable = true;
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) {
public SoundcloudStreamExtractor(final StreamingService service,
final LinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getUrl());
final String policy = track.getString("policy", EMPTY_STRING);
@ -50,9 +52,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
if (policy.equals("SNIP")) {
throw new SoundCloudGoPlusContentException();
}
if (policy.equals("BLOCK")) {
throw new GeographicRestrictionException("This track is not available in user's country");
}
if (policy.equals("BLOCK")) throw new GeographicRestrictionException(
"This track is not available in user's country");
throw new ContentNotAvailableException("Content not available: policy " + policy);
}
}
@ -80,7 +81,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString("created_at")));
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString(
"created_at")));
}
@Nonnull
@ -220,9 +222,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Nonnull
private static String getTranscodingUrl(final String endpointUrl, final String protocol) throws IOException, ExtractionException {
private static String getTranscodingUrl(final String endpointUrl,
final String protocol)
throws IOException, ExtractionException {
final Downloader downloader = NewPipe.getDownloader();
final String apiStreamUrl = endpointUrl + "?client_id=" + SoundcloudParsingHelper.clientId();
final String apiStreamUrl = endpointUrl + "?client_id="
+ SoundcloudParsingHelper.clientId();
final String response = downloader.get(apiStreamUrl).responseBody();
final JsonObject urlObject;
try {
@ -255,7 +260,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
final String mediaUrl;
final String preset = transcodingJsonObject.getString("preset");
final String protocol = transcodingJsonObject.getObject("format").getString("protocol");
final String protocol = transcodingJsonObject.getObject("format")
.getString("protocol");
MediaFormat mediaFormat = null;
int bitrate = 0;
if (preset.contains("mp3")) {
@ -285,7 +291,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
}
/** Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
/**
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
* <p>
* This method downloads the provided manifest URL, find all web occurrences in the manifest,
* get the last segment URL, changes its segment range to {@code 0/track-length} and return
@ -293,7 +300,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
* @param hlsManifestUrl the URL of the manifest to be parsed
* @return a single URL that contains a range equal to the length of the track
*/
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl) throws ParsingException {
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl)
throws ParsingException {
final Downloader dl = NewPipe.getDownloader();
final String hlsManifestResponse;
@ -306,11 +314,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
final String[] lines = hlsManifestResponse.split("\\r?\\n");
for (int l = lines.length - 1; l >= 0; l--) {
final String line = lines[l];
// get the last URL from manifest, because it contains the range of the stream
// Get the last URL from manifest, because it contains the range of the stream
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
final String[] hlsLastRangeUrlArray = line.split("/");
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + "/"
+ hlsLastRangeUrlArray[6];
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
+ "/" + hlsLastRangeUrlArray[6];
}
}
throw new ParsingException("Could not get any URL from HLS manifest");
@ -399,15 +407,15 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
// tags are separated by spaces, but they can be multiple words escaped by quotes "
final String[] tag_list = track.getString("tag_list").split(" ");
// Tags are separated by spaces, but they can be multiple words escaped by quotes "
final String[] tagList = track.getString("tag_list").split(" ");
final List<String> tags = new ArrayList<>();
String escapedTag = "";
boolean isEscaped = false;
for (int i = 0; i < tag_list.length; i++) {
String tag = tag_list[i];
for (int i = 0; i < tagList.length; i++) {
String tag = tagList[i];
if (tag.startsWith("\"")) {
escapedTag += tag_list[i].replace("\"", "");
escapedTag += tagList[i].replace("\"", "");
isEscaped = true;
} else if (isEscaped) {
if (tag.endsWith("\"")) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View file

@ -14,7 +14,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
protected final JsonObject itemObject;
public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) {
public SoundcloudStreamInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject;
}

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java
View file

@ -13,12 +13,15 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
/**
* Extract the "followings" from a user in SoundCloud.
*/
public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
public SoundcloudSubscriptionExtractor(SoundcloudService service) {
public SoundcloudSubscriptionExtractor(final SoundcloudService service) {
super(service, Collections.singletonList(ContentSource.CHANNEL_URL));
}
@ -28,20 +31,22 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
}
@Override
public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
if (channelUrl == null) throw new InvalidSourceException("channel url is null");
public List<SubscriptionItem> fromChannelUrl(final String channelUrl) throws IOException,
ExtractionException {
if (channelUrl == null) throw new InvalidSourceException("Channel url is null");
String id;
final String id;
try {
id = service.getChannelLHFactory().fromUrl(getUrlFrom(channelUrl)).getId();
} catch (ExtractionException e) {
} catch (final ExtractionException e) {
throw new InvalidSourceException(e);
}
String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings"
final String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=200";
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId());
final ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service
.getServiceId());
// ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough
SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl);
@ -49,13 +54,13 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
}
private String getUrlFrom(String channelUrl) {
channelUrl = channelUrl.replace("http://", "https://").trim();
channelUrl = replaceHttpWithHttps(channelUrl);
if (!channelUrl.startsWith("https://")) {
if (!channelUrl.startsWith(HTTPS)) {
if (!channelUrl.contains("soundcloud.com/")) {
channelUrl = "https://soundcloud.com/" + channelUrl;
} else {
channelUrl = "https://" + channelUrl;
channelUrl = HTTPS + channelUrl;
}
}
@ -66,9 +71,9 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
// Utils
//////////////////////////////////////////////////////////////////////////*/
private List<SubscriptionItem> toSubscriptionItems(List<ChannelInfoItem> items) {
List<SubscriptionItem> result = new ArrayList<>(items.size());
for (ChannelInfoItem item : items) {
private List<SubscriptionItem> toSubscriptionItems(final List<ChannelInfoItem> items) {
final List<SubscriptionItem> result = new ArrayList<>(items.size());
for (final ChannelInfoItem item : items) {
result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName()));
}
return result;

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java
View file

@ -21,30 +21,29 @@ import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public SoundcloudSuggestionExtractor(StreamingService service) {
public SoundcloudSuggestionExtractor(final StreamingService service) {
super(service);
}
@Override
public List<String> suggestionList(String query) throws IOException, ExtractionException {
List<String> suggestions = new ArrayList<>();
public List<String> suggestionList(final String query) throws IOException,
ExtractionException {
final List<String> suggestions = new ArrayList<>();
final Downloader dl = NewPipe.getDownloader();
final String url = "https://api-v2.soundcloud.com/search/queries" + "?q="
+ URLEncoder.encode(query, UTF_8) + "&client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=10";
final String response = dl.get(url, getExtractorLocalization()).responseBody();
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";
String response = dl.get(url, getExtractorLocalization()).responseBody();
try {
JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (Object suggestion : collection) {
if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query"));
final JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (final Object suggestion : collection) {
if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion)
.getString("query"));
}
return suggestions;
} catch (JsonParserException e) {
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java
View file

@ -9,9 +9,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.util.List;
public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudChannelLinkHandlerFactory instance = new SoundcloudChannelLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
private static final SoundcloudChannelLinkHandlerFactory instance =
new SoundcloudChannelLinkHandlerFactory();
private static final String URL_PATTERN ="^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
public static SoundcloudChannelLinkHandlerFactory getInstance() {
return instance;
@ -19,21 +20,24 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id);
} catch (Exception e) {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/users/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java
View file

@ -6,12 +6,13 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
private static final String TOP_URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private static final String URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
@Override
public String getId(String url) {
public String getId(final String url) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50";
} else {
@ -20,7 +21,9 @@ public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) {
if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top";
} else {

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java
View file

@ -11,36 +11,40 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsing
public class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudCommentsLinkHandlerFactory instance = new SoundcloudCommentsLinkHandlerFactory();
private static final SoundcloudCommentsLinkHandlerFactory instance =
new SoundcloudCommentsLinkHandlerFactory();
public static SoundcloudCommentsLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try {
return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" + clientId() +
"&threaded=0" + "&filter_replies=1"; // anything but 1 = sort by new
return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id="
+ clientId() + "&threaded=0" + "&filter_replies=1";
// Anything but 1 = sort by new
// + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10)
// + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl)
} catch (ExtractionException | IOException e) {
} catch (final ExtractionException | IOException e) {
throw new ParsingException("Could not get comments");
}
}
@Override
public String getId(String url) throws ParsingException {
// delagation to avoid duplicate code, as we need the same id
public String getId(final String url) throws ParsingException {
// Delegation to avoid duplicate code, as we need the same id
return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url);
}
@Override
public boolean onAcceptUrl(String url) {
public boolean onAcceptUrl(final String url) {
try {
getId(url);
return true;
} catch (ParsingException e) {
} catch (final ParsingException e) {
return false;
}
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java
View file

@ -9,30 +9,36 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.util.List;
public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudPlaylistLinkHandlerFactory instance = new SoundcloudPlaylistLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/sets/[0-9a-z_-]+/?([#?].*)?$";
private static final SoundcloudPlaylistLinkHandlerFactory instance =
new SoundcloudPlaylistLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "/sets/[0-9a-z_-]+/?([#?].*)?$";
public static SoundcloudPlaylistLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
} catch (final Exception e) {
throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(),
e);
}
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id);
} catch (Exception e) {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/playlists/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java
View file

@ -23,11 +23,14 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
public static final int ITEMS_PER_PAGE = 10;
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
try {
String url = "https://api-v2.soundcloud.com/search";
if (contentFilter.size() > 0) {
if (!contentFilter.isEmpty()) {
switch (contentFilter.get(0)) {
case TRACKS:
url += "/tracks";
@ -44,16 +47,15 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
}
}
return url + "?q=" + URLEncoder.encode(id, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=" + ITEMS_PER_PAGE
return url + "?q=" + URLEncoder.encode(id, UTF_8) + "&client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=" + ITEMS_PER_PAGE
+ "&offset=0";
} catch (UnsupportedEncodingException e) {
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
} catch (ReCaptchaException e) {
} catch (final ReCaptchaException e) {
throw new ParsingException("ReCaptcha required", e);
} catch (IOException | ExtractionException e) {
} catch (final IOException | ExtractionException e) {
throw new ParsingException("Could not get client id", e);
}
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java
View file

@ -7,9 +7,10 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final SoundcloudStreamLinkHandlerFactory instance = new SoundcloudStreamLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
private static final SoundcloudStreamLinkHandlerFactory instance =
new SoundcloudStreamLinkHandlerFactory();
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
private SoundcloudStreamLinkHandlerFactory() {
}
@ -19,21 +20,22 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
}
@Override
public String getUrl(String id) throws ParsingException {
public String getUrl(final String id) throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id);
} catch (Exception e) {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
"https://api.soundcloud.com/tracks/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getId(String url) throws ParsingException {
public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View file

@ -15,10 +15,11 @@ public class Utils {
public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
private Utils() {
//no instance
// no instance
}
/**
@ -30,7 +31,7 @@ public class Utils {
* @param toRemove string to remove non-digit chars
* @return a string that contains only digits
*/
public static String removeNonDigitCharacters(String toRemove) {
public static String removeNonDigitCharacters(final String toRemove) {
return toRemove.replaceAll("\\D+", "");
}
@ -48,7 +49,8 @@ public class Utils {
* @throws NumberFormatException
* @throws ParsingException
*/
public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException {
public static long mixedNumberWordToLong(final String numberWord) throws NumberFormatException,
ParsingException {
String multiplier = "";
try {
multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
@ -74,7 +76,7 @@ public class Utils {
* @param pattern the pattern that will be used to check the url
* @param url the url to be tested
*/
public static void checkUrl(String pattern, String url) throws ParsingException {
public static void checkUrl(final String pattern, final String url) throws ParsingException {
if (isNullOrEmpty(url)) {
throw new IllegalArgumentException("Url can't be null or empty");
}
@ -101,14 +103,14 @@ public class Utils {
}
/**
* get the value of a URL-query by name.
* if a url-query is give multiple times, only the value of the first query is returned
* Get the value of a URL-query by name.
* If a url-query is give multiple times, only the value of the first query is returned
*
* @param url the url to be used
* @param parameterName the pattern that will be used to check the url
* @return a string that contains the value of the query parameter or null if nothing was found
*/
public static String getQueryValue(URL url, String parameterName) {
public static String getQueryValue(final URL url, final String parameterName) {
String urlQuery = url.getQuery();
if (urlQuery != null) {
@ -118,8 +120,9 @@ public class Utils {
String query;
try {
query = URLDecoder.decode(params[0], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
} catch (final UnsupportedEncodingException e) {
System.err.println(
"Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
query = params[0];
}
@ -127,8 +130,9 @@ public class Utils {
if (query.equals(parameterName)) {
try {
return URLDecoder.decode(params[1], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
} catch (final UnsupportedEncodingException e) {
System.err.println(
"Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
return params[1];
}
@ -146,7 +150,7 @@ public class Utils {
* @param url the string to be converted to a URL-Object
* @return a URL-Object containing the url
*/
public static URL stringToURL(String url) throws MalformedURLException {
public static URL stringToURL(final String url) throws MalformedURLException {
try {
return new URL(url);
} catch (MalformedURLException e) {
@ -159,7 +163,7 @@ public class Utils {
}
}
public static boolean isHTTP(URL url) {
public static boolean isHTTP(final URL url) {
// make sure its http or https
String protocol = url.getProtocol();
if (!protocol.equals("http") && !protocol.equals("https")) {
@ -172,7 +176,10 @@ public class Utils {
return setsNoPort || usesDefaultPort;
}
public static String removeWWWFromUrl(String url) {
public static String removeMAndWWWFromUrl(final String url) {
if (M_PATTERN.matcher(url).find()) {
return url.replace("m.", "");
}
if (WWW_PATTERN.matcher(url).find()) {
return url.replace("www.", "");
}
@ -216,7 +223,8 @@ public class Utils {
try {
final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url),
UTF_8);
}
} catch (final Exception ignored) {
}
@ -258,7 +266,8 @@ public class Utils {
return true;
}
public static String join(final CharSequence delimiter, final Iterable<? extends CharSequence> elements) {
public static String join(final CharSequence delimiter,
final Iterable<? extends CharSequence> elements) {
final StringBuilder stringBuilder = new StringBuilder();
final Iterator<? extends CharSequence> iterator = elements.iterator();
while (iterator.hasNext()) {
@ -283,7 +292,8 @@ public class Utils {
/**
* Concatenate all non-null, non-empty and strings which are not equal to <code>"null"</code>.
*/
public static String nonEmptyAndNullJoin(final CharSequence delimiter, final String[] elements) {
public static String nonEmptyAndNullJoin(final CharSequence delimiter,
final String[] elements) {
final List<String> list = new java.util.ArrayList<>(Arrays.asList(elements));
list.removeIf(s -> isNullOrEmpty(s) || s.equals("null"));
return join(delimiter, list);