CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[SoundCloud] Fix SoundCloud ID extraction

Browse source 
resolveIdWithEmbedPlayer() does not work anymore because the JSON data has been extracted to an API call. For this reason, replace resolveIdWithEmbedPlayer() with resolveIdWithWidgetApi)( which performs the API call.
This commit is contained in:
TobiGr 2021-04-26 17:58:30 +02:00
parent 8f023c1ec7
commit 9c12dc5609
6 changed files with 35 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file

@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
@ -140,27 +141,37 @@ public class SoundcloudParsingHelper {
}
/**
* Fetch the embed player with the url and return the id (like the id from the json api).
* Fetch the widget API with the url and return the id (like the id from the json api).
*
* @return the resolved id
*/
public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException {
// Remove the tailing slash from URLs due to issues with the SoundCloud API
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
// Make URL lower case and remove www. if it exists.
// Without doing this, the widget API does not recognize the URL.
urlString = Utils.removeWWWFromUrl(urlString.toLowerCase());
URL url;
final URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
try {
final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
+ URLEncoder.encode(url.toString(), UTF_8)
+ "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
final String response = NewPipe.getDownloader().get(widgetUrl,
SoundCloud.getLocalization()).responseBody();
final JsonObject o = JsonParser.object().from(response);
return String.valueOf(JsonUtils.getValue(o, "id"));
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON response", e);
} catch (ExtractionException e) {
throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e);
}
}
/**

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java
View file

@ -23,7 +23,7 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java
View file

@ -22,7 +22,7 @@ public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java
View file

@ -32,7 +32,7 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View file

@ -7,6 +7,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.*;
import java.util.regex.Pattern;
public class Utils {
@ -14,6 +15,7 @@ public class Utils {
public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
private Utils() {
//no instance
@ -170,6 +172,13 @@ public class Utils {
return setsNoPort || usesDefaultPort;
}
public static String removeWWWFromUrl(String url) {
if (WWW_PATTERN.matcher(url).find()) {
return url.replace("www.", "");
}
return url;
}
public static String removeUTF8BOM(String s) {
if (s.startsWith("\uFEFF")) {
s = s.substring(1);

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java
View file

@ -29,9 +29,9 @@ public class SoundcloudParsingHelperTest {
}
@Test
public void resolveIdWithEmbedPlayerTest() throws Exception {
Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity"));
Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds"));
public void resolveIdWithWidgetApiTest() throws Exception {
Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity"));
Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds"));
}