From 8f023c1ec7a505b0b413bc295c390ef7791cf389 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 26 Apr 2021 12:41:05 +0200 Subject: [PATCH 1/2] [SoundCloud] Update client id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 2c377846..e92610e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -40,7 +40,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.*; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "Kl9G8jQT22DxqatQk09IjWRujGlut5Vd"; // Updated on 04/03/21 + private static final String HARDCODED_CLIENT_ID = "NcIaRZItQCNQp3Vq9Plvzf7tvjmVJnF6"; // Updated on 26/04/21 private static String clientId; private SoundcloudParsingHelper() { From 9c12dc5609b7d1567e855b90a5a19520a7f6855f Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 26 Apr 2021 17:58:30 +0200 Subject: [PATCH 2/2] [SoundCloud] Fix SoundCloud ID extraction resolveIdWithEmbedPlayer() does not work anymore because the JSON data has been extracted to an API call. For this reason, replace resolveIdWithEmbedPlayer() with resolveIdWithWidgetApi)( which performs the API call. --- .../soundcloud/SoundcloudParsingHelper.java | 29 +++++++++++++------ .../SoundcloudChannelLinkHandlerFactory.java | 2 +- .../SoundcloudPlaylistLinkHandlerFactory.java | 2 +- .../SoundcloudStreamLinkHandlerFactory.java | 2 +- .../schabi/newpipe/extractor/utils/Utils.java | 9 ++++++ .../SoundcloudParsingHelperTest.java | 6 ++-- 6 files changed, 35 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index e92610e4..ecf831d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Utils; @@ -140,27 +141,37 @@ public class SoundcloudParsingHelper { } /** - * Fetch the embed player with the url and return the id (like the id from the json api). + * Fetch the widget API with the url and return the id (like the id from the json api). * * @return the resolved id */ - public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException { + public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException { // Remove the tailing slash from URLs due to issues with the SoundCloud API if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1); + // Make URL lower case and remove www. if it exists. + // Without doing this, the widget API does not recognize the URL. + urlString = Utils.removeWWWFromUrl(urlString.toLowerCase()); - URL url; + final URL url; try { url = Utils.stringToURL(urlString); } catch (MalformedURLException e) { throw new IllegalArgumentException("The given URL is not valid"); } - String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody(); - // handle playlists / sets different and get playlist id via uir field in JSON - if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets")) - return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response); - return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response); + try { + final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url=" + + URLEncoder.encode(url.toString(), UTF_8) + + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); + final String response = NewPipe.getDownloader().get(widgetUrl, + SoundCloud.getLocalization()).responseBody(); + final JsonObject o = JsonParser.object().from(response); + return String.valueOf(JsonUtils.getValue(o, "id")); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON response", e); + } catch (ExtractionException e) { + throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e); + } } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java index e49834eb..c1308a96 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java @@ -23,7 +23,7 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory Utils.checkUrl(URL_PATTERN, url); try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java index 3adade10..c89e261c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java @@ -22,7 +22,7 @@ public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory Utils.checkUrl(URL_PATTERN, url); try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); } catch (Exception e) { throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index 48aa836e..55eafa8a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -32,7 +32,7 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { Utils.checkUrl(URL_PATTERN, url); try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 82ff5826..baf502ea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -7,6 +7,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.*; +import java.util.regex.Pattern; public class Utils { @@ -14,6 +15,7 @@ public class Utils { public static final String HTTPS = "https://"; public static final String UTF_8 = "UTF-8"; public static final String EMPTY_STRING = ""; + private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\."); private Utils() { //no instance @@ -170,6 +172,13 @@ public class Utils { return setsNoPort || usesDefaultPort; } + public static String removeWWWFromUrl(String url) { + if (WWW_PATTERN.matcher(url).find()) { + return url.replace("www.", ""); + } + return url; + } + public static String removeUTF8BOM(String s) { if (s.startsWith("\uFEFF")) { s = s.substring(1); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index f1a9b35c..7d5c156b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -29,9 +29,9 @@ public class SoundcloudParsingHelperTest { } @Test - public void resolveIdWithEmbedPlayerTest() throws Exception { - Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity")); - Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds")); + public void resolveIdWithWidgetApiTest() throws Exception { + Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity")); + Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds")); }