[SoundCloud] Fix SoundCloud ID extraction

resolveIdWithEmbedPlayer() does not work anymore because the JSON data has been extracted to an API call. For this reason, replace resolveIdWithEmbedPlayer() with resolveIdWithWidgetApi)( which performs the API call.

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCha
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -140,27 +141,37 @@ public class SoundcloudParsingHelper {
     }
 
     /**
-     * Fetch the embed player with the url and return the id (like the id from the json api).
+     * Fetch the widget API with the url and return the id (like the id from the json api).
      *
      * @return the resolved id
      */
-    public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
+    public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException {
         // Remove the tailing slash from URLs due to issues with the SoundCloud API
         if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
+        // Make URL lower case and remove www. if it exists.
+        // Without doing this, the widget API does not recognize the URL.
+        urlString = Utils.removeWWWFromUrl(urlString.toLowerCase());
 
-        URL url;
+        final URL url;
         try {
             url = Utils.stringToURL(urlString);
         } catch (MalformedURLException e) {
             throw new IllegalArgumentException("The given URL is not valid");
         }
 
-        String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+        try {
-                + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
+            final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
-        // handle playlists / sets different and get playlist id via uir field in JSON
+                    + URLEncoder.encode(url.toString(), UTF_8)
-        if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
+                    + "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
-            return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
+            final String response = NewPipe.getDownloader().get(widgetUrl,
-        return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
+                    SoundCloud.getLocalization()).responseBody();
+            final JsonObject o = JsonParser.object().from(response);
+            return String.valueOf(JsonUtils.getValue(o, "id"));
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse JSON response", e);
+        } catch (ExtractionException e) {
+            throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e);
+        }
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java

@@ -23,7 +23,7 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java

@@ -22,7 +22,7 @@ public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java

@@ -32,7 +32,7 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -7,6 +7,7 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.util.*;
+import java.util.regex.Pattern;
 
 public class Utils {
 
@@ -14,6 +15,7 @@ public class Utils {
     public static final String HTTPS = "https://";
     public static final String UTF_8 = "UTF-8";
     public static final String EMPTY_STRING = "";
+    private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
 
     private Utils() {
         //no instance
@@ -170,6 +172,13 @@ public class Utils {
         return setsNoPort || usesDefaultPort;
     }
 
+    public static String removeWWWFromUrl(String url) {
+        if (WWW_PATTERN.matcher(url).find()) {
+            return url.replace("www.", "");
+        }
+        return url;
+    }
+
     public static String removeUTF8BOM(String s) {
         if (s.startsWith("\uFEFF")) {
             s = s.substring(1);

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java

@@ -29,9 +29,9 @@ public class SoundcloudParsingHelperTest {
     }
 
     @Test
-    public void resolveIdWithEmbedPlayerTest() throws Exception {
+    public void resolveIdWithWidgetApiTest() throws Exception {
-        Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity"));
+        Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity"));
-        Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds"));
+        Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds"));
 
     }