Merge branch 'dev'

extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor;
 
+import edu.umd.cs.findbugs.annotations.NonNull;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
@@ -14,42 +15,28 @@ public abstract class Extractor {
      */
     private final StreamingService service;
 
-    /**
+    private final UrlIdHandler urlIdHandler;
-     * Dirty/original url that was passed in the constructor.
-     * <p>
-     * What makes a url "dirty" or not is, for example, the additional parameters
-     * (not important as—in this case—the id):
-     * <pre>
-     *     https://www.youtube.com/watch?v=a9Zf_258aTI<i>&amp;t=4s</i>  →  <i><b>&amp;t=4s</b></i>
-     * </pre>
-     * But as you can imagine, the time parameter is very important when calling {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
-     */
-    private final String originalUrl;
 
-    /**
-     * The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}).
-     * <p>
-     * Is lazily-cleaned by calling {@link #getCleanUrl()}
-     */
     @Nullable
-    private String cleanUrl;
     private boolean pageFetched = false;
     private final Downloader downloader;
 
-    public Extractor(final StreamingService service, final String url) {
+    public Extractor(final StreamingService service, final UrlIdHandler urlIdHandler) {
         if(service == null) throw new NullPointerException("service is null");
-        if(url == null) throw new NullPointerException("url is null");
+        if(urlIdHandler == null) throw new NullPointerException("UrlIdHandler is null");
         this.service = service;
-        this.originalUrl = url;
+        this.urlIdHandler = urlIdHandler;
         this.downloader = NewPipe.getDownloader();
         if(downloader == null) throw new NullPointerException("downloader is null");
     }
 
     /**
-     * @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler).
+     * @return The {@link UrlIdHandler} of the current extractor object (e.g. a ChannelExtractor should return a channel url handler).
      */
     @Nonnull
-    protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException;
+    protected  UrlIdHandler getUrlIdHandler() {
+        return urlIdHandler;
+    }
 
     /**
      * Fetch the current page.
@@ -79,7 +66,9 @@ public abstract class Extractor {
     public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
 
     @Nonnull
-    public abstract String getId() throws ParsingException;
+    public String getId() throws ParsingException {
+        return urlIdHandler.getId();
+    }
 
     /**
      * Get the name
@@ -90,26 +79,13 @@ public abstract class Extractor {
     public abstract String getName() throws ParsingException;
 
     @Nonnull
-    public String getOriginalUrl() {
+    public String getOriginalUrl() throws ParsingException {
-        return originalUrl;
+        return urlIdHandler.getOriginalUrl();
     }
 
-    /**
-     * Get a clean url and as a fallback the original url.
-     * @return the clean url or the original url
-     */
     @Nonnull
-    public String getCleanUrl() {
+    public String getUrl() throws ParsingException {
-        if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;
+        return urlIdHandler.getUrl();
-
-        try {
-            cleanUrl = getUrlIdHandler().cleanUrl(originalUrl);
-        } catch (Exception e) {
-            cleanUrl = null;
-            // Fallback to the original url
-            return originalUrl;
-        }
-        return cleanUrl;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java

@@ -12,8 +12,8 @@ import java.util.List;
  */
 public abstract class ListExtractor<R extends InfoItem> extends Extractor {
 
-    public ListExtractor(StreamingService service, String url) {
+    public ListExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java

@@ -0,0 +1,42 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+public abstract class ListUrlIdHandler extends UrlIdHandler {
+
+    protected String[] contentFilter;
+    protected String sortFilter;
+
+    public ListUrlIdHandler setQuery(String id, String[] contentFilter, String softFilter) throws ParsingException {
+        setId(id);
+        this.contentFilter = contentFilter;
+        this.sortFilter = softFilter;
+        return this;
+    }
+
+    public ListUrlIdHandler setUrl(String url) throws ParsingException {
+        return (ListUrlIdHandler) super.setUrl(url);
+    }
+
+    public ListUrlIdHandler setId(String id) throws ParsingException {
+        return (ListUrlIdHandler) super.setId(id);
+    }
+
+    /**
+     * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc.
+     *
+     * @return filter that can be applied when building a query for getting a list
+     */
+    public String[] getAvailableContentFilter() {
+        return new String[0];
+    }
+
+    /**
+     * Will returns sort filter the corresponding extractor can handle like "A-Z", "oldest first", "size", etc.
+     *
+     * @return filter that can be applied when building a query for getting a list
+     */
+    public String[] getAvailableSortFilter() {
+        return new String[0];
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor;
 
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchEngine;
@@ -62,22 +63,52 @@ public abstract class StreamingService {
         return serviceId + ":" + serviceInfo.getName();
     }
 
+    ////////////////////////////////////////////
+    // Url Id handler
+    ////////////////////////////////////////////
     public abstract UrlIdHandler getStreamUrlIdHandler();
-    public abstract UrlIdHandler getChannelUrlIdHandler();
+    public abstract ListUrlIdHandler getChannelUrlIdHandler();
-    public abstract UrlIdHandler getPlaylistUrlIdHandler();
+    public abstract ListUrlIdHandler getPlaylistUrlIdHandler();
 
+
+    ////////////////////////////////////////////
+    // Extractor
+    ////////////////////////////////////////////
     public abstract SearchEngine getSearchEngine();
     public abstract SuggestionExtractor getSuggestionExtractor();
-    public abstract StreamExtractor getStreamExtractor(String url);
-    public abstract KioskList getKioskList() throws ExtractionException;
-    public abstract ChannelExtractor getChannelExtractor(String url);
-    public abstract PlaylistExtractor getPlaylistExtractor(String url);
     public abstract SubscriptionExtractor getSubscriptionExtractor();
+    public abstract KioskList getKioskList() throws ExtractionException;
+
+    public abstract ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException;
+    public abstract PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException;
+    public abstract StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException;
+
+    public ChannelExtractor getChannelExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException {
+        return getChannelExtractor(getChannelUrlIdHandler().setQuery(id, contentFilter, sortFilter));
+    }
+
+    public PlaylistExtractor getPlaylistExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException {
+        return getPlaylistExtractor(getPlaylistUrlIdHandler().setQuery(id, contentFilter, sortFilter));
+    }
+
+    public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
+        return getChannelExtractor(getChannelUrlIdHandler().setUrl(url));
+    }
+
+    public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
+        return getPlaylistExtractor(getPlaylistUrlIdHandler().setUrl(url));
+    }
+
+    public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
+        return getStreamExtractor(getStreamUrlIdHandler().setUrl(url));
+    }
+
+
 
     /**
      * figure out where the link is pointing to (a channel, video, playlist, etc.)
      */
-    public final LinkType getLinkTypeByUrl(String url) {
+    public final LinkType getLinkTypeByUrl(String url) throws ParsingException {
         UrlIdHandler sH = getStreamUrlIdHandler();
         UrlIdHandler cH = getChannelUrlIdHandler();
         UrlIdHandler pH = getPlaylistUrlIdHandler();

extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor;
 
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 /*
@@ -22,16 +23,52 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-public interface UrlIdHandler {
+public abstract class UrlIdHandler {
 
-    String getUrl(String id) throws ParsingException;
+    protected String id = "";
-    String getId(String url) throws ParsingException;
+    protected String originalUrl = "";
-    String cleanUrl(String complexUrl) throws ParsingException;
+
+    public abstract String onGetIdFromUrl(String url) throws ParsingException;
+    public abstract String getUrl() throws ParsingException;
+    public abstract boolean onAcceptUrl(final String url) throws ParsingException;
+
+
+    public UrlIdHandler setUrl(String url) throws ParsingException {
+        if(url == null) throw new IllegalArgumentException("url can not be null");
+        originalUrl = url;
+        id = onGetIdFromUrl(url);
+        return this;
+    }
+
+    public UrlIdHandler setId(String id) throws ParsingException {
+        if(id == null) throw new IllegalArgumentException("id can not be null");
+        this.id = id;
+        if(!acceptUrl(getUrl())) {
+            throw new ParsingException("Malformed unacceptable url: " + getUrl());
+        }
+        return this;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getOriginalUrl() throws ParsingException {
+        return (originalUrl == null || originalUrl.isEmpty())
+                ? getUrl()
+                : originalUrl;
+    }
 
     /**
      * When a VIEW_ACTION is caught this function will test if the url delivered within the calling
      * Intent was meant to be watched with this Service.
      * Return false if this service shall not allow to be called through ACTIONs.
      */
-    boolean acceptUrl(String url);
+    public boolean acceptUrl(final String url) {
+        try {
+            return onAcceptUrl(url);
+        } catch (Exception e) {
+            return false;
+        }
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java

@@ -1,8 +1,10 @@
 package org.schabi.newpipe.extractor.channel;
 
 import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
@@ -30,8 +32,8 @@ import javax.annotation.Nonnull;
 
 public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
 
-    public ChannelExtractor(StreamingService service, String url) {
+    public ChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java

@@ -54,7 +54,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
 
         // important data
         int serviceId = extractor.getServiceId();
-        String url = extractor.getCleanUrl();
+        String url = extractor.getUrl();
         String originalUrl = extractor.getOriginalUrl();
         String id = extractor.getId();
         String name = extractor.getName();

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java

@@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.kiosk;
  */
 
 import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -33,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
     private final String id;
 
     public KioskExtractor(StreamingService streamingService,
-                          String url,
+                          ListUrlIdHandler urlIdHandler,
-                          String kioskId)
+                          String kioskId) {
-        throws ExtractionException {
+        super(streamingService, urlIdHandler);
-        super(streamingService, url);
         this.id = kioskId;
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java

@@ -71,7 +71,7 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
         int serviceId = extractor.getServiceId();
         String name = extractor.getName();
         String id = extractor.getId();
-        String url = extractor.getCleanUrl();
+        String url = extractor.getUrl();
         String originalUrl = extractor.getOriginalUrl();
 
         KioskInfo info = new KioskInfo(serviceId, id, url, originalUrl, name);

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java

@@ -73,7 +73,7 @@ public  class KioskList {
             throw new ExtractionException("No kiosk found with the type: " + kioskId);
         } else {
             return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
-                    ke.handler.getUrl(kioskId), kioskId);
+                    ke.handler.setId(kioskId).getUrl(), kioskId);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java

@@ -1,8 +1,10 @@
 package org.schabi.newpipe.extractor.playlist;
 
 import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
@@ -10,8 +12,8 @@ import javax.annotation.Nonnull;
 
 public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
 
-    public PlaylistExtractor(StreamingService service, String url) {
+    public PlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java

@@ -38,7 +38,7 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
     public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException {
 
         int serviceId = extractor.getServiceId();
-        String url = extractor.getCleanUrl();
+        String url = extractor.getUrl();
         String originalUrl = extractor.getOriginalUrl();
         String id = extractor.getId();
         String name = extractor.getName();

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -25,14 +26,14 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
     private StreamInfoItemsCollector streamInfoItemsCollector = null;
     private String nextPageUrl = null;
 
-    public SoundcloudChannelExtractor(StreamingService service, String url) {
+    public SoundcloudChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
 
-        userId = getUrlIdHandler().getId(getOriginalUrl());
+        userId = getUrlIdHandler().getId();
         String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
                 "?client_id=" + SoundcloudParsingHelper.clientId();
 
@@ -44,12 +45,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
         }
     }
 
-    @Nonnull
-    @Override
-    public String getCleanUrl() {
-        return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl();
-    }
-
     @Nonnull
     @Override
     public String getId() {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -10,7 +11,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
 
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
-public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
+public class SoundcloudChannelUrlIdHandler extends ListUrlIdHandler {
     private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
     private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
             "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
@@ -19,17 +20,9 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
         return instance;
     }
 
-    @Override
-    public String getUrl(String id) throws ParsingException {
-        try {
-            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id);
-        } catch (Exception e) {
-            throw new ParsingException(e.getMessage(), e);
-        }
-    }
 
     @Override
-    public String getId(String url) throws ParsingException {
+    public String onGetIdFromUrl(String url) throws ParsingException {
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
@@ -40,21 +33,16 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String cleanUrl(String complexUrl) throws ParsingException {
+    public String getUrl() throws ParsingException {
-        Utils.checkUrl(URL_PATTERN, complexUrl);
-
         try {
-            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
+            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id);
-                    .select("meta[property=og:url]").first();
-
-            return replaceHttpWithHttps(ogElement.attr("content"));
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
     }
 
     @Override
-    public boolean acceptUrl(String url) {
+    public boolean onAcceptUrl(final String url) {
         return Parser.isMatch(URL_PATTERN, url.toLowerCase());
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
 import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -14,15 +15,11 @@ import java.util.Arrays;
 import java.util.List;
 
 public class SoundcloudChartsExtractor extends KioskExtractor {
-	private String url;
-
 	private StreamInfoItemsCollector collector = null;
 	private String nextPageUrl = null;
 
-    public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId)
+    public SoundcloudChartsExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId) {
-            throws ExtractionException {
+        super(service, urlIdHandler, kioskId);
-        super(service, url, kioskId);
-        this.url = url;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java

@@ -1,13 +1,25 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 
-public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
+public class SoundcloudChartsUrlIdHandler extends ListUrlIdHandler {
     private final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
     private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
 
-    public String getUrl(String id) {
+
+    @Override
+    public String onGetIdFromUrl(String url) {
+        if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
+            return "Top 50";
+        } else {
+            return "New & hot";
+        }
+    }
+
+    public String getUrl() {
         if (id.equals("Top 50")) {
             return "https://soundcloud.com/charts/top";
         } else {
@@ -16,25 +28,7 @@ public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getId(String url) {
+    public boolean onAcceptUrl(final String url) {
-        if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
-            return "Top 50";
-        } else {
-            return "New & hot";
-        }
-    }
-
-    @Override
-    public String cleanUrl(String url) {
-        if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
-            return "https://soundcloud.com/charts/top";
-        } else {
-            return "https://soundcloud.com/charts/new";
-        }
-    }
-
-    @Override
-    public boolean acceptUrl(String url) {
         return Parser.isMatch(URL_PATTERN, url.toLowerCase());
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -24,14 +25,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     private StreamInfoItemsCollector streamInfoItemsCollector = null;
     private String nextPageUrl = null;
 
-    public SoundcloudPlaylistExtractor(StreamingService service, String url) {
+    public SoundcloudPlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
 
-        playlistId = getUrlIdHandler().getId(getOriginalUrl());
+        playlistId = getUrlIdHandler().getId();
         String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
                 "?client_id=" + SoundcloudParsingHelper.clientId() +
                 "&representation=compact";
@@ -44,12 +45,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
         }
     }
 
-    @Nonnull
-    @Override
-    public String getCleanUrl() {
-        return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl();
-    }
-
     @Nonnull
     @Override
     public String getId() {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java

@@ -1,16 +1,11 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.jsoup.Jsoup;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
-import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+public class SoundcloudPlaylistUrlIdHandler extends ListUrlIdHandler {
-
-public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
     private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
     private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
             "/sets/[0-9a-z_-]+/?([#?].*)?$";
@@ -20,7 +15,18 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getUrl(String id) throws ParsingException {
+    public String onGetIdFromUrl(String url) throws ParsingException {
+        Utils.checkUrl(URL_PATTERN, url);
+
+        try {
+            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+        } catch (Exception e) {
+            throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
         try {
             return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id);
         } catch (Exception e) {
@@ -29,32 +35,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getId(String url) throws ParsingException {
+    public boolean onAcceptUrl(final String url) throws ParsingException {
-        Utils.checkUrl(URL_PATTERN, url);
-
-        try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
-        } catch (Exception e) {
-            throw new ParsingException(e.getMessage(), e);
-        }
-    }
-
-    @Override
-    public String cleanUrl(String complexUrl) throws ParsingException {
-        Utils.checkUrl(URL_PATTERN, complexUrl);
-
-        try {
-            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
-                    .select("meta[property=og:url]").first();
-
-            return replaceHttpWithHttps(ogElement.attr("content"));
-        } catch (Exception e) {
-            throw new ParsingException(e.getMessage(), e);
-        }
-    }
-
-    @Override
-    public boolean acceptUrl(String url) {
         return Parser.isMatch(URL_PATTERN, url.toLowerCase());
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.SuggestionExtractor;
 import org.schabi.newpipe.extractor.UrlIdHandler;
@@ -32,29 +33,29 @@ public class SoundcloudService extends StreamingService {
     }
 
     @Override
-    public UrlIdHandler getChannelUrlIdHandler() {
+    public ListUrlIdHandler getChannelUrlIdHandler() {
         return SoundcloudChannelUrlIdHandler.getInstance();
     }
 
     @Override
-    public UrlIdHandler getPlaylistUrlIdHandler() {
+    public ListUrlIdHandler getPlaylistUrlIdHandler() {
         return SoundcloudPlaylistUrlIdHandler.getInstance();
     }
 
 
     @Override
-    public StreamExtractor getStreamExtractor(String url) {
+    public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException {
-        return new SoundcloudStreamExtractor(this, url);
+        return new SoundcloudStreamExtractor(this, urlIdHandler);
     }
 
     @Override
-    public ChannelExtractor getChannelExtractor(String url) {
+    public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
-        return new SoundcloudChannelExtractor(this, url);
+        return new SoundcloudChannelExtractor(this, urlIdHandler);
     }
 
     @Override
-    public PlaylistExtractor getPlaylistExtractor(String url) {
+    public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
-        return new SoundcloudPlaylistExtractor(this, url);
+        return new SoundcloudPlaylistExtractor(this, urlIdHandler);
     }
 
     @Override
@@ -71,8 +72,7 @@ public class SoundcloudService extends StreamingService {
                                                  String id)
                     throws ExtractionException {
                 return new SoundcloudChartsExtractor(SoundcloudService.this,
-                        url,
+                        new SoundcloudChartsUrlIdHandler().setUrl(url), id);
-                        id);
             }
         };
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -22,8 +22,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
 
-    public SoundcloudStreamExtractor(StreamingService service, String url) {
+    public SoundcloudStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Override
@@ -36,12 +36,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         }
     }
 
-    @Nonnull
-    @Override
-    public String getCleanUrl() {
-        return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl();
-    }
-
     @Nonnull
     @Override
     public String getId() {
@@ -214,6 +208,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         return new String[0];
     }
 
+    @Override
+    public String[] getAffiliateLinks() {
+        return new String[0];
+    }
+
     @Override
     public String getErrorMessage() {
         return null;

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java

@@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
 
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
-public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
+public class SoundcloudStreamUrlIdHandler extends UrlIdHandler {
     private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
     private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
             "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
@@ -23,7 +23,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getUrl(String id) throws ParsingException {
+    public String getUrl() throws ParsingException {
         try {
             return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id);
         } catch (Exception e) {
@@ -32,7 +32,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getId(String url) throws ParsingException {
+    public String onGetIdFromUrl(String url) throws ParsingException {
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
@@ -43,21 +43,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String cleanUrl(String complexUrl) throws ParsingException {
+    public boolean onAcceptUrl(final String url) throws ParsingException {
-        Utils.checkUrl(URL_PATTERN, complexUrl);
-
-        try {
-            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
-                    .select("meta[property=og:url]").first();
-
-            return replaceHttpWithHttps(ogElement.attr("content"));
-        } catch (Exception e) {
-            throw new ParsingException(e.getMessage(), e);
-        }
-    }
-
-    @Override
-    public boolean acceptUrl(String url) {
         return Parser.isMatch(URL_PATTERN, url.toLowerCase());
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java

@@ -31,7 +31,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
 
         String id;
         try {
-            id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl));
+            id = service.getChannelUrlIdHandler().setUrl(getUrlFrom(channelUrl)).getId();
         } catch (ExtractionException e) {
             throw new InvalidSourceException(e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java

@@ -43,6 +43,9 @@ public class ItagItem {
             new ItagItem(139, AUDIO, M4A, 48),
             new ItagItem(140, AUDIO, M4A, 128),
             new ItagItem(141, AUDIO, M4A, 256),
+            new ItagItem(249, AUDIO, OPUS, 50),
+            new ItagItem(250, AUDIO, OPUS, 70),
+            new ItagItem(160, AUDIO, OPUS, 160),
 
             /// VIDEO ONLY ////////////////////////////////////////////
             //           ID      Type     Format  Resolution  FPS  ///

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.SuggestionExtractor;
 import org.schabi.newpipe.extractor.UrlIdHandler;
@@ -9,6 +10,11 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.services.youtube.extractors.*;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubePlaylistUrlIdHandler;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 
@@ -53,28 +59,28 @@ public class YoutubeService extends StreamingService {
     }
 
     @Override
-    public UrlIdHandler getChannelUrlIdHandler() {
+    public ListUrlIdHandler getChannelUrlIdHandler() {
         return YoutubeChannelUrlIdHandler.getInstance();
     }
 
     @Override
-    public UrlIdHandler getPlaylistUrlIdHandler() {
+    public ListUrlIdHandler getPlaylistUrlIdHandler() {
         return YoutubePlaylistUrlIdHandler.getInstance();
     }
 
     @Override
-    public StreamExtractor getStreamExtractor(String url) {
+    public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException {
-        return new YoutubeStreamExtractor(this, url);
+        return new YoutubeStreamExtractor(this, urlIdHandler);
     }
 
     @Override
-    public ChannelExtractor getChannelExtractor(String url) {
+    public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
-        return new YoutubeChannelExtractor(this, url);
+        return new YoutubeChannelExtractor(this, urlIdHandler);
     }
 
     @Override
-    public PlaylistExtractor getPlaylistExtractor(String url) {
+    public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
-        return new YoutubePlaylistExtractor(this, url);
+        return new YoutubePlaylistExtractor(this, urlIdHandler);
     }
 
     @Override
@@ -92,7 +98,8 @@ public class YoutubeService extends StreamingService {
                 @Override
                 public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id)
                 throws ExtractionException {
-                    return new YoutubeTrendingExtractor(YoutubeService.this, url, id);
+                    return new YoutubeTrendingExtractor(YoutubeService.this,
+                            new YoutubeTrendingUrlIdHandler().setUrl(url), id);
                 }
             }, new YoutubeTrendingUrlIdHandler(), "Trending");
             list.setDefaultKiosk("Trending");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -1,17 +1,13 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import com.sun.org.apache.xerces.internal.xs.StringList;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.*;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -52,16 +48,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
     private Document doc;
 
-    public YoutubeChannelExtractor(StreamingService service, String url) {
+    public YoutubeChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
+        String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
         String pageContent = downloader.download(channelUrl);
         doc = Jsoup.parse(pageContent, channelUrl);
-
     }
 
     @Override
@@ -71,11 +66,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
     @Nonnull
     @Override
-    public String getCleanUrl() {
+    public String getUrl() throws ParsingException {
         try {
             return "https://www.youtube.com/channel/" + getId();
         } catch (ParsingException e) {
-            return super.getCleanUrl();
+            return super.getUrl();
         }
     }
 
@@ -196,7 +191,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
             }
             for(Element a : linkHolder.select("a")) {
                 String link = a.attr("abs:href");
-                if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) {
+                if(DonationLinkHelper.getDonatoinServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) {
                     links.add(link);
                 }
             }
@@ -236,7 +231,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         collector.reset();
 
         final String uploaderName = getName();
-        final String uploaderUrl = getCleanUrl();
+        final String uploaderUrl = getUrl();
         for (final Element li : element.children()) {
             if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
                 collector.commit(new YoutubeStreamInfoItemExtractor(li) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
@@ -6,13 +6,11 @@ import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.*;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
@@ -26,14 +24,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
     private Document doc;
 
-    public YoutubePlaylistExtractor(StreamingService service, String url) {
+    public YoutubePlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) throws ExtractionException {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        String pageContent = downloader.download(getCleanUrl());
+        String pageContent = downloader.download(getUrl());
-        doc = Jsoup.parse(pageContent, getCleanUrl());
+        doc = Jsoup.parse(pageContent, getUrl());
     }
 
     @Override
@@ -41,16 +39,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         return getNextPageUrlFrom(doc);
     }
 
-    @Nonnull
-    @Override
-    public String getId() throws ParsingException {
-        try {
-            return getUrlIdHandler().getId(getCleanUrl());
-        } catch (Exception e) {
-            throw new ParsingException("Could not get playlist id");
-        }
-    }
-
     @Nonnull
     @Override
     public String getName() throws ParsingException {
@@ -202,7 +190,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                 @Override
                 public String getUrl() throws ParsingException {
                     try {
-                        return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
+                        return streamUrlIdHandler.setId(li.attr("data-video-id")).getUrl();
                     } catch (Exception e) {
                         throw new ParsingException("Could not get web page url for the video", e);
                     }
@@ -267,7 +255,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                 @Override
                 public String getThumbnailUrl() throws ParsingException {
                     try {
-                        return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getUrl()) + "/hqdefault.jpg";
+                        return "https://i.ytimg.com/vi/" + streamUrlIdHandler.setUrl(getUrl()).getId() + "/hqdefault.jpg";
                     } catch (Exception e) {
                         throw new ParsingException("Could not get thumbnail url", e);
                     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
@@ -10,14 +10,12 @@ import org.jsoup.nodes.Element;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
-import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.*;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.Subtitles;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 import org.schabi.newpipe.extractor.stream.*;
 import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -86,24 +84,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     private boolean isAgeRestricted;
 
-    public YoutubeStreamExtractor(StreamingService service, String url) {
+    public YoutubeStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) throws ExtractionException {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     /*//////////////////////////////////////////////////////////////////////////
     // Impl
     //////////////////////////////////////////////////////////////////////////*/
 
-    @Nonnull
-    @Override
-    public String getId() throws ParsingException {
-        try {
-            return getUrlIdHandler().getId(getCleanUrl());
-        } catch (Exception e) {
-            throw new ParsingException("Could not get stream id");
-        }
-    }
-
     @Nonnull
     @Override
     public String getName() throws ParsingException {
@@ -534,7 +522,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         try {
             ArrayList<String> donationLinks = new ArrayList<>();
             for (String s : Parser.getLinksFromString(getDescription())) {
-                if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) {
+                if (DonationLinkHelper.getDonatoinServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) {
                     donationLinks.add(s);
                 }
             }
@@ -546,6 +534,23 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    @Override
+    public String[] getAffiliateLinks() throws ParsingException {
+        try {
+            ArrayList<String> donationLinks = new ArrayList<>();
+            for (String s : Parser.getLinksFromString(getDescription())) {
+                if (DonationLinkHelper.getAffiliateServiceByLink(s) != DonationLinkHelper.AffiliateService.NO_AFILIATE) {
+                    donationLinks.add(s);
+                }
+            }
+            String[] donlret = new String[donationLinks.size()];
+            donlret = donationLinks.toArray(donlret);
+            return donlret;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get afiliate links", e);
+        }
+    }
+
 
     /*//////////////////////////////////////////////////////////////////////////
     // Fetch page
@@ -564,7 +569,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private String pageHtml = null;
 
     private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
-        final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS;
+        final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
         if (pageHtml == null) {
             pageHtml = downloader.download(verifiedUrl);
         }
@@ -574,7 +579,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         final String pageContent = getPageHtml(downloader);
-        doc = Jsoup.parse(pageContent, getCleanUrl());
+        doc = Jsoup.parse(pageContent, getUrl());
 
         final String playerUrl;
         // Check if the video is age restricted

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -1,7 +1,8 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -135,7 +136,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
             Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
             if (meta == null) return -1;
 
+            // This case can happen if google releases a special video
+            if(meta.select("li").size() < 2)  return -1;
+
             input = meta.select("li").get(1).text();
+
         } catch (IndexOutOfBoundsException e) {
             throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java

@@ -1,9 +1,10 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
 import org.schabi.newpipe.extractor.utils.Parser;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonParser;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 /*
  * Created by Christian Schabesberger on 12.08.17.
@@ -25,11 +25,13 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
@@ -40,15 +42,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
 
     private Document doc;
 
-    public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId)
+    public YoutubeTrendingExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId)
             throws ExtractionException {
-        super(service, url, kioskId);
+        super(service, urlIdHandler, kioskId);
     }
 
     @Override
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         final String contentCountry = getContentCountry();
-        String url = getCleanUrl();
+        String url = getUrl();
         if(contentCountry != null && !contentCountry.isEmpty()) {
             url += "?gl=" + contentCountry;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java

@@ -1,6 +1,6 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
 
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 
@@ -24,7 +24,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
+public class YoutubeChannelUrlIdHandler extends ListUrlIdHandler {
 
     private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler();
     private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)";
@@ -34,22 +34,17 @@ public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getUrl(String id) {
+    public String onGetIdFromUrl(String url) throws ParsingException {
-        return "https://www.youtube.com/" + id;
-    }
-
-    @Override
-    public String getId(String url) throws ParsingException {
         return Parser.matchGroup1(ID_PATTERN, url);
     }
 
     @Override
-    public String cleanUrl(String complexUrl) throws ParsingException {
+    public String getUrl() {
-        return getUrl(getId(complexUrl));
+        return "https://www.youtube.com/" + id;
     }
 
     @Override
-    public boolean acceptUrl(String url) {
+    public boolean onAcceptUrl(String url) {
         return (url.contains("youtube") || url.contains("youtu.be") || url.contains("hooktube.com"))
                 && (url.contains("/user/") || url.contains("/channel/"));
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
 
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java

@@ -1,11 +1,11 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
 
 
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 
-public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
+public class YoutubePlaylistUrlIdHandler extends ListUrlIdHandler {
 
     private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
     private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})";
@@ -15,12 +15,12 @@ public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getUrl(String id) {
+    public String getUrl() {
         return "https://www.youtube.com/playlist?list=" + id;
     }
 
     @Override
-    public String getId(String url) throws ParsingException {
+    public String onGetIdFromUrl(String url) throws ParsingException {
         try {
             return Parser.matchGroup1("list=" + ID_PATTERN, url);
         } catch (final Exception exception) {
@@ -28,13 +28,9 @@ public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
         }
     }
 
-    @Override
-    public String cleanUrl(String complexUrl) throws ParsingException {
-        return getUrl(getId(complexUrl));
-    }
 
     @Override
-    public boolean acceptUrl(String url) {
+    public boolean onAcceptUrl(final String url) {
         final boolean hasNotEmptyUrl = url != null && !url.isEmpty();
         final boolean isYoutubeDomain = hasNotEmptyUrl && (url.contains("youtube") || url.contains("youtu.be"));
         return isYoutubeDomain && url.contains("list=");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -37,7 +37,7 @@ import java.net.URLDecoder;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
+public class YoutubeStreamUrlIdHandler extends UrlIdHandler {
 
     private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler();
     private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})";
@@ -50,12 +50,12 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String getUrl(String id) {
+    public String getUrl() {
         return "https://www.youtube.com/watch?v=" + id;
     }
 
     @Override
-    public String getId(String url) throws ParsingException, IllegalArgumentException {
+    public String onGetIdFromUrl(String url) throws ParsingException, IllegalArgumentException {
         if (url.isEmpty()) {
             throw new IllegalArgumentException("The url parameter should not be empty");
         }
@@ -167,19 +167,14 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
     }
 
     @Override
-    public String cleanUrl(String complexUrl) throws ParsingException {
+    public boolean onAcceptUrl(final String url) {
-        return getUrl(getId(complexUrl));
+        final String lowercaseUrl = url.toLowerCase();
-    }
-
-    @Override
-    public boolean acceptUrl(String url) {
-        String lowercaseUrl = url.toLowerCase();
         if (lowercaseUrl.contains("youtube")
                 || lowercaseUrl.contains("youtu.be")
                 || lowercaseUrl.contains("hooktube")) {
             // bad programming I know
             try {
-                getId(url);
+                onGetIdFromUrl(url);
                 return true;
             } catch (Exception e) {
                 return false;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java

@@ -1,4 +1,4 @@
-package org.schabi.newpipe.extractor.services.youtube;
+package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
 
 /*
  * Created by Christian Schabesberger on 12.08.17.
@@ -20,27 +20,24 @@ package org.schabi.newpipe.extractor.services.youtube;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+import org.schabi.newpipe.extractor.ListUrlIdHandler;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 
-public class YoutubeTrendingUrlIdHandler implements UrlIdHandler {
+public class YoutubeTrendingUrlIdHandler extends ListUrlIdHandler {
 
-    public String getUrl(String id) {
+    public String getUrl() {
         return "https://www.youtube.com/feed/trending";
     }
 
     @Override
-    public String getId(String url) {
+    public String onGetIdFromUrl(String url) {
         return "Trending";
     }
 
     @Override
-    public String cleanUrl(String url) {
+    public boolean onAcceptUrl(final String url) {
-        return getUrl("");
-    }
-
-    @Override
-    public boolean acceptUrl(String url) {
         return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -39,8 +39,8 @@ public abstract class StreamExtractor extends Extractor {
 
     public static final int NO_AGE_LIMIT = 0;
 
-    public StreamExtractor(StreamingService service, String url) {
+    public StreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) {
-        super(service, url);
+        super(service, urlIdHandler);
     }
 
     @Nonnull
@@ -142,6 +142,7 @@ public abstract class StreamExtractor extends Extractor {
     public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
 
     public abstract String[] getDonationLinks() throws ExtractionException;
+    public abstract String[] getAffiliateLinks() throws ExtractionException;
 
     /**
      * Analyses the webpage's document and extracts any error message there might be.

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -85,7 +85,7 @@ public class StreamInfo extends Info {
         // if one of these is not available an exception is meant to be thrown directly into the frontend.
 
         int serviceId = extractor.getServiceId();
-        String url = extractor.getCleanUrl();
+        String url = extractor.getUrl();
         String originalUrl = extractor.getOriginalUrl();
         StreamType streamType = extractor.getStreamType();
         String id = extractor.getId();
@@ -242,6 +242,16 @@ public class StreamInfo extends Info {
         } catch (Exception e) {
             streamInfo.addError(e);
         }
+        try {
+            streamInfo.setAffiliateLinks(extractor.getAffiliateLinks());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setDonationLinks(extractor.getDonationLinks());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
 
         streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
         return streamInfo;
@@ -274,6 +284,9 @@ public class StreamInfo extends Info {
     private long startPosition = 0;
     private List<Subtitles> subtitles;
 
+    private String[] donationLinks;
+    private String[] affiliateLinks;
+
     /**
      * Get the stream type
      *
@@ -466,4 +479,20 @@ public class StreamInfo extends Info {
     public void setSubtitles(List<Subtitles> subtitles) {
         this.subtitles = subtitles;
     }
+
+    public String[] getDonationLinks() {
+        return donationLinks;
+    }
+
+    public void setDonationLinks(String[] donationLinks) {
+        this.donationLinks = donationLinks;
+    }
+
+    public String[] getAffiliateLinks() {
+        return affiliateLinks;
+    }
+
+    public void setAffiliateLinks(String[] affiliateLinks) {
+        this.affiliateLinks = affiliateLinks;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java

@@ -7,12 +7,16 @@ public class DonationLinkHelper {
     public enum DonationService {
         NO_DONATION,
         PATREON,
-        PAYPAL
+        PAYPAL,
     }
 
+    public enum AffiliateService {
+        NO_AFILIATE,
+        AMAZON,
+    }
 
-    public static DonationService getServiceByLink(String link) throws MalformedURLException {
+    public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException {
-        URL url = new URL(link);
+        URL url = new URL(fixLink(link));
         switch (url.getHost()) {
             case "www.patreon.com":
                 return DonationService.PATREON;
@@ -27,5 +31,18 @@ public class DonationLinkHelper {
         }
     }
 
+    public static AffiliateService getAffiliateServiceByLink(String link) throws MalformedURLException {
+        URL url = new URL(fixLink(link));
+        switch (url.getHost()) {
+            case "amzn.to": return AffiliateService.AMAZON;
+            default: return AffiliateService.NO_AFILIATE;
+        }
+    }
+
+    private static String fixLink(String link) {
+        return (link.startsWith("https://") || link.startsWith("http://"))
+                ? link
+                : "https://" + link;
+    }
 
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java

@@ -5,6 +5,6 @@ public interface BaseExtractorTest {
     void testServiceId() throws Exception;
     void testName() throws Exception;
     void testId() throws Exception;
-    void testCleanUrl() throws Exception;
+    void testUrl() throws Exception;
     void testOriginalUrl() throws Exception;
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java

@@ -4,7 +4,6 @@ package org.schabi.newpipe.extractor.services;
 public interface BasePlaylistExtractorTest extends BaseListExtractorTest {
     void testThumbnailUrl() throws Exception;
     void testBannerUrl() throws Exception;
-    void testUploaderUrl() throws Exception;
     void testUploaderName() throws Exception;
     void testUploaderAvatarUrl() throws Exception;
     void testStreamCount() throws Exception;

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java

@@ -5,6 +5,7 @@ import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
 
 import static org.junit.Assert.*;
@@ -48,12 +49,12 @@ public class SoundcloudChannelExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://soundcloud.com/liluzivert", extractor.getCleanUrl());
+            assertEquals("https://soundcloud.com/liluzivert", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl());
         }
 
@@ -118,7 +119,7 @@ public class SoundcloudChannelExtractorTest {
 
         @Test
         public void testGetPageInNewExtractor() throws Exception {
-            final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getCleanUrl());
+            final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl());
             defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
         }
 
@@ -142,12 +143,12 @@ public class SoundcloudChannelExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://soundcloud.com/dubmatix", extractor.getCleanUrl());
+            assertEquals("https://soundcloud.com/dubmatix", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl());
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -88,6 +88,6 @@ public class SoundcloudChartsExtractorTest {
 
     @Test
     public void testGetCleanUrl() throws Exception {
-        assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top");
+        assertEquals(extractor.getUrl(), "https://soundcloud.com/charts/top");
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java

@@ -4,6 +4,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
@@ -22,19 +23,19 @@ public class SoundcloudChartsUrlIdHandlerTest {
     }
 
     @Test
-    public void getUrl() {
+    public void getUrl() throws Exception {
-        assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top");
+        assertEquals(urlIdHandler.setId("Top 50").getUrl(), "https://soundcloud.com/charts/top");
-        assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new");
+        assertEquals(urlIdHandler.setId("New & hot").getUrl(), "https://soundcloud.com/charts/new");
     }
 
     @Test
-    public void getId() {
+    public void getId() throws ParsingException {
-        assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50");
+        assertEquals(urlIdHandler.setUrl("http://soundcloud.com/charts/top?genre=all-music").getId(), "Top 50");
-        assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot");
+        assertEquals(urlIdHandler.setUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries").getId(), "New & hot");
     }
 
     @Test
-    public void acceptUrl() {
+    public void acceptUrl() throws ParsingException {
         assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts"));
         assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/"));
         assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new"));

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -51,12 +51,12 @@ public class SoundcloudPlaylistExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws Exception {
-            assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getCleanUrl());
+            assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws Exception {
             assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r?test=123", extractor.getOriginalUrl());
         }
 
@@ -125,7 +125,7 @@ public class SoundcloudPlaylistExtractorTest {
         public static void setUp() throws Exception {
             NewPipe.init(Downloader.getInstance());
             extractor = (SoundcloudPlaylistExtractor) SoundCloud
-                    .getPlaylistExtractor("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2");
+                    .getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2");
             extractor.fetchPage();
         }
 
@@ -140,22 +140,22 @@ public class SoundcloudPlaylistExtractorTest {
 
         @Test
         public void testName() {
-            assertEquals("Random House & Dance Music #2", extractor.getName());
+            assertEquals("House, Electro , Dance Music 2", extractor.getName());
         }
 
         @Test
         public void testId() {
-            assertEquals("436855608", extractor.getId());
+            assertEquals("310980722", extractor.getId());
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws Exception {
-            assertEquals("https://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getCleanUrl());
+            assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws Exception {
-            assertEquals("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getOriginalUrl());
+            assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl());
         }
 
         /*//////////////////////////////////////////////////////////////////////////
@@ -191,12 +191,12 @@ public class SoundcloudPlaylistExtractorTest {
         public void testUploaderUrl() {
             final String uploaderUrl = extractor.getUploaderUrl();
             assertIsSecureUrl(uploaderUrl);
-            assertTrue(uploaderUrl, uploaderUrl.contains("finn-trapple"));
+            assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader"));
         }
 
         @Test
         public void testUploaderName() {
-            assertEquals("Finn TrApple", extractor.getUploaderName());
+            assertEquals("Gosu", extractor.getUploaderName());
         }
 
         @Test
@@ -227,7 +227,7 @@ public class SoundcloudPlaylistExtractorTest {
 
         @Test
         public void testGetPageInNewExtractor() throws Exception {
-            final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getCleanUrl());
+            final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
             defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
         }
 
@@ -251,12 +251,12 @@ public class SoundcloudPlaylistExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws Exception {
-            assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getCleanUrl());
+            assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws Exception {
             assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getOriginalUrl());
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java

@@ -25,7 +25,7 @@ public class SoundcloudStreamUrlIdHandlerTest {
 
     @Test(expected = IllegalArgumentException.class)
     public void getIdWithNullAsUrl() throws ParsingException {
-        urlIdHandler.getId(null);
+        urlIdHandler.setUrl(null).getId();
     }
 
     @Test
@@ -37,7 +37,7 @@ public class SoundcloudStreamUrlIdHandlerTest {
         for (String invalidUrl : invalidUrls) {
             Throwable exception = null;
             try {
-                urlIdHandler.getId(invalidUrl);
+                urlIdHandler.setUrl(invalidUrl).getId();
             } catch (ParsingException e) {
                 exception = e;
             }
@@ -49,21 +49,21 @@ public class SoundcloudStreamUrlIdHandlerTest {
 
     @Test
     public void getId() throws Exception {
-        assertEquals("309689103", urlIdHandler.getId("https://soundcloud.com/liluzivert/15-ysl"));
+        assertEquals("309689103", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/15-ysl").getId());
-        assertEquals("309689082", urlIdHandler.getId("https://www.soundcloud.com/liluzivert/15-luv-scars-ko"));
+        assertEquals("309689082", urlIdHandler.setUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko").getId());
-        assertEquals("309689035", urlIdHandler.getId("http://soundcloud.com/liluzivert/15-boring-shit"));
+        assertEquals("309689035", urlIdHandler.setUrl("http://soundcloud.com/liluzivert/15-boring-shit").getId());
-        assertEquals("294488599", urlIdHandler.getId("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats"));
+        assertEquals("294488599", urlIdHandler.setUrl("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats").getId());
-        assertEquals("294488438", urlIdHandler.getId("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz"));
+        assertEquals("294488438", urlIdHandler.setUrl("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz").getId());
-        assertEquals("294488147", urlIdHandler.getId("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69"));
+        assertEquals("294488147", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69").getId());
-        assertEquals("294487876", urlIdHandler.getId("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09"));
+        assertEquals("294487876", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09").getId());
-        assertEquals("294487684", urlIdHandler.getId("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9"));
+        assertEquals("294487684", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9").getId());
-        assertEquals("294487428", urlIdHandler.getId("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s"));
+        assertEquals("294487428", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s").getId());
-        assertEquals("294487157", urlIdHandler.getId("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s"));
+        assertEquals("294487157", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s").getId());
     }
 
 
     @Test
-    public void testAcceptUrl() {
+    public void testAcceptUrl() throws ParsingException {
         assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/liluzivert/15-ysl"));
         assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko"));
         assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/liluzivert/15-boring-shit"));

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java

@@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
 
@@ -64,7 +65,7 @@ public class SoundcloudSubscriptionExtractorTest {
         }
     }
 
-    private void testList(List<SubscriptionItem> subscriptionItems) {
+    private void testList(List<SubscriptionItem> subscriptionItems) throws ParsingException {
         for (SubscriptionItem item : subscriptionItems) {
             assertNotNull(item.getName());
             assertNotNull(item.getUrl());

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -1,13 +1,14 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
 
 import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@@ -49,12 +50,12 @@ public class YoutubeChannelExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getCleanUrl());
+            assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("http://www.youtube.com/user/Gronkh", extractor.getOriginalUrl());
         }
 
@@ -130,7 +131,7 @@ public class YoutubeChannelExtractorTest {
 
         @Test
         public void testGetPageInNewExtractor() throws Exception {
-            final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getCleanUrl());
+            final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
             defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId());
         }
 
@@ -155,12 +156,12 @@ public class YoutubeChannelExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getCleanUrl());
+            assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getOriginalUrl());
         }
 
@@ -251,12 +252,12 @@ public class YoutubeChannelExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getCleanUrl());
+            assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("https://www.youtube.com/user/CaptainDisillusion/videos", extractor.getOriginalUrl());
         }
 
@@ -340,12 +341,12 @@ public class YoutubeChannelExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getCleanUrl());
+            assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getOriginalUrl());
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java

@@ -5,6 +5,7 @@ import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -39,17 +40,17 @@ public class YoutubeChannelUrlIdHandlerTest {
 
     @Test
     public void getIdFromUrl() throws ParsingException {
-        assertEquals("user/Gronkh", urlIdHandler.getId("https://www.youtube.com/user/Gronkh"));
+        assertEquals("user/Gronkh", urlIdHandler.setUrl("https://www.youtube.com/user/Gronkh").getId());
-        assertEquals("user/Netzkino", urlIdHandler.getId("https://www.youtube.com/user/Netzkino/videos"));
+        assertEquals("user/Netzkino", urlIdHandler.setUrl("https://www.youtube.com/user/Netzkino/videos").getId());
 
-        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA"));
+        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA").getId());
-        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
+        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
 
 
-        assertEquals("user/Gronkh", urlIdHandler.getId("https://hooktube.com/user/Gronkh"));
+        assertEquals("user/Gronkh", urlIdHandler.setUrl("https://hooktube.com/user/Gronkh").getId());
-        assertEquals("user/Netzkino", urlIdHandler.getId("https://hooktube.com/user/Netzkino/videos"));
+        assertEquals("user/Netzkino", urlIdHandler.setUrl("https://hooktube.com/user/Netzkino/videos").getId());
 
-        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA"));
+        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId());
-        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
+        assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -7,8 +7,10 @@ import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import static org.junit.Assert.assertEquals;
@@ -53,12 +55,12 @@ public class YoutubePlaylistExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getCleanUrl());
+            assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getOriginalUrl());
         }
 
@@ -135,7 +137,7 @@ public class YoutubePlaylistExtractorTest {
 
         @Test
         public void testGetPageInNewExtractor() throws Exception {
-            final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getCleanUrl());
+            final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl());
             defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId());
         }
 
@@ -160,12 +162,12 @@ public class YoutubePlaylistExtractorTest {
         }
 
         @Test
-        public void testCleanUrl() {
+        public void testUrl() throws ParsingException {
-            assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getCleanUrl());
+            assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getUrl());
         }
 
         @Test
-        public void testOriginalUrl() {
+        public void testOriginalUrl() throws ParsingException {
             assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getOriginalUrl());
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java

@@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
 import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchEngine;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java

@@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
 import org.schabi.newpipe.extractor.stream.VideoStream;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java

@@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.*;
 import org.schabi.newpipe.extractor.utils.Utils;
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java

@@ -25,6 +25,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 
 import static org.junit.Assert.assertTrue;
@@ -49,4 +50,10 @@ public class YoutubeStreamExtractorDonationTest {
         assertTrue(String.valueOf(extractor.getDonationLinks().length),
                 extractor.getDonationLinks().length == 2);
     }
+
+    @Test
+    public void getAffiliateLinksTest() throws Exception {
+        assertTrue(String.valueOf(extractor.getAffiliateLinks().length),
+                extractor.getAffiliateLinks().length == 1);
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java

@@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
 import org.schabi.newpipe.extractor.stream.VideoStream;
@@ -108,7 +110,7 @@ public class YoutubeStreamExtractorRestrictedTest {
         streams.addAll(extractor.getVideoStreams());
         streams.addAll(extractor.getVideoOnlyStreams());
 
-        assertTrue(streams.size() > 0);
+        assertTrue(Integer.toString(streams.size()),streams.size() > 0);
         for (VideoStream s : streams) {
             assertTrue(s.getUrl(),
                     s.getUrl().contains(HTTPS));

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java

@@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -25,14 +26,14 @@ public class YoutubeStreamUrlIdHandlerTest {
         NewPipe.init(Downloader.getInstance());
     }
 
-    @Test(expected = NullPointerException.class)
+    @Test(expected = IllegalArgumentException.class)
     public void getIdWithNullAsUrl() throws ParsingException {
-        urlIdHandler.getId(null);
+        urlIdHandler.setId(null);
     }
 
     @Test(expected = FoundAdException.class)
     public void getIdForAd() throws ParsingException {
-        urlIdHandler.getId(AD_URL);
+        urlIdHandler.setUrl(AD_URL).getId();
     }
 
     @Test
@@ -44,7 +45,7 @@ public class YoutubeStreamUrlIdHandlerTest {
         for (String invalidUrl : invalidUrls) {
             Throwable exception = null;
             try {
-                urlIdHandler.getId(invalidUrl);
+                urlIdHandler.setUrl(invalidUrl).getId();
             } catch (ParsingException e) {
                 exception = e;
             }
@@ -56,42 +57,42 @@ public class YoutubeStreamUrlIdHandlerTest {
 
     @Test
     public void getIdfromYt() throws Exception {
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI").getId());
-        assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q"));
+        assertEquals("W-fFHeTX70Q", urlIdHandler.setUrl("https://www.youtube.com/watch?v=W-fFHeTX70Q").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://youtu.be/jZViOEv90dI?t=9s").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://Youtu.be/jZViOEv90dI?t=9s").getId());
-        assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
+        assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY").getId());
-        assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY"));
+        assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/embed/jZViOEv90dI").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/watch?v=jZViOEv90dI").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtube.com/watch?v=jZViOEv90dI").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtu.be/jZViOEv90dI?t=9s").getId());
-        assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8"));
+        assertEquals("7_WWz2DSnT8", urlIdHandler.setUrl("https://youtu.be/7_WWz2DSnT8").getId());
-        assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY"));
+        assertEquals("oy6NvWeVruY", urlIdHandler.setUrl("https://m.youtube.com/watch?v=oy6NvWeVruY").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/embed/jZViOEv90dI").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId());
-        assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
+        assertEquals("EhxJLojIE_o", urlIdHandler.setUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
-        assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube:jZViOEv90dI").getId());
     }
 
     @Test
     public void getIdfromSharedLinksYt() throws Exception {
         String sharedId = "7JIArTByb3E";
         String realId = "Q7JsK50NGaA";
-        assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
+        assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link").getId());
-        assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId));
+        assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId).getId());
-        assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId));
+        assertEquals(realId, urlIdHandler.setUrl("https://www.youtube.com/shared?ci=" + sharedId).getId());
     }
 
 
     @Test
-    public void testAcceptYtUrl() {
+    public void testAcceptYtUrl() throws ParsingException {
         assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI"));
         assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
         assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
@@ -111,7 +112,7 @@ public class YoutubeStreamUrlIdHandlerTest {
     }
 
     @Test
-    public void testAcceptSharedYtUrl() {
+    public void testAcceptSharedYtUrl() throws ParsingException {
         String sharedId = "8A940MXKFmQ";
         assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
         assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId));
@@ -119,7 +120,7 @@ public class YoutubeStreamUrlIdHandlerTest {
     }
 
     @Test
-    public void testAcceptHookUrl() {
+    public void testAcceptHookUrl() throws ParsingException {
         assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU"));
         assertTrue(urlIdHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE"));
         assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
@@ -130,11 +131,11 @@ public class YoutubeStreamUrlIdHandlerTest {
 
     @Test
     public void testGetHookIdfromUrl() throws ParsingException {
-        assertEquals("TglNG-yjabU", urlIdHandler.getId("https://hooktube.com/watch?v=TglNG-yjabU"));
+        assertEquals("TglNG-yjabU", urlIdHandler.setUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId());
-        assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch?v=3msbfr6pBNE"));
+        assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch?v=3msbfr6pBNE").getId());
-        assertEquals("ocH3oSnZG3c", urlIdHandler.getId("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
+        assertEquals("ocH3oSnZG3c", urlIdHandler.setUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
-        assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch/3msbfr6pBNE"));
+        assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch/3msbfr6pBNE").getId());
-        assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/v/3msbfr6pBNE"));
+        assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/v/3msbfr6pBNE").getId());
-        assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/embed/3msbfr6pBNE"));
+        assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/embed/3msbfr6pBNE").getId());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java

@@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java

@@ -25,6 +25,8 @@ import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -48,6 +50,7 @@ public class YoutubeTrendingExtractorTest {
                 .getKioskList()
                 .getExtractorById("Trending", null);
         extractor.fetchPage();
+        extractor.setContentCountry("de");
     }
 
     @Test
@@ -91,7 +94,7 @@ public class YoutubeTrendingExtractorTest {
     }
 
     @Test
-    public void testGetCleanUrl() {
+    public void testGetUrl() throws Exception {
-        assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending");
+        assertEquals(extractor.getUrl(), extractor.getUrl(), "https://www.youtube.com/feed/trending");
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java

@@ -45,7 +45,7 @@ public class YoutubeTrendingKioskInfoTest {
         StreamingService service = YouTube;
         UrlIdHandler urlIdHandler = service.getKioskList().getUrlIdHandlerByType("Trending");
 
-        kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.getUrl("Trending"), null);
+        kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.setId("Trending").getUrl(), null);
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java

@@ -25,6 +25,7 @@ import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
 
 import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
@@ -46,13 +47,13 @@ public class YoutubeTrendingUrlIdHandlerTest {
     @Test
     public void getUrl()
             throws Exception {
-        assertEquals(urlIdHandler.getUrl(""), "https://www.youtube.com/feed/trending");
+        assertEquals(urlIdHandler.setId("").getUrl(), "https://www.youtube.com/feed/trending");
     }
 
     @Test
     public void getId()
             throws Exception {
-        assertEquals(urlIdHandler.getId(""), "Trending");
+        assertEquals(urlIdHandler.setUrl("").getId(), "Trending");
     }
 
     @Test