diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index f904d333..832ec562 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import edu.umd.cs.findbugs.annotations.NonNull; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -14,42 +15,28 @@ public abstract class Extractor { */ private final StreamingService service; - /** - * Dirty/original url that was passed in the constructor. - *

- * What makes a url "dirty" or not is, for example, the additional parameters - * (not important as—in this case—the id): - * 

-     *     https://www.youtube.com/watch?v=a9Zf_258aTI&t=4s  →  &t=4s
-     *
- * But as you can imagine, the time parameter is very important when calling {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}. - */ - private final String originalUrl; + private final UrlIdHandler urlIdHandler; - /** - * The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}). - *

- * Is lazily-cleaned by calling {@link #getCleanUrl()} - */ @Nullable - private String cleanUrl; private boolean pageFetched = false; private final Downloader downloader; - public Extractor(final StreamingService service, final String url) { + public Extractor(final StreamingService service, final UrlIdHandler urlIdHandler) { if(service == null) throw new NullPointerException("service is null"); - if(url == null) throw new NullPointerException("url is null"); + if(urlIdHandler == null) throw new NullPointerException("UrlIdHandler is null"); this.service = service; - this.originalUrl = url; + this.urlIdHandler = urlIdHandler; this.downloader = NewPipe.getDownloader(); if(downloader == null) throw new NullPointerException("downloader is null"); } /** - * @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler). + * @return The {@link UrlIdHandler} of the current extractor object (e.g. a ChannelExtractor should return a channel url handler). */ @Nonnull - protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException; + protected UrlIdHandler getUrlIdHandler() { + return urlIdHandler; + } /** * Fetch the current page. @@ -79,7 +66,9 @@ public abstract class Extractor { public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException; @Nonnull - public abstract String getId() throws ParsingException; + public String getId() throws ParsingException { + return urlIdHandler.getId(); + } /** * Get the name @@ -90,26 +79,13 @@ public abstract class Extractor { public abstract String getName() throws ParsingException; @Nonnull - public String getOriginalUrl() { - return originalUrl; + public String getOriginalUrl() throws ParsingException { + return urlIdHandler.getOriginalUrl(); } - /** - * Get a clean url and as a fallback the original url. - * @return the clean url or the original url - */ @Nonnull - public String getCleanUrl() { - if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl; - - try { - cleanUrl = getUrlIdHandler().cleanUrl(originalUrl); - } catch (Exception e) { - cleanUrl = null; - // Fallback to the original url - return originalUrl; - } - return cleanUrl; + public String getUrl() throws ParsingException { + return urlIdHandler.getUrl(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index a25f8823..e1a79011 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -12,8 +12,8 @@ import java.util.List; */ public abstract class ListExtractor extends Extractor { - public ListExtractor(StreamingService service, String url) { - super(service, url); + public ListExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java new file mode 100644 index 00000000..43ff19c8 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public abstract class ListUrlIdHandler extends UrlIdHandler { + + protected String[] contentFilter; + protected String sortFilter; + + public ListUrlIdHandler setQuery(String id, String[] contentFilter, String softFilter) throws ParsingException { + setId(id); + this.contentFilter = contentFilter; + this.sortFilter = softFilter; + return this; + } + + public ListUrlIdHandler setUrl(String url) throws ParsingException { + return (ListUrlIdHandler) super.setUrl(url); + } + + public ListUrlIdHandler setId(String id) throws ParsingException { + return (ListUrlIdHandler) super.setId(id); + } + + /** + * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. + * + * @return filter that can be applied when building a query for getting a list + */ + public String[] getAvailableContentFilter() { + return new String[0]; + } + + /** + * Will returns sort filter the corresponding extractor can handle like "A-Z", "oldest first", "size", etc. + * + * @return filter that can be applied when building a query for getting a list + */ + public String[] getAvailableSortFilter() { + return new String[0]; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 7fb66757..fc68dd30 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; @@ -62,22 +63,52 @@ public abstract class StreamingService { return serviceId + ":" + serviceInfo.getName(); } + //////////////////////////////////////////// + // Url Id handler + //////////////////////////////////////////// public abstract UrlIdHandler getStreamUrlIdHandler(); - public abstract UrlIdHandler getChannelUrlIdHandler(); - public abstract UrlIdHandler getPlaylistUrlIdHandler(); + public abstract ListUrlIdHandler getChannelUrlIdHandler(); + public abstract ListUrlIdHandler getPlaylistUrlIdHandler(); + + //////////////////////////////////////////// + // Extractor + //////////////////////////////////////////// public abstract SearchEngine getSearchEngine(); public abstract SuggestionExtractor getSuggestionExtractor(); - public abstract StreamExtractor getStreamExtractor(String url); - public abstract KioskList getKioskList() throws ExtractionException; - public abstract ChannelExtractor getChannelExtractor(String url); - public abstract PlaylistExtractor getPlaylistExtractor(String url); public abstract SubscriptionExtractor getSubscriptionExtractor(); + public abstract KioskList getKioskList() throws ExtractionException; + + public abstract ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException; + public abstract PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException; + public abstract StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException; + + public ChannelExtractor getChannelExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException { + return getChannelExtractor(getChannelUrlIdHandler().setQuery(id, contentFilter, sortFilter)); + } + + public PlaylistExtractor getPlaylistExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException { + return getPlaylistExtractor(getPlaylistUrlIdHandler().setQuery(id, contentFilter, sortFilter)); + } + + public ChannelExtractor getChannelExtractor(String url) throws ExtractionException { + return getChannelExtractor(getChannelUrlIdHandler().setUrl(url)); + } + + public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException { + return getPlaylistExtractor(getPlaylistUrlIdHandler().setUrl(url)); + } + + public StreamExtractor getStreamExtractor(String url) throws ExtractionException { + return getStreamExtractor(getStreamUrlIdHandler().setUrl(url)); + } + + /** * figure out where the link is pointing to (a channel, video, playlist, etc.) */ - public final LinkType getLinkTypeByUrl(String url) { + public final LinkType getLinkTypeByUrl(String url) throws ParsingException { UrlIdHandler sH = getStreamUrlIdHandler(); UrlIdHandler cH = getChannelUrlIdHandler(); UrlIdHandler pH = getPlaylistUrlIdHandler(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java index 6fe43ec8..a9fda4d3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; /* @@ -22,16 +23,52 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; * along with NewPipe. If not, see . */ -public interface UrlIdHandler { +public abstract class UrlIdHandler { - String getUrl(String id) throws ParsingException; - String getId(String url) throws ParsingException; - String cleanUrl(String complexUrl) throws ParsingException; + protected String id = ""; + protected String originalUrl = ""; + + public abstract String onGetIdFromUrl(String url) throws ParsingException; + public abstract String getUrl() throws ParsingException; + public abstract boolean onAcceptUrl(final String url) throws ParsingException; + + + public UrlIdHandler setUrl(String url) throws ParsingException { + if(url == null) throw new IllegalArgumentException("url can not be null"); + originalUrl = url; + id = onGetIdFromUrl(url); + return this; + } + + public UrlIdHandler setId(String id) throws ParsingException { + if(id == null) throw new IllegalArgumentException("id can not be null"); + this.id = id; + if(!acceptUrl(getUrl())) { + throw new ParsingException("Malformed unacceptable url: " + getUrl()); + } + return this; + } + + public String getId() { + return id; + } + + public String getOriginalUrl() throws ParsingException { + return (originalUrl == null || originalUrl.isEmpty()) + ? getUrl() + : originalUrl; + } /** * When a VIEW_ACTION is caught this function will test if the url delivered within the calling * Intent was meant to be watched with this Service. * Return false if this service shall not allow to be called through ACTIONs. */ - boolean acceptUrl(String url); + public boolean acceptUrl(final String url) { + try { + return onAcceptUrl(url); + } catch (Exception e) { + return false; + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 05a5bc58..625f6381 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,8 +1,10 @@ package org.schabi.newpipe.extractor.channel; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -30,8 +32,8 @@ import javax.annotation.Nonnull; public abstract class ChannelExtractor extends ListExtractor { - public ChannelExtractor(StreamingService service, String url) { - super(service, url); + public ChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 0d8a860b..fd5868b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -54,7 +54,7 @@ public class ChannelInfo extends ListInfo { // important data int serviceId = extractor.getServiceId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); String id = extractor.getId(); String name = extractor.getName(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 31b496b4..95ce34f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.kiosk; */ import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -33,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor { private final String id; public KioskExtractor(StreamingService streamingService, - String url, - String kioskId) - throws ExtractionException { - super(streamingService, url); + ListUrlIdHandler urlIdHandler, + String kioskId) { + super(streamingService, urlIdHandler); this.id = kioskId; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 550b8b57..f11ebb9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -71,7 +71,7 @@ public class KioskInfo extends ListInfo { int serviceId = extractor.getServiceId(); String name = extractor.getName(); String id = extractor.getId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); KioskInfo info = new KioskInfo(serviceId, id, url, originalUrl, name); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index fb8e4868..11c8f141 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -73,7 +73,7 @@ public class KioskList { throw new ExtractionException("No kiosk found with the type: " + kioskId); } else { return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id), - ke.handler.getUrl(kioskId), kioskId); + ke.handler.setId(kioskId).getUrl(), kioskId); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 4569e7b5..cc7107f7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,8 +1,10 @@ package org.schabi.newpipe.extractor.playlist; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -10,8 +12,8 @@ import javax.annotation.Nonnull; public abstract class PlaylistExtractor extends ListExtractor { - public PlaylistExtractor(StreamingService service, String url) { - super(service, url); + public PlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index c8910651..29f7b144 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -38,7 +38,7 @@ public class PlaylistInfo extends ListInfo { public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException { int serviceId = extractor.getServiceId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); String id = extractor.getId(); String name = extractor.getName(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 7232ac5a..6c2aba0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -25,14 +26,14 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { private StreamInfoItemsCollector streamInfoItemsCollector = null; private String nextPageUrl = null; - public SoundcloudChannelExtractor(StreamingService service, String url) { - super(service, url); + public SoundcloudChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - userId = getUrlIdHandler().getId(getOriginalUrl()); + userId = getUrlIdHandler().getId(); String apiUrl = "https://api-v2.soundcloud.com/users/" + userId + "?client_id=" + SoundcloudParsingHelper.clientId(); @@ -44,12 +45,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } } - @Nonnull - @Override - public String getCleanUrl() { - return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl(); - } - @Nonnull @Override public String getId() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java index 04902eb3..063ad7aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.jsoup.Jsoup; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -10,7 +11,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; -public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { +public class SoundcloudChannelUrlIdHandler extends ListUrlIdHandler { private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; @@ -19,17 +20,9 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { return instance; } - @Override - public String getUrl(String id) throws ParsingException { - try { - return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); try { @@ -40,21 +33,16 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - Utils.checkUrl(URL_PATTERN, complexUrl); - + public String getUrl() throws ParsingException { try { - Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) - .select("meta[property=og:url]").first(); - - return replaceHttpWithHttps(ogElement.attr("content")); + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } } @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index bfea06e7..655d4c06 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -14,15 +15,11 @@ import java.util.Arrays; import java.util.List; public class SoundcloudChartsExtractor extends KioskExtractor { - private String url; - private StreamInfoItemsCollector collector = null; private String nextPageUrl = null; - public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId) - throws ExtractionException { - super(service, url, kioskId); - this.url = url; + public SoundcloudChartsExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId) { + super(service, urlIdHandler, kioskId); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java index b26f0deb..03a846cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java @@ -1,13 +1,25 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; -public class SoundcloudChartsUrlIdHandler implements UrlIdHandler { +public class SoundcloudChartsUrlIdHandler extends ListUrlIdHandler { private final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$"; private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$"; - public String getUrl(String id) { + + @Override + public String onGetIdFromUrl(String url) { + if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { + return "Top 50"; + } else { + return "New & hot"; + } + } + + public String getUrl() { if (id.equals("Top 50")) { return "https://soundcloud.com/charts/top"; } else { @@ -16,25 +28,7 @@ public class SoundcloudChartsUrlIdHandler implements UrlIdHandler { } @Override - public String getId(String url) { - if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { - return "Top 50"; - } else { - return "New & hot"; - } - } - - @Override - public String cleanUrl(String url) { - if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { - return "https://soundcloud.com/charts/top"; - } else { - return "https://soundcloud.com/charts/new"; - } - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 84379b9e..d6647539 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -24,14 +25,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private StreamInfoItemsCollector streamInfoItemsCollector = null; private String nextPageUrl = null; - public SoundcloudPlaylistExtractor(StreamingService service, String url) { - super(service, url); + public SoundcloudPlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - playlistId = getUrlIdHandler().getId(getOriginalUrl()); + playlistId = getUrlIdHandler().getId(); String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + "?client_id=" + SoundcloudParsingHelper.clientId() + "&representation=compact"; @@ -44,12 +45,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } } - @Nonnull - @Override - public String getCleanUrl() { - return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl(); - } - @Nonnull @Override public String getId() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java index 9518f1f9..2855c1d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java @@ -1,16 +1,11 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - -public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { +public class SoundcloudPlaylistUrlIdHandler extends ListUrlIdHandler { private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + "/sets/[0-9a-z_-]+/?([#?].*)?$"; @@ -20,7 +15,18 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { + Utils.checkUrl(URL_PATTERN, url); + + try { + return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + } catch (Exception e) { + throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e); + } + } + + @Override + public String getUrl() throws ParsingException { try { return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id); } catch (Exception e) { @@ -29,32 +35,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { } @Override - public String getId(String url) throws ParsingException { - Utils.checkUrl(URL_PATTERN, url); - - try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } - - @Override - public String cleanUrl(String complexUrl) throws ParsingException { - Utils.checkUrl(URL_PATTERN, complexUrl); - - try { - Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) - .select("meta[property=og:url]").first(); - - return replaceHttpWithHttps(ogElement.attr("content")); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) throws ParsingException { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index b65be7b2..b3e90e72 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; @@ -32,29 +33,29 @@ public class SoundcloudService extends StreamingService { } @Override - public UrlIdHandler getChannelUrlIdHandler() { + public ListUrlIdHandler getChannelUrlIdHandler() { return SoundcloudChannelUrlIdHandler.getInstance(); } @Override - public UrlIdHandler getPlaylistUrlIdHandler() { + public ListUrlIdHandler getPlaylistUrlIdHandler() { return SoundcloudPlaylistUrlIdHandler.getInstance(); } @Override - public StreamExtractor getStreamExtractor(String url) { - return new SoundcloudStreamExtractor(this, url); + public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException { + return new SoundcloudStreamExtractor(this, urlIdHandler); } @Override - public ChannelExtractor getChannelExtractor(String url) { - return new SoundcloudChannelExtractor(this, url); + public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new SoundcloudChannelExtractor(this, urlIdHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(String url) { - return new SoundcloudPlaylistExtractor(this, url); + public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new SoundcloudPlaylistExtractor(this, urlIdHandler); } @Override @@ -71,8 +72,7 @@ public class SoundcloudService extends StreamingService { String id) throws ExtractionException { return new SoundcloudChartsExtractor(SoundcloudService.this, - url, - id); + new SoundcloudChartsUrlIdHandler().setUrl(url), id); } }; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 8de597d8..46fb76fb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -22,8 +22,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; - public SoundcloudStreamExtractor(StreamingService service, String url) { - super(service, url); + public SoundcloudStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override @@ -36,12 +36,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } } - @Nonnull - @Override - public String getCleanUrl() { - return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl(); - } - @Nonnull @Override public String getId() { @@ -214,6 +208,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return new String[0]; } + @Override + public String[] getAffiliateLinks() { + return new String[0]; + } + @Override public String getErrorMessage() { return null; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java index b38dac03..7a760a2b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; -public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { +public class SoundcloudStreamUrlIdHandler extends UrlIdHandler { private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; @@ -23,7 +23,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) throws ParsingException { + public String getUrl() throws ParsingException { try { return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id); } catch (Exception e) { @@ -32,7 +32,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { } @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); try { @@ -43,21 +43,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - Utils.checkUrl(URL_PATTERN, complexUrl); - - try { - Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) - .select("meta[property=og:url]").first(); - - return replaceHttpWithHttps(ogElement.attr("content")); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) throws ParsingException { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java index 49e64414..d34a4f8d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java @@ -31,7 +31,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { String id; try { - id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl)); + id = service.getChannelUrlIdHandler().setUrl(getUrlFrom(channelUrl)).getId(); } catch (ExtractionException e) { throw new InvalidSourceException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index 1950c0b5..3be29c98 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -43,6 +43,9 @@ public class ItagItem { new ItagItem(139, AUDIO, M4A, 48), new ItagItem(140, AUDIO, M4A, 128), new ItagItem(141, AUDIO, M4A, 256), + new ItagItem(249, AUDIO, OPUS, 50), + new ItagItem(250, AUDIO, OPUS, 70), + new ItagItem(160, AUDIO, OPUS, 160), /// VIDEO ONLY //////////////////////////////////////////// // ID Type Format Resolution FPS /// diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 2ac5dff2..77929139 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; @@ -9,6 +10,11 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.services.youtube.extractors.*; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubePlaylistUrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; @@ -53,28 +59,28 @@ public class YoutubeService extends StreamingService { } @Override - public UrlIdHandler getChannelUrlIdHandler() { + public ListUrlIdHandler getChannelUrlIdHandler() { return YoutubeChannelUrlIdHandler.getInstance(); } @Override - public UrlIdHandler getPlaylistUrlIdHandler() { + public ListUrlIdHandler getPlaylistUrlIdHandler() { return YoutubePlaylistUrlIdHandler.getInstance(); } @Override - public StreamExtractor getStreamExtractor(String url) { - return new YoutubeStreamExtractor(this, url); + public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException { + return new YoutubeStreamExtractor(this, urlIdHandler); } @Override - public ChannelExtractor getChannelExtractor(String url) { - return new YoutubeChannelExtractor(this, url); + public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new YoutubeChannelExtractor(this, urlIdHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(String url) { - return new YoutubePlaylistExtractor(this, url); + public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new YoutubePlaylistExtractor(this, urlIdHandler); } @Override @@ -92,7 +98,8 @@ public class YoutubeService extends StreamingService { @Override public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) throws ExtractionException { - return new YoutubeTrendingExtractor(YoutubeService.this, url, id); + return new YoutubeTrendingExtractor(YoutubeService.this, + new YoutubeTrendingUrlIdHandler().setUrl(url), id); } }, new YoutubeTrendingUrlIdHandler(), "Trending"); list.setDefaultKiosk("Trending"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java similarity index 93% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 195d20ab..a602f0ea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,17 +1,13 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import com.sun.org.apache.xerces.internal.xs.StringList; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -52,16 +48,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private Document doc; - public YoutubeChannelExtractor(StreamingService service, String url) { - super(service, url); + public YoutubeChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS; + String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; String pageContent = downloader.download(channelUrl); doc = Jsoup.parse(pageContent, channelUrl); - } @Override @@ -71,11 +66,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override - public String getCleanUrl() { + public String getUrl() throws ParsingException { try { return "https://www.youtube.com/channel/" + getId(); } catch (ParsingException e) { - return super.getCleanUrl(); + return super.getUrl(); } } @@ -196,7 +191,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } for(Element a : linkHolder.select("a")) { String link = a.attr("abs:href"); - if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { + if(DonationLinkHelper.getDonatoinServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { links.add(link); } } @@ -236,7 +231,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { collector.reset(); final String uploaderName = getName(); - final String uploaderUrl = getCleanUrl(); + final String uploaderUrl = getUrl(); for (final Element li : element.children()) { if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) { collector.commit(new YoutubeStreamInfoItemExtractor(li) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index a04d8562..0c1e5254 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java similarity index 91% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index eff7adc1..601f6cae 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -6,13 +6,11 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; @@ -26,14 +24,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private Document doc; - public YoutubePlaylistExtractor(StreamingService service, String url) { - super(service, url); + public YoutubePlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) throws ExtractionException { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String pageContent = downloader.download(getCleanUrl()); - doc = Jsoup.parse(pageContent, getCleanUrl()); + String pageContent = downloader.download(getUrl()); + doc = Jsoup.parse(pageContent, getUrl()); } @Override @@ -41,16 +39,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return getNextPageUrlFrom(doc); } - @Nonnull - @Override - public String getId() throws ParsingException { - try { - return getUrlIdHandler().getId(getCleanUrl()); - } catch (Exception e) { - throw new ParsingException("Could not get playlist id"); - } - } - @Nonnull @Override public String getName() throws ParsingException { @@ -202,7 +190,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUrl() throws ParsingException { try { - return streamUrlIdHandler.getUrl(li.attr("data-video-id")); + return streamUrlIdHandler.setId(li.attr("data-video-id")).getUrl(); } catch (Exception e) { throw new ParsingException("Could not get web page url for the video", e); } @@ -267,7 +255,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getUrl()) + "/hqdefault.jpg"; + return "https://i.ytimg.com/vi/" + streamUrlIdHandler.setUrl(getUrl()).getId() + "/hqdefault.jpg"; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index fe6e7768..1b5fa449 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java index d393c360..cecfbd18 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java similarity index 96% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c9839ac6..87571e8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -10,14 +10,12 @@ import org.jsoup.nodes.Element; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.Subtitles; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.DonationLinkHelper; import org.schabi.newpipe.extractor.utils.Parser; @@ -86,24 +84,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { private boolean isAgeRestricted; - public YoutubeStreamExtractor(StreamingService service, String url) { - super(service, url); + public YoutubeStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) throws ExtractionException { + super(service, urlIdHandler); } /*////////////////////////////////////////////////////////////////////////// // Impl //////////////////////////////////////////////////////////////////////////*/ - @Nonnull - @Override - public String getId() throws ParsingException { - try { - return getUrlIdHandler().getId(getCleanUrl()); - } catch (Exception e) { - throw new ParsingException("Could not get stream id"); - } - } - @Nonnull @Override public String getName() throws ParsingException { @@ -534,7 +522,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { ArrayList donationLinks = new ArrayList<>(); for (String s : Parser.getLinksFromString(getDescription())) { - if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) { + if (DonationLinkHelper.getDonatoinServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) { donationLinks.add(s); } } @@ -546,6 +534,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Override + public String[] getAffiliateLinks() throws ParsingException { + try { + ArrayList donationLinks = new ArrayList<>(); + for (String s : Parser.getLinksFromString(getDescription())) { + if (DonationLinkHelper.getAffiliateServiceByLink(s) != DonationLinkHelper.AffiliateService.NO_AFILIATE) { + donationLinks.add(s); + } + } + String[] donlret = new String[donationLinks.size()]; + donlret = donationLinks.toArray(donlret); + return donlret; + } catch (Exception e) { + throw new ParsingException("Could not get afiliate links", e); + } + } + /*////////////////////////////////////////////////////////////////////////// // Fetch page @@ -564,7 +569,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private String pageHtml = null; private String getPageHtml(Downloader downloader) throws IOException, ExtractionException { - final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS; + final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; if (pageHtml == null) { pageHtml = downloader.download(verifiedUrl); } @@ -574,7 +579,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String pageContent = getPageHtml(downloader); - doc = Jsoup.parse(pageContent, getCleanUrl()); + doc = Jsoup.parse(pageContent, getUrl()); final String playerUrl; // Check if the video is age restricted diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java similarity index 95% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 7e792c1a..3b1bb719 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,7 +1,8 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; @@ -135,7 +136,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { Element meta = item.select("div[class=\"yt-lockup-meta\"]").first(); if (meta == null) return -1; + // This case can happen if google releases a special video + if(meta.select("li").size() < 2) return -1; + input = meta.select("li").get(1).text(); + } catch (IndexOutOfBoundsException e) { throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java index 32efe16b..f6b6b8bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java @@ -1,9 +1,10 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; import org.schabi.newpipe.extractor.utils.Parser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index cd98f25b..5d77ab0c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java similarity index 94% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 81c2c268..9c4cb080 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; /* * Created by Christian Schabesberger on 12.08.17. @@ -25,11 +25,13 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -40,15 +42,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; - public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId) + public YoutubeTrendingExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId) throws ExtractionException { - super(service, url, kioskId); + super(service, urlIdHandler, kioskId); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String contentCountry = getContentCountry(); - String url = getCleanUrl(); + String url = getUrl(); if(contentCountry != null && !contentCountry.isEmpty()) { url += "?gl=" + contentCountry; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java similarity index 77% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java index bc9ce681..e6dbf09b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java @@ -1,6 +1,6 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; @@ -24,7 +24,7 @@ import org.schabi.newpipe.extractor.utils.Parser; * along with NewPipe. If not, see . */ -public class YoutubeChannelUrlIdHandler implements UrlIdHandler { +public class YoutubeChannelUrlIdHandler extends ListUrlIdHandler { private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler(); private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)"; @@ -34,22 +34,17 @@ public class YoutubeChannelUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) { - return "https://www.youtube.com/" + id; - } - - @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { return Parser.matchGroup1(ID_PATTERN, url); } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - return getUrl(getId(complexUrl)); + public String getUrl() { + return "https://www.youtube.com/" + id; } @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(String url) { return (url.contains("youtube") || url.contains("youtu.be") || url.contains("hooktube.com")) && (url.contains("/user/") || url.contains("/channel/")); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java similarity index 96% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java index a0ae0c6c..90f464ed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java similarity index 68% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java index 0c24f993..262cbfc2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java @@ -1,11 +1,11 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; -public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { +public class YoutubePlaylistUrlIdHandler extends ListUrlIdHandler { private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler(); private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})"; @@ -15,12 +15,12 @@ public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) { + public String getUrl() { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { try { return Parser.matchGroup1("list=" + ID_PATTERN, url); } catch (final Exception exception) { @@ -28,13 +28,9 @@ public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { } } - @Override - public String cleanUrl(String complexUrl) throws ParsingException { - return getUrl(getId(complexUrl)); - } @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { final boolean hasNotEmptyUrl = url != null && !url.isEmpty(); final boolean isYoutubeDomain = hasNotEmptyUrl && (url.contains("youtube") || url.contains("youtu.be")); return isYoutubeDomain && url.contains("list="); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java similarity index 93% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java index 9addec48..75870662 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -37,7 +37,7 @@ import java.net.URLDecoder; * along with NewPipe. If not, see . */ -public class YoutubeStreamUrlIdHandler implements UrlIdHandler { +public class YoutubeStreamUrlIdHandler extends UrlIdHandler { private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler(); private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})"; @@ -50,12 +50,12 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) { + public String getUrl() { return "https://www.youtube.com/watch?v=" + id; } @Override - public String getId(String url) throws ParsingException, IllegalArgumentException { + public String onGetIdFromUrl(String url) throws ParsingException, IllegalArgumentException { if (url.isEmpty()) { throw new IllegalArgumentException("The url parameter should not be empty"); } @@ -167,19 +167,14 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler { } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - return getUrl(getId(complexUrl)); - } - - @Override - public boolean acceptUrl(String url) { - String lowercaseUrl = url.toLowerCase(); + public boolean onAcceptUrl(final String url) { + final String lowercaseUrl = url.toLowerCase(); if (lowercaseUrl.contains("youtube") || lowercaseUrl.contains("youtu.be") || lowercaseUrl.contains("hooktube")) { // bad programming I know try { - getId(url); + onGetIdFromUrl(url); return true; } catch (Exception e) { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java similarity index 75% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java index 63a35181..c3696cd7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; /* * Created by Christian Schabesberger on 12.08.17. @@ -20,27 +20,24 @@ package org.schabi.newpipe.extractor.services.youtube; * along with NewPipe. If not, see . */ +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; -public class YoutubeTrendingUrlIdHandler implements UrlIdHandler { +public class YoutubeTrendingUrlIdHandler extends ListUrlIdHandler { - public String getUrl(String id) { + public String getUrl() { return "https://www.youtube.com/feed/trending"; } @Override - public String getId(String url) { + public String onGetIdFromUrl(String url) { return "Trending"; } @Override - public String cleanUrl(String url) { - return getUrl(""); - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 0aeedd02..a03bb613 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -39,8 +39,8 @@ public abstract class StreamExtractor extends Extractor { public static final int NO_AGE_LIMIT = 0; - public StreamExtractor(StreamingService service, String url) { - super(service, url); + public StreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Nonnull @@ -142,6 +142,7 @@ public abstract class StreamExtractor extends Extractor { public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; public abstract String[] getDonationLinks() throws ExtractionException; + public abstract String[] getAffiliateLinks() throws ExtractionException; /** * Analyses the webpage's document and extracts any error message there might be. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 716afa4b..9443e1ff 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -85,7 +85,7 @@ public class StreamInfo extends Info { // if one of these is not available an exception is meant to be thrown directly into the frontend. int serviceId = extractor.getServiceId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); StreamType streamType = extractor.getStreamType(); String id = extractor.getId(); @@ -242,6 +242,16 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setAffiliateLinks(extractor.getAffiliateLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setDonationLinks(extractor.getDonationLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -274,6 +284,9 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles; + private String[] donationLinks; + private String[] affiliateLinks; + /** * Get the stream type * @@ -466,4 +479,20 @@ public class StreamInfo extends Info { public void setSubtitles(List subtitles) { this.subtitles = subtitles; } + + public String[] getDonationLinks() { + return donationLinks; + } + + public void setDonationLinks(String[] donationLinks) { + this.donationLinks = donationLinks; + } + + public String[] getAffiliateLinks() { + return affiliateLinks; + } + + public void setAffiliateLinks(String[] affiliateLinks) { + this.affiliateLinks = affiliateLinks; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java index b62e062f..817595ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java @@ -7,12 +7,16 @@ public class DonationLinkHelper { public enum DonationService { NO_DONATION, PATREON, - PAYPAL + PAYPAL, } + public enum AffiliateService { + NO_AFILIATE, + AMAZON, + } - public static DonationService getServiceByLink(String link) throws MalformedURLException { - URL url = new URL(link); + public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException { + URL url = new URL(fixLink(link)); switch (url.getHost()) { case "www.patreon.com": return DonationService.PATREON; @@ -27,5 +31,18 @@ public class DonationLinkHelper { } } + public static AffiliateService getAffiliateServiceByLink(String link) throws MalformedURLException { + URL url = new URL(fixLink(link)); + switch (url.getHost()) { + case "amzn.to": return AffiliateService.AMAZON; + default: return AffiliateService.NO_AFILIATE; + } + } + + private static String fixLink(String link) { + return (link.startsWith("https://") || link.startsWith("http://")) + ? link + : "https://" + link; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java index 020cceaf..4ff51295 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java @@ -5,6 +5,6 @@ public interface BaseExtractorTest { void testServiceId() throws Exception; void testName() throws Exception; void testId() throws Exception; - void testCleanUrl() throws Exception; + void testUrl() throws Exception; void testOriginalUrl() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java index da5bfdcb..ea097a3b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java @@ -4,7 +4,6 @@ package org.schabi.newpipe.extractor.services; public interface BasePlaylistExtractorTest extends BaseListExtractorTest { void testThumbnailUrl() throws Exception; void testBannerUrl() throws Exception; - void testUploaderUrl() throws Exception; void testUploaderName() throws Exception; void testUploaderAvatarUrl() throws Exception; void testStreamCount() throws Exception; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 9266aaa8..30884adc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import static org.junit.Assert.*; @@ -48,12 +49,12 @@ public class SoundcloudChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/liluzivert", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/liluzivert", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl()); } @@ -118,7 +119,7 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getCleanUrl()); + final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); } @@ -142,12 +143,12 @@ public class SoundcloudChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/dubmatix", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/dubmatix", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index f3e720c8..a7723024 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -88,6 +88,6 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetCleanUrl() throws Exception { - assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top"); + assertEquals(extractor.getUrl(), "https://soundcloud.com/charts/top"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java index e805b297..4fa09ffa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java @@ -4,6 +4,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; @@ -22,19 +23,19 @@ public class SoundcloudChartsUrlIdHandlerTest { } @Test - public void getUrl() { - assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top"); - assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new"); + public void getUrl() throws Exception { + assertEquals(urlIdHandler.setId("Top 50").getUrl(), "https://soundcloud.com/charts/top"); + assertEquals(urlIdHandler.setId("New & hot").getUrl(), "https://soundcloud.com/charts/new"); } @Test - public void getId() { - assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50"); - assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot"); + public void getId() throws ParsingException { + assertEquals(urlIdHandler.setUrl("http://soundcloud.com/charts/top?genre=all-music").getId(), "Top 50"); + assertEquals(urlIdHandler.setUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries").getId(), "New & hot"); } @Test - public void acceptUrl() { + public void acceptUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts")); assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/")); assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 017efc50..29bf4d17 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -51,12 +51,12 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getCleanUrl()); + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws Exception { assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r?test=123", extractor.getOriginalUrl()); } @@ -125,7 +125,7 @@ public class SoundcloudPlaylistExtractorTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); extractor = (SoundcloudPlaylistExtractor) SoundCloud - .getPlaylistExtractor("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2"); + .getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2"); extractor.fetchPage(); } @@ -140,22 +140,22 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testName() { - assertEquals("Random House & Dance Music #2", extractor.getName()); + assertEquals("House, Electro , Dance Music 2", extractor.getName()); } @Test public void testId() { - assertEquals("436855608", extractor.getId()); + assertEquals("310980722", extractor.getId()); } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getCleanUrl()); + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl()); } @Test - public void testOriginalUrl() { - assertEquals("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getOriginalUrl()); + public void testOriginalUrl() throws Exception { + assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -191,12 +191,12 @@ public class SoundcloudPlaylistExtractorTest { public void testUploaderUrl() { final String uploaderUrl = extractor.getUploaderUrl(); assertIsSecureUrl(uploaderUrl); - assertTrue(uploaderUrl, uploaderUrl.contains("finn-trapple")); + assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader")); } @Test public void testUploaderName() { - assertEquals("Finn TrApple", extractor.getUploaderName()); + assertEquals("Gosu", extractor.getUploaderName()); } @Test @@ -227,7 +227,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getCleanUrl()); + final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); } @@ -251,12 +251,12 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getCleanUrl()); + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws Exception { assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java index f2e5e221..5c1d35cf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java @@ -25,7 +25,7 @@ public class SoundcloudStreamUrlIdHandlerTest { @Test(expected = IllegalArgumentException.class) public void getIdWithNullAsUrl() throws ParsingException { - urlIdHandler.getId(null); + urlIdHandler.setUrl(null).getId(); } @Test @@ -37,7 +37,7 @@ public class SoundcloudStreamUrlIdHandlerTest { for (String invalidUrl : invalidUrls) { Throwable exception = null; try { - urlIdHandler.getId(invalidUrl); + urlIdHandler.setUrl(invalidUrl).getId(); } catch (ParsingException e) { exception = e; } @@ -49,21 +49,21 @@ public class SoundcloudStreamUrlIdHandlerTest { @Test public void getId() throws Exception { - assertEquals("309689103", urlIdHandler.getId("https://soundcloud.com/liluzivert/15-ysl")); - assertEquals("309689082", urlIdHandler.getId("https://www.soundcloud.com/liluzivert/15-luv-scars-ko")); - assertEquals("309689035", urlIdHandler.getId("http://soundcloud.com/liluzivert/15-boring-shit")); - assertEquals("294488599", urlIdHandler.getId("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats")); - assertEquals("294488438", urlIdHandler.getId("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz")); - assertEquals("294488147", urlIdHandler.getId("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69")); - assertEquals("294487876", urlIdHandler.getId("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09")); - assertEquals("294487684", urlIdHandler.getId("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9")); - assertEquals("294487428", urlIdHandler.getId("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s")); - assertEquals("294487157", urlIdHandler.getId("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s")); + assertEquals("309689103", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/15-ysl").getId()); + assertEquals("309689082", urlIdHandler.setUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko").getId()); + assertEquals("309689035", urlIdHandler.setUrl("http://soundcloud.com/liluzivert/15-boring-shit").getId()); + assertEquals("294488599", urlIdHandler.setUrl("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats").getId()); + assertEquals("294488438", urlIdHandler.setUrl("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz").getId()); + assertEquals("294488147", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69").getId()); + assertEquals("294487876", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09").getId()); + assertEquals("294487684", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9").getId()); + assertEquals("294487428", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s").getId()); + assertEquals("294487157", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s").getId()); } @Test - public void testAcceptUrl() { + public void testAcceptUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/liluzivert/15-ysl")); assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko")); assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/liluzivert/15-boring-shit")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java index cbe1948e..e0e99803 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; @@ -64,7 +65,7 @@ public class SoundcloudSubscriptionExtractorTest { } } - private void testList(List subscriptionItems) { + private void testList(List subscriptionItems) throws ParsingException { for (SubscriptionItem item : subscriptionItems) { assertNotNull(item.getName()); assertNotNull(item.getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 510f210a..c9ab333f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,13 +1,14 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; @@ -49,12 +50,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("http://www.youtube.com/user/Gronkh", extractor.getOriginalUrl()); } @@ -130,7 +131,7 @@ public class YoutubeChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getCleanUrl()); + final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); } @@ -155,12 +156,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getOriginalUrl()); } @@ -251,12 +252,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/user/CaptainDisillusion/videos", extractor.getOriginalUrl()); } @@ -340,12 +341,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java index 57bccdf9..345d8b95 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -39,17 +40,17 @@ public class YoutubeChannelUrlIdHandlerTest { @Test public void getIdFromUrl() throws ParsingException { - assertEquals("user/Gronkh", urlIdHandler.getId("https://www.youtube.com/user/Gronkh")); - assertEquals("user/Netzkino", urlIdHandler.getId("https://www.youtube.com/user/Netzkino/videos")); + assertEquals("user/Gronkh", urlIdHandler.setUrl("https://www.youtube.com/user/Gronkh").getId()); + assertEquals("user/Netzkino", urlIdHandler.setUrl("https://www.youtube.com/user/Netzkino/videos").getId()); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA")); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); - assertEquals("user/Gronkh", urlIdHandler.getId("https://hooktube.com/user/Gronkh")); - assertEquals("user/Netzkino", urlIdHandler.getId("https://hooktube.com/user/Netzkino/videos")); + assertEquals("user/Gronkh", urlIdHandler.setUrl("https://hooktube.com/user/Gronkh").getId()); + assertEquals("user/Netzkino", urlIdHandler.setUrl("https://hooktube.com/user/Netzkino/videos").getId()); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA")); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 6b51d0d0..957ceacc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -7,8 +7,10 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.junit.Assert.assertEquals; @@ -53,12 +55,12 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getOriginalUrl()); } @@ -135,7 +137,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getCleanUrl()); + final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); } @@ -160,12 +162,12 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java index 57422ad7..5742f96c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchEngine; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java index de8fdca6..e36faeb2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java @@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.VideoStream; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 88aa1b96..982ceb98 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java index 8bc82ded..60ee0546 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java @@ -25,6 +25,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import static org.junit.Assert.assertTrue; @@ -49,4 +50,10 @@ public class YoutubeStreamExtractorDonationTest { assertTrue(String.valueOf(extractor.getDonationLinks().length), extractor.getDonationLinks().length == 2); } + + @Test + public void getAffiliateLinksTest() throws Exception { + assertTrue(String.valueOf(extractor.getAffiliateLinks().length), + extractor.getAffiliateLinks().length == 1); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index 93b0677b..35d7760d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -108,7 +110,7 @@ public class YoutubeStreamExtractorRestrictedTest { streams.addAll(extractor.getVideoStreams()); streams.addAll(extractor.getVideoOnlyStreams()); - assertTrue(streams.size() > 0); + assertTrue(Integer.toString(streams.size()),streams.size() > 0); for (VideoStream s : streams) { assertTrue(s.getUrl(), s.getUrl().contains(HTTPS)); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java index 2711c7ae..57f00079 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; import java.util.ArrayList; import java.util.List; @@ -25,14 +26,14 @@ public class YoutubeStreamUrlIdHandlerTest { NewPipe.init(Downloader.getInstance()); } - @Test(expected = NullPointerException.class) + @Test(expected = IllegalArgumentException.class) public void getIdWithNullAsUrl() throws ParsingException { - urlIdHandler.getId(null); + urlIdHandler.setId(null); } @Test(expected = FoundAdException.class) public void getIdForAd() throws ParsingException { - urlIdHandler.getId(AD_URL); + urlIdHandler.setUrl(AD_URL).getId(); } @Test @@ -44,7 +45,7 @@ public class YoutubeStreamUrlIdHandlerTest { for (String invalidUrl : invalidUrls) { Throwable exception = null; try { - urlIdHandler.getId(invalidUrl); + urlIdHandler.setUrl(invalidUrl).getId(); } catch (ParsingException e) { exception = e; } @@ -56,42 +57,42 @@ public class YoutubeStreamUrlIdHandlerTest { @Test public void getIdfromYt() throws Exception { - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI")); - assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s")); - assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY")); - assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s")); - assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8")); - assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI")); - assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("W-fFHeTX70Q", urlIdHandler.setUrl("https://www.youtube.com/watch?v=W-fFHeTX70Q").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://youtu.be/jZViOEv90dI?t=9s").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://Youtu.be/jZViOEv90dI?t=9s").getId()); + assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY").getId()); + assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtu.be/jZViOEv90dI?t=9s").getId()); + assertEquals("7_WWz2DSnT8", urlIdHandler.setUrl("https://youtu.be/7_WWz2DSnT8").getId()); + assertEquals("oy6NvWeVruY", urlIdHandler.setUrl("https://m.youtube.com/watch?v=oy6NvWeVruY").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); + assertEquals("EhxJLojIE_o", urlIdHandler.setUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube:jZViOEv90dI").getId()); } @Test public void getIdfromSharedLinksYt() throws Exception { String sharedId = "7JIArTByb3E"; String realId = "Q7JsK50NGaA"; - assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link")); - assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId)); - assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId)); + assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link").getId()); + assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId).getId()); + assertEquals(realId, urlIdHandler.setUrl("https://www.youtube.com/shared?ci=" + sharedId).getId()); } @Test - public void testAcceptYtUrl() { + public void testAcceptYtUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI")); assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100")); assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100")); @@ -111,7 +112,7 @@ public class YoutubeStreamUrlIdHandlerTest { } @Test - public void testAcceptSharedYtUrl() { + public void testAcceptSharedYtUrl() throws ParsingException { String sharedId = "8A940MXKFmQ"; assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link")); assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId)); @@ -119,7 +120,7 @@ public class YoutubeStreamUrlIdHandlerTest { } @Test - public void testAcceptHookUrl() { + public void testAcceptHookUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU")); assertTrue(urlIdHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE")); assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); @@ -130,11 +131,11 @@ public class YoutubeStreamUrlIdHandlerTest { @Test public void testGetHookIdfromUrl() throws ParsingException { - assertEquals("TglNG-yjabU", urlIdHandler.getId("https://hooktube.com/watch?v=TglNG-yjabU")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch?v=3msbfr6pBNE")); - assertEquals("ocH3oSnZG3c", urlIdHandler.getId("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch/3msbfr6pBNE")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/v/3msbfr6pBNE")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/embed/3msbfr6pBNE")); + assertEquals("TglNG-yjabU", urlIdHandler.setUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch?v=3msbfr6pBNE").getId()); + assertEquals("ocH3oSnZG3c", urlIdHandler.setUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch/3msbfr6pBNE").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/v/3msbfr6pBNE").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/embed/3msbfr6pBNE").getId()); } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index 59c84636..6b5f2875 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index ea8312a1..ae8c9f61 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -25,6 +25,8 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.Utils; @@ -48,6 +50,7 @@ public class YoutubeTrendingExtractorTest { .getKioskList() .getExtractorById("Trending", null); extractor.fetchPage(); + extractor.setContentCountry("de"); } @Test @@ -91,7 +94,7 @@ public class YoutubeTrendingExtractorTest { } @Test - public void testGetCleanUrl() { - assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending"); + public void testGetUrl() throws Exception { + assertEquals(extractor.getUrl(), extractor.getUrl(), "https://www.youtube.com/feed/trending"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java index 3fff4419..55f374a1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java @@ -45,7 +45,7 @@ public class YoutubeTrendingKioskInfoTest { StreamingService service = YouTube; UrlIdHandler urlIdHandler = service.getKioskList().getUrlIdHandlerByType("Trending"); - kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.getUrl("Trending"), null); + kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.setId("Trending").getUrl(), null); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java index 7ee68a39..315bdbfc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; @@ -46,13 +47,13 @@ public class YoutubeTrendingUrlIdHandlerTest { @Test public void getUrl() throws Exception { - assertEquals(urlIdHandler.getUrl(""), "https://www.youtube.com/feed/trending"); + assertEquals(urlIdHandler.setId("").getUrl(), "https://www.youtube.com/feed/trending"); } @Test public void getId() throws Exception { - assertEquals(urlIdHandler.getId(""), "Trending"); + assertEquals(urlIdHandler.setUrl("").getId(), "Trending"); } @Test