Created gradle module and moved existing code to new one

extractor/build.gradle

@@ -0,0 +1,10 @@
+dependencies {
+    implementation project(':timeago-parser')
+
+    implementation 'com.grack:nanojson:1.1'
+    implementation 'org.jsoup:jsoup:1.9.2'
+    implementation 'org.mozilla:rhino:1.7.7.1'
+    implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
+
+    testImplementation 'junit:junit:4.12'
+}

extractor/src/main/java/org/schabi/newpipe/extractor/Collector.java

@@ -0,0 +1,46 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.util.List;
+
+/**
+ * Collectors are used to simplify the collection of information
+ * from extractors
+ * @param <I> the item type
+ * @param <E> the extractor type
+ */
+public interface Collector<I, E> {
+
+    /**
+     * Try to add an extractor to the collection
+     * @param extractor the extractor to add
+     */
+    void commit(E extractor);
+
+    /**
+     * Try to extract the item from an extractor without adding it to the collection
+     * @param extractor the extractor to use
+     * @return the item
+     * @throws ParsingException thrown if there is an error extracting the
+     *                          <b>required</b> fields of the item.
+     */
+    I extract(E extractor) throws ParsingException;
+
+    /**
+     * Get all items
+     * @return the items
+     */
+    List<I> getItems();
+
+    /**
+     * Get all errors
+     * @return the errors
+     */
+    List<Throwable> getErrors();
+
+    /**
+     * Reset all collected items and errors
+     */
+    void reset();
+}

extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java

@@ -0,0 +1,61 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+
+import java.io.IOException;
+import java.util.Map;
+
+/*
+ * Created by Christian Schabesberger on 28.01.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * Downloader.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public interface Downloader {
+
+    /**
+     * Download the text file at the supplied URL as in download(String),
+     * but set the HTTP header field "Accept-Language" to the supplied string.
+     *
+     * @param siteUrl  the URL of the text file to return the contents of
+     * @param language the language (usually a 2-character code) to set as the preferred language
+     * @return the contents of the specified text file
+     * @throws IOException
+     */
+    String download(String siteUrl, String language) throws IOException, ReCaptchaException;
+
+    /**
+     * Download the text file at the supplied URL as in download(String),
+     * but set the HTTP header field "Accept-Language" to the supplied string.
+     *
+     * @param siteUrl          the URL of the text file to return the contents of
+     * @param customProperties set request header properties
+     * @return the contents of the specified text file
+     * @throws IOException
+     */
+    String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
+
+    /**
+     * Download (via HTTP) the text file located at the supplied URL, and return its contents.
+     * Primarily intended for downloading web pages.
+     *
+     * @param siteUrl the URL of the text file to download
+     * @return the contents of the specified text file
+     * @throws IOException
+     */
+    String download(String siteUrl) throws IOException, ReCaptchaException;
+}

extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -0,0 +1,123 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+
+public abstract class Extractor {
+    /**
+     * {@link StreamingService} currently related to this extractor.<br/>
+     * Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
+     */
+    private final StreamingService service;
+
+    /**
+     * Dirty/original url that was passed in the constructor.
+     * <p>
+     * What makes a url "dirty" or not is, for example, the additional parameters
+     * (not important as—in this case—the id):
+     * <pre>
+     *     https://www.youtube.com/watch?v=a9Zf_258aTI<i>&t=4s</i>  →  <i><b>&t=4s</b></i>
+     * </pre>
+     * But as you can imagine, the time parameter is very important when calling {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
+     */
+    private final String originalUrl;
+
+    /**
+     * The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}).
+     * <p>
+     * Is lazily-cleaned by calling {@link #getCleanUrl()}
+     */
+    @Nullable
+    private String cleanUrl;
+    private boolean pageFetched = false;
+    private final Downloader downloader;
+
+    public Extractor(final StreamingService service, final String url) {
+        if(service == null) throw new NullPointerException("service is null");
+        if(url == null) throw new NullPointerException("url is null");
+        this.service = service;
+        this.originalUrl = url;
+        this.downloader = NewPipe.getDownloader();
+        if(downloader == null) throw new NullPointerException("downloader is null");
+    }
+
+    /**
+     * @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler).
+     */
+    @Nonnull
+    protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException;
+
+    /**
+     * Fetch the current page.
+     * @throws IOException if the page can not be loaded
+     * @throws ExtractionException if the pages content is not understood
+     */
+    public void fetchPage() throws IOException, ExtractionException {
+        if(pageFetched) return;
+        onFetchPage(downloader);
+        pageFetched = true;
+    }
+
+    protected void assertPageFetched() {
+        if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
+    }
+
+    protected boolean isPageFetched() {
+        return pageFetched;
+    }
+
+    /**
+     * Fetch the current page.
+     * @param downloader the download to use
+     * @throws IOException if the page can not be loaded
+     * @throws ExtractionException if the pages content is not understood
+     */
+    public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
+
+    @Nonnull
+    public abstract String getId() throws ParsingException;
+
+    /**
+     * Get the name
+     * @return the name
+     * @throws ParsingException if the name cannot be extracted
+     */
+    @Nonnull
+    public abstract String getName() throws ParsingException;
+
+    @Nonnull
+    public String getOriginalUrl() {
+        return originalUrl;
+    }
+
+    /**
+     * Get a clean url and as a fallback the original url.
+     * @return the clean url or the original url
+     */
+    @Nonnull
+    public String getCleanUrl() {
+        if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;
+
+        try {
+            cleanUrl = getUrlIdHandler().cleanUrl(originalUrl);
+        } catch (Exception e) {
+            cleanUrl = null;
+            // Fallback to the original url
+            return originalUrl;
+        }
+        return cleanUrl;
+    }
+
+    @Nonnull
+    public StreamingService getService() {
+        return service;
+    }
+
+    public int getServiceId() {
+        return service.getServiceId();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/Info.java

@@ -0,0 +1,60 @@
+package org.schabi.newpipe.extractor;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public abstract class Info implements Serializable {
+
+    private final int serviceId;
+    /**
+     * Id of this Info object <br>
+     * e.g. Youtube:  https://www.youtube.com/watch?v=RER5qCTzZ7     &gt;    RER5qCTzZ7
+     */
+    private final String id;
+    private final String url;
+    private final String name;
+
+    private final List<Throwable> errors = new ArrayList<>();
+
+    public void addError(Throwable throwable) {
+        this.errors.add(throwable);
+    }
+
+    public void addAllErrors(Collection<Throwable> errors) {
+        this.errors.addAll(errors);
+    }
+
+    public Info(int serviceId, String id, String url, String name) {
+        this.serviceId = serviceId;
+        this.id = id;
+        this.url = url;
+        this.name = name;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "[url=\"" + url + "\", name=\"" + name + "\"]";
+    }
+
+    public int getServiceId() {
+        return serviceId;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public List<Throwable> getErrors() {
+        return errors;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java

@@ -0,0 +1,73 @@
+package org.schabi.newpipe.extractor;
+
+/*
+ * Created by Christian Schabesberger on 11.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * InfoItem.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import java.io.Serializable;
+
+public abstract class InfoItem implements Serializable {
+    private final InfoType infoType;
+    private final int serviceId;
+    private final String url;
+    private final String name;
+    private String thumbnailUrl;
+
+    public InfoItem(InfoType infoType, int serviceId, String url, String name) {
+        this.infoType = infoType;
+        this.serviceId = serviceId;
+        this.url = url;
+        this.name = name;
+    }
+
+    public InfoType getInfoType() {
+        return infoType;
+    }
+
+    public int getServiceId() {
+        return serviceId;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setThumbnailUrl(String thumbnailUrl) {
+        this.thumbnailUrl = thumbnailUrl;
+    }
+
+    public String getThumbnailUrl() {
+        return thumbnailUrl;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "[url=\"" + url + "\", name=\"" + name + "\"]";
+    }
+
+    public enum InfoType {
+        STREAM,
+        PLAYLIST,
+        CHANNEL
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java

@@ -0,0 +1,9 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+public interface InfoItemExtractor {
+    String getName() throws ParsingException;
+    String getUrl() throws ParsingException;
+    String getThumbnailUrl() throws ParsingException;
+}

extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java

@@ -0,0 +1,94 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.FoundAdException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/*
+ * Created by Christian Schabesberger on 12.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * InfoItemsCollector.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public abstract class InfoItemsCollector<I extends InfoItem, E> implements Collector<I,E> {
+
+    private final List<I> itemList = new ArrayList<>();
+    private final List<Throwable> errors = new ArrayList<>();
+    private final int serviceId;
+
+    /**
+     * Create a new collector
+     * @param serviceId the service id
+     */
+    public InfoItemsCollector(int serviceId) {
+        this.serviceId = serviceId;
+    }
+
+    @Override
+    public List<I> getItems() {
+        return Collections.unmodifiableList(itemList);
+    }
+
+    @Override
+    public List<Throwable> getErrors() {
+        return Collections.unmodifiableList(errors);
+    }
+
+    @Override
+    public void reset() {
+        itemList.clear();
+        errors.clear();
+    }
+
+    /**
+     * Add an error
+     * @param error the error
+     */
+    protected void addError(Exception error) {
+        errors.add(error);
+    }
+
+    /**
+     * Add an item
+     * @param item the item
+     */
+    protected void addItem(I item) {
+        itemList.add(item);
+    }
+
+    /**
+     * Get the service id
+     * @return the service id
+     */
+    public int getServiceId() {
+        return serviceId;
+    }
+
+    @Override
+    public void commit(E extractor) {
+        try {
+            addItem(extract(extractor));
+        } catch (FoundAdException ae) {
+            // found an ad. Maybe a debug line could be placed here
+        } catch (ParsingException e) {
+            addError(e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java

@@ -0,0 +1,119 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Base class to extractors that have a list (e.g. playlists, users).
+ */
+public abstract class ListExtractor<R extends InfoItem> extends Extractor {
+
+    public ListExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    /**
+     * A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page where the items are from the initial request and
+     * the nextPageUrl relative to it.
+     *
+     * @return a {@link InfoItemsPage} corresponding to the initial page
+     */
+    @Nonnull
+    public abstract InfoItemsPage<R> getInitialPage() throws IOException, ExtractionException;
+
+    /**
+     * Returns an url that can be used to get the next page relative to the initial one.<br/>
+     * <p>Usually, these links will only work in the implementation itself.</p>
+     *
+     * @return an url pointing to the next page relative to the initial page
+     * @see #getPage(String)
+     */
+    public abstract String getNextPageUrl() throws IOException, ExtractionException;
+
+    /**
+     * Get a list of items corresponding to the specific requested page.
+     *
+     * @param nextPageUrl any next page url got from the exclusive implementation of the list extractor
+     * @return a {@link InfoItemsPage} corresponding to the requested page
+     * @see #getNextPageUrl()
+     * @see InfoItemsPage#getNextPageUrl()
+     */
+    public abstract InfoItemsPage<R> getPage(final String nextPageUrl) throws IOException, ExtractionException;
+
+    public boolean hasNextPage() throws IOException, ExtractionException {
+        final String nextPageUrl = getNextPageUrl();
+        return nextPageUrl != null && !nextPageUrl.isEmpty();
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Inner
+    //////////////////////////////////////////////////////////////////////////*/
+
+    /**
+     * A class that is used to wrap a list of gathered items and eventual errors, it
+     * also contains a field that points to the next available page ({@link #nextPageUrl}).
+     */
+    public static class InfoItemsPage<T extends InfoItem> {
+        private static final InfoItemsPage<InfoItem> EMPTY =
+                new InfoItemsPage<>(Collections.<InfoItem>emptyList(), "", Collections.<Throwable>emptyList());
+
+        /**
+         * A convenient method that returns a representation of an empty page.
+         *
+         * @return a type-safe page with the list of items and errors empty and the nextPageUrl set to an empty string.
+         */
+        public static <T extends InfoItem> InfoItemsPage<T> emptyPage() {
+            //noinspection unchecked
+            return (InfoItemsPage<T>) EMPTY;
+        }
+
+
+        /**
+         * The current list of items of this page
+         */
+        private final List<T> itemsList;
+
+        /**
+         * Url pointing to the next page relative to this one
+         *
+         * @see ListExtractor#getPage(String)
+         */
+        private final String nextPageUrl;
+
+        /**
+         * Errors that happened during the extraction
+         */
+        private final List<Throwable> errors;
+
+        public InfoItemsPage(InfoItemsCollector<T, ?> collector, String nextPageUrl) {
+            this(collector.getItems(), nextPageUrl, collector.getErrors());
+        }
+
+        public InfoItemsPage(List<T> itemsList, String nextPageUrl, List<Throwable> errors) {
+            this.itemsList = itemsList;
+            this.nextPageUrl = nextPageUrl;
+            this.errors = errors;
+        }
+
+        public boolean hasNextPage() {
+            return nextPageUrl != null && !nextPageUrl.isEmpty();
+        }
+
+        public List<T> getItems() {
+            return itemsList;
+        }
+
+        public String getNextPageUrl() {
+            return nextPageUrl;
+        }
+
+        public List<Throwable> getErrors() {
+            return errors;
+        }
+    }
+
+}

extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java

@@ -0,0 +1,32 @@
+package org.schabi.newpipe.extractor;
+
+import java.util.List;
+
+public abstract class ListInfo<T extends InfoItem> extends Info {
+    private List<T> relatedItems;
+    private String nextPageUrl = null;
+
+    public ListInfo(int serviceId, String id, String url, String name) {
+        super(serviceId, id, url, name);
+    }
+
+    public List<T> getRelatedItems() {
+        return relatedItems;
+    }
+
+    public void setRelatedItems(List<T> relatedItems) {
+        this.relatedItems = relatedItems;
+    }
+
+    public boolean hasNextPage() {
+        return nextPageUrl != null && !nextPageUrl.isEmpty();
+    }
+
+    public String getNextPageUrl() {
+        return nextPageUrl;
+    }
+
+    public void setNextPageUrl(String pageUrl) {
+        this.nextPageUrl = pageUrl;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java

@@ -0,0 +1,142 @@
+package org.schabi.newpipe.extractor;
+
+/*
+ * Created by Adam Howard on 08/11/15.
+ *
+ * Copyright (c) Christian Schabesberger <chris.schabesberger@mailbox.org>
+ *     and Adam Howard <achdisposable1@gmail.com> 2015
+ *
+ * MediaFormat.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Static data about various media formats support by NewPipe, eg mime type, extension
+ */
+
+public enum MediaFormat {
+    //video and audio combined formats
+    //           id      name    suffix  mime type
+    MPEG_4      (0x0,   "MPEG-4", "mp4", "video/mp4"),
+    v3GPP       (0x1,   "3GPP",   "3gp", "video/3gpp"),
+    WEBM        (0x2,   "WebM",  "webm", "video/webm"),
+    // audio formats
+    M4A         (0x3,   "m4a",   "m4a",  "audio/mp4"),
+    WEBMA       (0x4,   "WebM",  "webm", "audio/webm"),
+    MP3         (0x5,   "MP3",   "mp3",  "audio/mpeg");
+
+    public final int id;
+    public final String name;
+    public final String suffix;
+    public final String mimeType;
+
+    MediaFormat(int id, String name, String suffix, String mimeType) {
+        this.id = id;
+        this.name = name;
+        this.suffix = suffix;
+        this.mimeType = mimeType;
+    }
+
+    /**
+     * Return the friendly name of the media format with the supplied id
+     *
+     * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
+     * @return the friendly name of the MediaFormat associated with this ids,
+     * or an empty String if none match it.
+     */
+    public static String getNameById(int ident) {
+        for (MediaFormat vf : MediaFormat.values()) {
+            if (vf.id == ident) return vf.name;
+        }
+        return "";
+    }
+
+    /**
+     * Return the file extension of the media format with the supplied id
+     *
+     * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
+     * @return the file extension of the MediaFormat associated with this ids,
+     * or an empty String if none match it.
+     */
+    public static String getSuffixById(int ident) {
+        for (MediaFormat vf : MediaFormat.values()) {
+            if (vf.id == ident) return vf.suffix;
+        }
+        return "";
+    }
+
+    /**
+     * Return the MIME type of the media format with the supplied id
+     *
+     * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
+     * @return the MIME type of the MediaFormat associated with this ids,
+     * or an empty String if none match it.
+     */
+    public static String getMimeById(int ident) {
+        for (MediaFormat vf : MediaFormat.values()) {
+            if (vf.id == ident) return vf.mimeType;
+        }
+        return "";
+    }
+
+    /**
+     * Return the MediaFormat with the supplied mime type
+     *
+     * @return MediaFormat associated with this mime type,
+     * or null if none match it.
+     */
+    public static MediaFormat getFromMimeType(String mimeType) {
+        for (MediaFormat vf : MediaFormat.values()) {
+            if (vf.mimeType.equals(mimeType)) return vf;
+        }
+        return null;
+    }
+
+    /**
+     * Get the media format by it's id.
+     * @param id the id
+     * @return the id of the media format or null.
+     */
+    public static MediaFormat getFormatById(int id) {
+        for (MediaFormat vf: values()) {
+            if (vf.id == id) return vf;
+        }
+        return null;
+    }
+
+    /**
+     * Get the name of the format
+     * @return the name of the format
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Get the filename extension
+     * @return the filename extension
+     */
+    public String getSuffix() {
+        return suffix;
+    }
+
+    /**
+     * Get the mime type
+     * @return the mime type
+     */
+    public String getMimeType() {
+        return mimeType;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java

@@ -0,0 +1,98 @@
+package org.schabi.newpipe.extractor;
+
+/*
+ * Created by Christian Schabesberger on 23.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * NewPipe.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.util.List;
+
+/**
+ * Provides access to streaming services supported by NewPipe.
+ */
+public class NewPipe {
+    private static Downloader downloader = null;
+
+    private NewPipe() {
+    }
+
+    public static void init(Downloader d) {
+        downloader = d;
+    }
+
+    public static Downloader getDownloader() {
+        return downloader;
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Utils
+    //////////////////////////////////////////////////////////////////////////*/
+
+    public static List<StreamingService> getServices() {
+        return ServiceList.all();
+    }
+
+    public static StreamingService getService(int serviceId) throws ExtractionException {
+        for (StreamingService service : ServiceList.all()) {
+            if (service.getServiceId() == serviceId) {
+                return service;
+            }
+        }
+        throw new ExtractionException("There's no service with the id = \"" + serviceId + "\"");
+    }
+
+    public static StreamingService getService(String serviceName) throws ExtractionException {
+        for (StreamingService service : ServiceList.all()) {
+            if (service.getServiceInfo().getName().equals(serviceName)) {
+                return service;
+            }
+        }
+        throw new ExtractionException("There's no service with the name = \"" + serviceName + "\"");
+    }
+
+    public static StreamingService getServiceByUrl(String url) throws ExtractionException {
+        for (StreamingService service : ServiceList.all()) {
+            if (service.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
+                return service;
+            }
+        }
+        throw new ExtractionException("No service can handle the url = \"" + url + "\"");
+    }
+
+    public static int getIdOfService(String serviceName) {
+        try {
+            //noinspection ConstantConditions
+            return getService(serviceName).getServiceId();
+        } catch (ExtractionException ignored) {
+            return -1;
+        }
+    }
+
+    public static String getNameOfService(int id) {
+        try {
+            //noinspection ConstantConditions
+            return getService(id).getServiceInfo().getName();
+        } catch (Exception e) {
+            System.err.println("Service id not known");
+            e.printStackTrace();
+            return "<unknown>";
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java

@@ -0,0 +1,36 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
+
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+
+/**
+ * A list of supported services.
+ */
+public final class ServiceList {
+    private ServiceList() {
+        //no instance
+    }
+
+    public static final YoutubeService YouTube;
+    public static final SoundcloudService SoundCloud;
+
+    private static final List<StreamingService> SERVICES = unmodifiableList(asList(
+            YouTube = new YoutubeService(0),
+            SoundCloud = new SoundcloudService(1)
+            // DailyMotion = new DailyMotionService(2);
+    ));
+
+    /**
+     * Get all the supported services.
+     *
+     * @return a unmodifiable list of all the supported services
+     */
+    public static List<StreamingService> all() {
+        return SERVICES;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -0,0 +1,95 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.kiosk.KioskList;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
+
+import java.util.Collections;
+import java.util.List;
+
+public abstract class StreamingService {
+    public static class ServiceInfo {
+        private final String name;
+        private final List<MediaCapability> mediaCapabilities;
+
+        public ServiceInfo(String name, List<MediaCapability> mediaCapabilities) {
+            this.name = name;
+            this.mediaCapabilities = Collections.unmodifiableList(mediaCapabilities);
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public List<MediaCapability> getMediaCapabilities() {
+            return mediaCapabilities;
+        }
+
+        public enum MediaCapability {
+            AUDIO, VIDEO, LIVE
+        }
+    }
+
+    public enum LinkType {
+        NONE,
+        STREAM,
+        CHANNEL,
+        PLAYLIST
+    }
+
+    private final int serviceId;
+    private final ServiceInfo serviceInfo;
+
+    public StreamingService(int id, String name, List<ServiceInfo.MediaCapability> capabilities) {
+        this.serviceId = id;
+        this.serviceInfo = new ServiceInfo(name, capabilities);
+    }
+
+    public final int getServiceId() {
+        return serviceId;
+    }
+
+    public ServiceInfo getServiceInfo() {
+        return serviceInfo;
+    }
+
+    @Override
+    public String toString() {
+        return serviceId + ":" + serviceInfo.getName();
+    }
+
+    public abstract UrlIdHandler getStreamUrlIdHandler();
+    public abstract UrlIdHandler getChannelUrlIdHandler();
+    public abstract UrlIdHandler getPlaylistUrlIdHandler();
+
+    public abstract SearchEngine getSearchEngine();
+    public abstract SuggestionExtractor getSuggestionExtractor();
+    public abstract StreamExtractor getStreamExtractor(String url);
+    public abstract KioskList getKioskList() throws ExtractionException;
+    public abstract ChannelExtractor getChannelExtractor(String url);
+    public abstract PlaylistExtractor getPlaylistExtractor(String url);
+    public abstract SubscriptionExtractor getSubscriptionExtractor();
+
+    /**
+     * figure out where the link is pointing to (a channel, video, playlist, etc.)
+     */
+    public final LinkType getLinkTypeByUrl(String url) {
+        UrlIdHandler sH = getStreamUrlIdHandler();
+        UrlIdHandler cH = getChannelUrlIdHandler();
+        UrlIdHandler pH = getPlaylistUrlIdHandler();
+
+        if (sH.acceptUrl(url)) {
+            return LinkType.STREAM;
+        } else if (cH.acceptUrl(url)) {
+            return LinkType.CHANNEL;
+        } else if (pH.acceptUrl(url)) {
+            return LinkType.PLAYLIST;
+        } else {
+            return LinkType.NONE;
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java

@@ -0,0 +1,34 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
+
+import java.io.Serializable;
+import java.util.Locale;
+
+public class Subtitles implements Serializable {
+    private final SubtitlesFormat format;
+    private final Locale locale;
+    private final String URL;
+    private final boolean autoGenerated;
+
+    public Subtitles(SubtitlesFormat format, Locale locale, String URL, boolean autoGenerated) {
+        this.format = format;
+        this.locale = locale;
+        this.URL = URL;
+        this.autoGenerated = autoGenerated;
+    }
+
+    public SubtitlesFormat getFileType() { return format; }
+
+    public Locale getLocale() {
+        return locale;
+    }
+
+    public String getURL() {
+        return URL;
+    }
+
+    public boolean isAutoGenerated() {
+        return autoGenerated;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java

@@ -0,0 +1,41 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.io.IOException;
+import java.util.List;
+
+/*
+ * Created by Christian Schabesberger on 28.09.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * SuggestionExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public abstract class SuggestionExtractor {
+
+    private final int serviceId;
+
+    public SuggestionExtractor(int serviceId) {
+        this.serviceId = serviceId;
+    }
+
+    public abstract List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException;
+
+    public int getServiceId() {
+        return serviceId;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java

@@ -0,0 +1,37 @@
+package org.schabi.newpipe.extractor;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+/*
+ * Created by Christian Schabesberger on 26.07.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * UrlIdHandler.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public interface UrlIdHandler {
+
+    String getUrl(String id) throws ParsingException;
+    String getId(String url) throws ParsingException;
+    String cleanUrl(String complexUrl) throws ParsingException;
+
+    /**
+     * When a VIEW_ACTION is caught this function will test if the url delivered within the calling
+     * Intent was meant to be watched with this Service.
+     * Return false if this service shall not allow to be called through ACTIONs.
+     */
+    boolean acceptUrl(String url);
+}

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java

@@ -0,0 +1,48 @@
+package org.schabi.newpipe.extractor.channel;
+
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+
+import javax.annotation.Nonnull;
+
+/*
+ * Created by Christian Schabesberger on 25.07.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * ChannelExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
+
+    public ChannelExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Nonnull
+    @Override
+    protected UrlIdHandler getUrlIdHandler() {
+        return getService().getChannelUrlIdHandler();
+    }
+
+    public abstract String getAvatarUrl() throws ParsingException;
+    public abstract String getBannerUrl() throws ParsingException;
+    public abstract String getFeedUrl() throws ParsingException;
+    public abstract long getSubscriberCount() throws ParsingException;
+    public abstract String getDescription() throws ParsingException;
+}

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java

@@ -0,0 +1,143 @@
+package org.schabi.newpipe.extractor.channel;
+
+import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
+import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.utils.ExtractorHelper;
+
+import java.io.IOException;
+
+/*
+ * Created by Christian Schabesberger on 31.07.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * ChannelInfo.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class ChannelInfo extends ListInfo<StreamInfoItem> {
+
+    public ChannelInfo(int serviceId, String url, String id, String name) {
+        super(serviceId, id, url, name);
+    }
+
+    public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url);
+    }
+
+    public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
+        ChannelExtractor extractor = service.getChannelExtractor(url);
+        extractor.fetchPage();
+        return getInfo(extractor);
+    }
+
+    public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException {
+        return service.getChannelExtractor(url).getPage(pageUrl);
+    }
+
+    public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
+
+        // important data
+        int serviceId = extractor.getServiceId();
+        String url = extractor.getCleanUrl();
+        String id = extractor.getId();
+        String name = extractor.getName();
+
+        ChannelInfo info = new ChannelInfo(serviceId, url, id, name);
+
+
+        try {
+            info.setAvatarUrl(extractor.getAvatarUrl());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setBannerUrl(extractor.getBannerUrl());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setFeedUrl(extractor.getFeedUrl());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+
+        final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
+        info.setRelatedItems(itemsPage.getItems());
+        info.setNextPageUrl(itemsPage.getNextPageUrl());
+
+        try {
+            info.setSubscriberCount(extractor.getSubscriberCount());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setDescription(extractor.getDescription());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+
+        return info;
+    }
+
+    private String avatarUrl;
+    private String bannerUrl;
+    private String feedUrl;
+    private long subscriberCount = -1;
+    private String description;
+
+    public String getAvatarUrl() {
+        return avatarUrl;
+    }
+
+    public void setAvatarUrl(String avatarUrl) {
+        this.avatarUrl = avatarUrl;
+    }
+
+    public String getBannerUrl() {
+        return bannerUrl;
+    }
+
+    public void setBannerUrl(String bannerUrl) {
+        this.bannerUrl = bannerUrl;
+    }
+
+    public String getFeedUrl() {
+        return feedUrl;
+    }
+
+    public void setFeedUrl(String feedUrl) {
+        this.feedUrl = feedUrl;
+    }
+
+    public long getSubscriberCount() {
+        return subscriberCount;
+    }
+
+    public void setSubscriberCount(long subscriberCount) {
+        this.subscriberCount = subscriberCount;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java

@@ -0,0 +1,59 @@
+package org.schabi.newpipe.extractor.channel;
+
+import org.schabi.newpipe.extractor.InfoItem;
+
+/*
+ * Created by Christian Schabesberger on 11.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * ChannelInfoItem.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class ChannelInfoItem extends InfoItem {
+
+    private String description;
+    private long subscriberCount = -1;
+    private long streamCount = -1;
+
+
+    public ChannelInfoItem(int serviceId, String url, String name) {
+        super(InfoType.CHANNEL, serviceId, url, name);
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public long getSubscriberCount() {
+        return subscriberCount;
+    }
+
+    public void setSubscriberCount(long subscriber_count) {
+        this.subscriberCount = subscriber_count;
+    }
+
+    public long getStreamCount() {
+        return streamCount;
+    }
+
+    public void setStreamCount(long stream_count) {
+        this.streamCount = stream_count;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.channel;
+
+import org.schabi.newpipe.extractor.InfoItemExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+/*
+ * Created by Christian Schabesberger on 12.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * ChannelInfoItemExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public interface ChannelInfoItemExtractor extends InfoItemExtractor {
+    String getDescription() throws ParsingException;
+
+    long getSubscriberCount() throws ParsingException;
+    long getStreamCount() throws ParsingException;
+}

extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java

@@ -0,0 +1,64 @@
+package org.schabi.newpipe.extractor.channel;
+
+import org.schabi.newpipe.extractor.InfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+/*
+ * Created by Christian Schabesberger on 12.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * ChannelInfoItemsCollector.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class ChannelInfoItemsCollector extends InfoItemsCollector<ChannelInfoItem, ChannelInfoItemExtractor> {
+    public ChannelInfoItemsCollector(int serviceId) {
+        super(serviceId);
+    }
+
+    @Override
+    public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
+        // important information
+        int serviceId = getServiceId();
+        String name = extractor.getName();
+        String  url = extractor.getUrl();
+
+        ChannelInfoItem resultItem = new ChannelInfoItem(serviceId, url, name);
+
+
+        // optional information
+        try {
+            resultItem.setSubscriberCount(extractor.getSubscriberCount());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setStreamCount(extractor.getStreamCount());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setThumbnailUrl(extractor.getThumbnailUrl());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setDescription(extractor.getDescription());
+        } catch (Exception e) {
+            addError(e);
+        }
+        return resultItem;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ContentNotAvailableException.java

@@ -0,0 +1,11 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+public class ContentNotAvailableException extends ParsingException {
+    public ContentNotAvailableException(String message) {
+        super(message);
+    }
+
+    public ContentNotAvailableException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ExtractionException.java

@@ -0,0 +1,35 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+/*
+ * Created by Christian Schabesberger on 30.01.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * ExtractionException.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class ExtractionException extends Exception {
+    public ExtractionException(String message) {
+        super(message);
+    }
+
+    public ExtractionException(Throwable cause) {
+        super(cause);
+    }
+
+    public ExtractionException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/FoundAdException.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+/*
+ * Created by Christian Schabesberger on 12.09.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * FoundAdException.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class FoundAdException extends ParsingException {
+    public FoundAdException(String message) {
+        super(message);
+    }
+
+    public FoundAdException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ParsingException.java

@@ -0,0 +1,32 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+/*
+ * Created by Christian Schabesberger on 31.01.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * ParsingException.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+public class ParsingException extends ExtractionException {
+    public ParsingException(String message) {
+        super(message);
+    }
+
+    public ParsingException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java

@@ -0,0 +1,27 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+/*
+ * Created by beneth <bmauduit@beneth.fr> on 07.12.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * ReCaptchaException.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class ReCaptchaException extends ExtractionException {
+    public ReCaptchaException(String message) {
+        super(message);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java

@@ -0,0 +1,76 @@
+package org.schabi.newpipe.extractor.kiosk;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * KioskExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+
+import javax.annotation.Nonnull;
+
+public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
+    private String contentCountry = null;
+    private final String id;
+
+    public KioskExtractor(StreamingService streamingService,
+                          String url,
+                          String kioskId)
+        throws ExtractionException {
+        super(streamingService, url);
+        this.id = kioskId;
+    }
+
+    /**
+     * For certain Websites the content of a kiosk will be different depending
+     * on the country you want to poen the website in. Therefore you should
+     * set the contentCountry.
+     * @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm
+     */
+    public void setContentCountry(String contentCountry) {
+        this.contentCountry = contentCountry;
+    }
+
+
+    @Nonnull
+    @Override
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Id should be the name of the kiosk, tho Id is used for identifing it in the frontend,
+     * so id should be kept in english.
+     * In order to get the name of the kiosk in the desired language we have to
+     * crawl if from the website.
+     * @return the tranlsated version of id
+     * @throws ParsingException
+     */
+    @Nonnull
+    @Override
+    public abstract String getName() throws ParsingException;
+
+
+    public String getContentCountry() {
+        return contentCountry;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java

@@ -0,0 +1,84 @@
+package org.schabi.newpipe.extractor.kiosk;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * KioskInfo.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.utils.ExtractorHelper;
+
+import java.io.IOException;
+
+public class KioskInfo extends ListInfo<StreamInfoItem> {
+
+    private KioskInfo(int serviceId, String id, String url, String name) {
+        super(serviceId, id, url, name);
+    }
+
+    public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
+                                                                           String url,
+                                                                           String pageUrl,
+                                                                           String contentCountry) throws IOException, ExtractionException {
+        KioskList kl = service.getKioskList();
+        KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
+        extractor.setContentCountry(contentCountry);
+        return extractor.getPage(pageUrl);
+    }
+
+    public static KioskInfo getInfo(String url,
+                                    String contentCountry) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry);
+    }
+
+    public static KioskInfo getInfo(StreamingService service,
+                                    String url,
+                                    String contentCountry) throws IOException, ExtractionException {
+        KioskList kl = service.getKioskList();
+        KioskExtractor extractor = kl.getExtractorByUrl(url, null);
+        extractor.setContentCountry(contentCountry);
+        extractor.fetchPage();
+        return getInfo(extractor);
+    }
+
+    /**
+     * Get KioskInfo from KioskExtractor
+     *
+     * @param extractor an extractor where fetchPage() was already got called on.
+     */
+    public static KioskInfo getInfo(KioskExtractor extractor) throws ExtractionException {
+
+        int serviceId = extractor.getServiceId();
+        String name = extractor.getName();
+        String id = extractor.getId();
+        String url = extractor.getCleanUrl();
+
+        KioskInfo info = new KioskInfo(serviceId, id, name, url);
+
+        final ListExtractor.InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
+        info.setRelatedItems(itemsPage.getItems());
+        info.setNextPageUrl(itemsPage.getNextPageUrl());
+
+        return info;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java

@@ -0,0 +1,98 @@
+package org.schabi.newpipe.extractor.kiosk;
+
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public  class KioskList {
+    public interface KioskExtractorFactory {
+        KioskExtractor createNewKiosk(final StreamingService streamingService,
+                                             final String url,
+                                             final String kioskId)
+            throws ExtractionException, IOException;
+    }
+
+    private final int service_id;
+    private final HashMap<String, KioskEntry> kioskList = new HashMap<>();
+    private String defaultKiosk = null;
+
+    private class KioskEntry {
+        public KioskEntry(KioskExtractorFactory ef, UrlIdHandler h) {
+            extractorFactory = ef;
+            handler = h;
+        }
+        final KioskExtractorFactory extractorFactory;
+        final UrlIdHandler handler;
+    }
+
+    public KioskList(int service_id) {
+        this.service_id = service_id;
+    }
+
+    public void addKioskEntry(KioskExtractorFactory extractorFactory, UrlIdHandler handler, String id)
+        throws Exception {
+        if(kioskList.get(id) != null) {
+            throw new Exception("Kiosk with type " + id + " already exists.");
+        }
+        kioskList.put(id, new KioskEntry(extractorFactory, handler));
+    }
+
+    public void setDefaultKiosk(String kioskType) {
+        defaultKiosk = kioskType;
+    }
+
+    public KioskExtractor getDefaultKioskExtractor(String nextPageUrl)
+            throws ExtractionException, IOException {
+        if(defaultKiosk != null && !defaultKiosk.equals("")) {
+            return getExtractorById(defaultKiosk, nextPageUrl);
+        } else {
+            if(!kioskList.isEmpty()) {
+                // if not set get any entry
+                Object[] keySet = kioskList.keySet().toArray();
+                return getExtractorById(keySet[0].toString(), nextPageUrl);
+            } else {
+                return null;
+            }
+        }
+    }
+
+    public String getDefaultKioskId() {
+        return defaultKiosk;
+    }
+
+    public KioskExtractor getExtractorById(String kioskId, String nextPageUrl)
+            throws ExtractionException, IOException {
+        KioskEntry ke = kioskList.get(kioskId);
+        if(ke == null) {
+            throw new ExtractionException("No kiosk found with the type: " + kioskId);
+        } else {
+            return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
+                    ke.handler.getUrl(kioskId), kioskId);
+        }
+    }
+
+    public Set<String> getAvailableKiosks() {
+        return kioskList.keySet();
+    }
+
+    public KioskExtractor getExtractorByUrl(String url, String nextPageUrl)
+            throws ExtractionException, IOException {
+        for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
+            KioskEntry ke = e.getValue();
+            if(ke.handler.acceptUrl(url)) {
+                return getExtractorById(e.getKey(), nextPageUrl);
+            }
+        }
+        throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);
+    }
+
+    public UrlIdHandler getUrlIdHandlerByType(String type) {
+        return kioskList.get(type).handler;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.playlist;
+
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+
+import javax.annotation.Nonnull;
+
+public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
+
+    public PlaylistExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Nonnull
+    @Override
+    protected UrlIdHandler getUrlIdHandler() {
+        return getService().getPlaylistUrlIdHandler();
+    }
+
+    public abstract String getThumbnailUrl() throws ParsingException;
+    public abstract String getBannerUrl() throws ParsingException;
+
+    public abstract String getUploaderUrl() throws ParsingException;
+    public abstract String getUploaderName() throws ParsingException;
+    public abstract String getUploaderAvatarUrl() throws ParsingException;
+
+    public abstract long getStreamCount() throws ParsingException;
+}

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java

@@ -0,0 +1,138 @@
+package org.schabi.newpipe.extractor.playlist;
+
+import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
+import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.utils.ExtractorHelper;
+
+import java.io.IOException;
+
+public class PlaylistInfo extends ListInfo<StreamInfoItem> {
+
+    public PlaylistInfo(int serviceId, String id, String url, String name) {
+        super(serviceId, id, url, name);
+    }
+
+    public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url);
+    }
+
+    public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
+        PlaylistExtractor extractor = service.getPlaylistExtractor(url);
+        extractor.fetchPage();
+        return getInfo(extractor);
+    }
+
+    public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException {
+        return service.getPlaylistExtractor(url).getPage(pageUrl);
+    }
+
+    /**
+     * Get PlaylistInfo from PlaylistExtractor
+     *
+     * @param extractor an extractor where fetchPage() was already got called on.
+     */
+    public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException {
+
+        int serviceId = extractor.getServiceId();
+        String url = extractor.getCleanUrl();
+        String id = extractor.getId();
+        String name = extractor.getName();
+        PlaylistInfo info = new PlaylistInfo(serviceId, id, url, name);
+
+        try {
+            info.setStreamCount(extractor.getStreamCount());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setThumbnailUrl(extractor.getThumbnailUrl());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setUploaderUrl(extractor.getUploaderUrl());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setUploaderName(extractor.getUploaderName());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setBannerUrl(extractor.getBannerUrl());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+
+        final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
+        info.setRelatedItems(itemsPage.getItems());
+        info.setNextPageUrl(itemsPage.getNextPageUrl());
+
+        return info;
+    }
+
+    private String thumbnailUrl;
+    private String bannerUrl;
+    private String uploaderUrl;
+    private String uploaderName;
+    private String uploaderAvatarUrl;
+    private long streamCount = 0;
+
+    public String getThumbnailUrl() {
+        return thumbnailUrl;
+    }
+
+    public void setThumbnailUrl(String thumbnailUrl) {
+        this.thumbnailUrl = thumbnailUrl;
+    }
+
+    public String getBannerUrl() {
+        return bannerUrl;
+    }
+
+    public void setBannerUrl(String bannerUrl) {
+        this.bannerUrl = bannerUrl;
+    }
+
+    public String getUploaderUrl() {
+        return uploaderUrl;
+    }
+
+    public void setUploaderUrl(String uploaderUrl) {
+        this.uploaderUrl = uploaderUrl;
+    }
+
+    public String getUploaderName() {
+        return uploaderName;
+    }
+
+    public void setUploaderName(String uploaderName) {
+        this.uploaderName = uploaderName;
+    }
+
+    public String getUploaderAvatarUrl() {
+        return uploaderAvatarUrl;
+    }
+
+    public void setUploaderAvatarUrl(String uploaderAvatarUrl) {
+        this.uploaderAvatarUrl = uploaderAvatarUrl;
+    }
+
+    public long getStreamCount() {
+        return streamCount;
+    }
+
+    public void setStreamCount(long streamCount) {
+        this.streamCount = streamCount;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java

@@ -0,0 +1,32 @@
+package org.schabi.newpipe.extractor.playlist;
+
+import org.schabi.newpipe.extractor.InfoItem;
+
+public class PlaylistInfoItem extends InfoItem {
+
+    private String uploaderName;
+    /**
+     * How many streams this playlist have
+     */
+    private long streamCount = 0;
+
+    public PlaylistInfoItem(int serviceId, String url, String name) {
+        super(InfoType.PLAYLIST, serviceId, url, name);
+    }
+
+    public String getUploaderName() {
+        return uploaderName;
+    }
+
+    public void setUploaderName(String uploader_name) {
+        this.uploaderName = uploader_name;
+    }
+
+    public long getStreamCount() {
+        return streamCount;
+    }
+
+    public void setStreamCount(long stream_count) {
+        this.streamCount = stream_count;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java

@@ -0,0 +1,21 @@
+package org.schabi.newpipe.extractor.playlist;
+
+import org.schabi.newpipe.extractor.InfoItemExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+public interface PlaylistInfoItemExtractor extends InfoItemExtractor {
+
+    /**
+     * Get the uploader name
+     * @return the uploader name
+     * @throws ParsingException
+     */
+    String getUploaderName() throws ParsingException;
+
+    /**
+     * Get the number of streams
+     * @return the number of streams
+     * @throws ParsingException
+     */
+    long getStreamCount() throws ParsingException;
+}

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java

@@ -0,0 +1,38 @@
+package org.schabi.newpipe.extractor.playlist;
+
+import org.schabi.newpipe.extractor.InfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+public class PlaylistInfoItemsCollector extends InfoItemsCollector<PlaylistInfoItem, PlaylistInfoItemExtractor> {
+
+    public PlaylistInfoItemsCollector(int serviceId) {
+        super(serviceId);
+    }
+
+    @Override
+    public PlaylistInfoItem extract(PlaylistInfoItemExtractor extractor) throws ParsingException {
+
+        String name = extractor.getName();
+        int serviceId = getServiceId();
+        String url = extractor.getUrl();
+
+        PlaylistInfoItem resultItem = new PlaylistInfoItem(serviceId, url, name);
+
+        try {
+            resultItem.setUploaderName(extractor.getUploaderName());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setThumbnailUrl(extractor.getThumbnailUrl());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setStreamCount(extractor.getStreamCount());
+        } catch (Exception e) {
+            addError(e);
+        }
+        return resultItem;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java

@@ -0,0 +1,81 @@
+package org.schabi.newpipe.extractor.search;
+
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.InfoItemExtractor;
+import org.schabi.newpipe.extractor.InfoItemsCollector;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+
+/*
+ * Created by Christian Schabesberger on 12.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * InfoItemsSearchCollector.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Collector for search results
+ *
+ * This collector can handle the following extractor types:
+ * <ul>
+ *     <li>{@link StreamInfoItemExtractor}</li>
+ *     <li>{@link ChannelInfoItemExtractor}</li>
+ *     <li>{@link PlaylistInfoItemExtractor}</li>
+ * </ul>
+ * Calling {@link #extract(InfoItemExtractor)} or {@link #commit(Object)} with any
+ * other extractor type will raise an exception.
+ */
+public class InfoItemsSearchCollector extends InfoItemsCollector<InfoItem, InfoItemExtractor> {
+    private String suggestion = "";
+    private final StreamInfoItemsCollector streamCollector;
+    private final ChannelInfoItemsCollector userCollector;
+    private final PlaylistInfoItemsCollector playlistCollector;
+
+    InfoItemsSearchCollector(int serviceId) {
+        super(serviceId);
+        streamCollector = new StreamInfoItemsCollector(serviceId);
+        userCollector = new ChannelInfoItemsCollector(serviceId);
+        playlistCollector = new PlaylistInfoItemsCollector(serviceId);
+    }
+
+    public void setSuggestion(String suggestion) {
+        this.suggestion = suggestion;
+    }
+
+    public SearchResult getSearchResult() throws ExtractionException {
+        return new SearchResult(getServiceId(), suggestion, getItems(), getErrors());
+    }
+
+    @Override
+    public InfoItem extract(InfoItemExtractor extractor) throws ParsingException {
+        // Use the corresponding collector for each item extractor type
+        if(extractor instanceof StreamInfoItemExtractor) {
+            return streamCollector.extract((StreamInfoItemExtractor) extractor);
+        } else if(extractor instanceof ChannelInfoItemExtractor) {
+            return userCollector.extract((ChannelInfoItemExtractor) extractor);
+        } else if(extractor instanceof PlaylistInfoItemExtractor) {
+            return playlistCollector.extract((PlaylistInfoItemExtractor) extractor);
+        } else {
+            throw new IllegalArgumentException("Invalid extractor type: " + extractor);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java

@@ -0,0 +1,50 @@
+package org.schabi.newpipe.extractor.search;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.io.IOException;
+
+/*
+ * Created by Christian Schabesberger on 10.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * SearchEngine.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public abstract class SearchEngine {
+    public enum Filter {
+        ANY, STREAM, CHANNEL, PLAYLIST
+    }
+
+    public static class NothingFoundException extends ExtractionException {
+        public NothingFoundException(String message) {
+            super(message);
+        }
+    }
+
+    private final InfoItemsSearchCollector collector;
+
+    public SearchEngine(int serviceId) {
+        collector = new InfoItemsSearchCollector(serviceId);
+    }
+
+    protected InfoItemsSearchCollector getInfoItemSearchCollector() {
+        return collector;
+    }
+
+    public abstract InfoItemsSearchCollector search(String query, int page, String contentCountry, Filter filter)
+            throws IOException, ExtractionException;
+}

extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java

@@ -0,0 +1,71 @@
+package org.schabi.newpipe.extractor.search;
+
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/*
+ * Created by Christian Schabesberger on 29.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * SearchResult.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class SearchResult {
+    private final int serviceId;
+    public final String suggestion;
+    @Nonnull
+    public final List<InfoItem> resultList;
+    @Nonnull
+    public final List<Throwable> errors;
+
+    public SearchResult(int serviceId, String suggestion, List<InfoItem> results, List<Throwable> errors) {
+        this.serviceId = serviceId;
+        this.suggestion = suggestion;
+        this.resultList = Collections.unmodifiableList(new ArrayList<>(results));
+        this.errors = Collections.unmodifiableList(new ArrayList<>(errors));
+    }
+
+    public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page,
+                                               final String languageCode, final SearchEngine.Filter filter)
+            throws IOException, ExtractionException {
+        return engine.search(query, page, languageCode, filter).getSearchResult();
+    }
+
+    public String getSuggestion() {
+        return suggestion;
+    }
+
+
+    @Nonnull
+    public List<InfoItem> getResults() {
+        return Collections.unmodifiableList(resultList);
+    }
+
+    @Nonnull
+    public List<Throwable> getErrors() {
+        return errors;
+    }
+
+    public int getServiceId() {
+        return serviceId;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -0,0 +1,136 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+@SuppressWarnings("WeakerAccess")
+public class SoundcloudChannelExtractor extends ChannelExtractor {
+    private String userId;
+    private JsonObject user;
+
+    private StreamInfoItemsCollector streamInfoItemsCollector = null;
+    private String nextPageUrl = null;
+
+    public SoundcloudChannelExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+
+        userId = getUrlIdHandler().getId(getOriginalUrl());
+        String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
+                "?client_id=" + SoundcloudParsingHelper.clientId();
+
+        String response = downloader.download(apiUrl);
+        try {
+            user = JsonParser.object().from(response);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getCleanUrl() {
+        return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl();
+    }
+
+    @Nonnull
+    @Override
+    public String getId() {
+        return userId;
+    }
+
+    @Nonnull
+    @Override
+    public String getName() {
+        return user.getString("username");
+    }
+
+    @Override
+    public String getAvatarUrl() {
+        return user.getString("avatar_url");
+    }
+
+    @Override
+    public String getBannerUrl() {
+        return user.getObject("visuals", new JsonObject())
+                .getArray("visuals", new JsonArray())
+                .getObject(0, new JsonObject())
+                .getString("visual_url");
+    }
+
+    @Override
+    public String getFeedUrl() {
+        return null;
+    }
+
+    @Override
+    public long getSubscriberCount() {
+        return user.getNumber("followers_count", 0).longValue();
+    }
+
+    @Override
+    public String getDescription() {
+        return user.getString("description", "");
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
+        if(streamInfoItemsCollector == null) {
+            computeNextPageAndGetStreams();
+        }
+        return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
+    }
+
+    @Override
+    public String getNextPageUrl() throws ExtractionException {
+        if(nextPageUrl == null) {
+            computeNextPageAndGetStreams();
+        }
+        return nextPageUrl;
+    }
+
+    private void computeNextPageAndGetStreams() throws ExtractionException {
+        try {
+            streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
+
+            String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+                    + "?client_id=" + SoundcloudParsingHelper.clientId()
+                    + "&limit=20"
+                    + "&linked_partitioning=1";
+
+            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
+        } catch (Exception e) {
+            throw new ExtractionException("Could not get next page", e);
+        }
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
+        if (pageUrl == null || pageUrl.isEmpty()) {
+            throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
+        }
+
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
+
+        return new InfoItemsPage<>(collector, nextPageUrl);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java

@@ -0,0 +1,44 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
+    private final JsonObject itemObject;
+
+    public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) {
+        this.itemObject = itemObject;
+    }
+
+    @Override
+    public String getName() {
+        return itemObject.getString("username");
+    }
+
+    @Override
+    public String getUrl() {
+        return replaceHttpWithHttps(itemObject.getString("permalink_url"));
+    }
+
+    @Override
+    public String getThumbnailUrl() {
+        return itemObject.getString("avatar_url", "");
+    }
+
+    @Override
+    public long getSubscriberCount() {
+        return itemObject.getNumber("followers_count", 0).longValue();
+    }
+
+    @Override
+    public long getStreamCount() {
+        return itemObject.getNumber("track_count", 0).longValue();
+    }
+
+    @Override
+    public String getDescription() {
+        return itemObject.getString("description", "");
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java

@@ -0,0 +1,60 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
+    private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
+    private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
+            "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
+
+    public static SoundcloudChannelUrlIdHandler getInstance() {
+        return instance;
+    }
+
+    @Override
+    public String getUrl(String id) throws ParsingException {
+        try {
+            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id);
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public String getId(String url) throws ParsingException {
+        Utils.checkUrl(URL_PATTERN, url);
+
+        try {
+            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public String cleanUrl(String complexUrl) throws ParsingException {
+        Utils.checkUrl(URL_PATTERN, complexUrl);
+
+        try {
+            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
+                    .select("meta[property=og:url]").first();
+
+            return replaceHttpWithHttps(ogElement.attr("content"));
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return Parser.isMatch(URL_PATTERN, url.toLowerCase());
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java

@@ -0,0 +1,95 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+public class SoundcloudChartsExtractor extends KioskExtractor {
+	private String url;
+
+	private StreamInfoItemsCollector collector = null;
+	private String nextPageUrl = null;
+
+    public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId)
+            throws ExtractionException {
+        super(service, url, kioskId);
+        this.url = url;
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) {
+    }
+
+    @Nonnull
+    @Override
+    public String getName() {
+        return "< Implement me (♥_♥) >";
+    }
+
+    @Nonnull
+    @Override
+    public UrlIdHandler getUrlIdHandler() {
+        return new SoundcloudChartsUrlIdHandler();
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
+        if (pageUrl == null || pageUrl.isEmpty()) {
+            throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
+        }
+
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true);
+
+        return new InfoItemsPage<>(collector, nextPageUrl);
+    }
+
+
+    private void computNextPageAndStreams() throws IOException, ExtractionException {
+        collector = new StreamInfoItemsCollector(getServiceId());
+
+        String apiUrl = "https://api-v2.soundcloud.com/charts" +
+                "?genre=soundcloud:genres:all-music" +
+                "&client_id=" + SoundcloudParsingHelper.clientId();
+
+        if (getId().equals("Top 50")) {
+            apiUrl += "&kind=top";
+        } else {
+            apiUrl += "&kind=trending";
+        }
+
+        List<String> supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US");
+        String contentCountry = getContentCountry();
+        if (supportedCountries.contains(contentCountry)) {
+            apiUrl += "&region=soundcloud:regions:" + contentCountry;
+        }
+
+        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
+    }
+
+    @Override
+    public String getNextPageUrl() throws IOException, ExtractionException {
+        if(nextPageUrl == null) {
+            computNextPageAndStreams();
+        }
+        return nextPageUrl;
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
+        if(collector == null) {
+            computNextPageAndStreams();
+        }
+        return new InfoItemsPage<>(collector, getNextPageUrl());
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java

@@ -0,0 +1,40 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
+    private final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
+    private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
+
+    public String getUrl(String id) {
+        if (id.equals("Top 50")) {
+            return "https://soundcloud.com/charts/top";
+        } else {
+            return "https://soundcloud.com/charts/new";
+        }
+    }
+
+    @Override
+    public String getId(String url) {
+        if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
+            return "Top 50";
+        } else {
+            return "New & hot";
+        }
+    }
+
+    @Override
+    public String cleanUrl(String url) {
+        if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
+            return "https://soundcloud.com/charts/top";
+        } else {
+            return "https://soundcloud.com/charts/new";
+        }
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return Parser.isMatch(URL_PATTERN, url.toLowerCase());
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -0,0 +1,232 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Parser.RegexException;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudParsingHelper {
+    private static String clientId;
+    
+    private SoundcloudParsingHelper() {
+    }
+
+    public static String clientId() throws ReCaptchaException, IOException, RegexException {
+        if (clientId != null && !clientId.isEmpty()) return clientId;
+
+        Downloader dl = NewPipe.getDownloader();
+
+        String response = dl.download("https://soundcloud.com");
+        Document doc = Jsoup.parse(response);
+
+        // TODO: Find a less heavy way to get the client_id
+        // Currently we are downloading a 1MB file (!) just to get the client_id,
+        // youtube-dl don't have a way too, they are just hardcoding and updating it when it becomes invalid.
+        // The embed mode has a way to get it, but we still have to download a heavy file (~800KB).
+        Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
+        String js = dl.download(jsElement.attr("src"));
+
+        return clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
+    }
+
+    public static String toDateString(String time) throws ParsingException {
+        try {
+            Date date;
+            // Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'.
+            try {
+                date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
+            } catch (Exception e) {
+                date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
+            }
+
+            SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+            return newDateFormat.format(date);
+        } catch (ParseException e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * Call the endpoint "/resolve" of the api.<p>
+     * 
+     * See https://developers.soundcloud.com/docs/api/reference#resolve
+     */
+    public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
+        String apiUrl = "https://api.soundcloud.com/resolve"
+                + "?url=" + URLEncoder.encode(url, "UTF-8")
+                + "&client_id=" + clientId();
+
+        try {
+            return JsonParser.object().from(downloader.download(apiUrl));
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+    }
+
+    /**
+     * Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).
+     *
+     * @return the url resolved
+     */
+    public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+
+        String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+                + URLEncoder.encode(apiUrl, "UTF-8"));
+
+        return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
+    }
+
+    /**
+     * Fetch the embed player with the url and return the id (like the id from the json api).
+     *
+     * @return the resolved id
+     */
+    public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
+
+        String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+                + URLEncoder.encode(url, "UTF-8"));
+        return Parser.matchGroup1(",\"id\":(.*?),", response);
+    }
+
+    /**
+     * Fetch the users from the given api and commit each of them to the collector.
+     * <p>
+     * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense that they will always
+     * get MIN_ITEMS or more.
+     *
+     * @param minItems the method will return only when it have extracted that many items (equal or more)
+     */
+    public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+        String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
+
+        while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
+            nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, nextPageUrl);
+        }
+
+        return nextPageUrl;
+    }
+
+    /**
+     * Fetch the user items from the given api and commit each of them to the collector.
+     *
+     * @return the next streams url, empty if don't have
+     */
+    public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+        String response = NewPipe.getDownloader().download(apiUrl);
+        JsonObject responseObject;
+        try {
+            responseObject = JsonParser.object().from(response);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+
+        JsonArray responseCollection = responseObject.getArray("collection");
+        for (Object o : responseCollection) {
+            if (o instanceof JsonObject) {
+                JsonObject object = (JsonObject) o;
+                collector.commit(new SoundcloudChannelInfoItemExtractor(object));
+            }
+        }
+
+        String nextPageUrl;
+        try {
+            nextPageUrl = responseObject.getString("next_href");
+            if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
+        } catch (Exception ignored) {
+            nextPageUrl = "";
+        }
+
+        return nextPageUrl;
+    }
+
+    /**
+     * Fetch the streams from the given api and commit each of them to the collector.
+     * <p>
+     * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense that they will always
+     * get MIN_ITEMS or more items.
+     *
+     * @param minItems the method will return only when it have extracted that many items (equal or more)
+     */
+    public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
+
+        while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
+            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl);
+        }
+
+        return nextPageUrl;
+    }
+
+    /**
+     * Fetch the streams from the given api and commit each of them to the collector.
+     *
+     * @return the next streams url, empty if don't have
+     */
+    public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
+        String response = NewPipe.getDownloader().download(apiUrl);
+        JsonObject responseObject;
+        try {
+            responseObject = JsonParser.object().from(response);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+
+        JsonArray responseCollection = responseObject.getArray("collection");
+        for (Object o : responseCollection) {
+            if (o instanceof JsonObject) {
+                JsonObject object = (JsonObject) o;
+                collector.commit(new SoundcloudStreamInfoItemExtractor(charts ? object.getObject("track") : object));
+            }
+        }
+
+        String nextPageUrl;
+        try {
+            nextPageUrl = responseObject.getString("next_href");
+            if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
+        } catch (Exception ignored) {
+            nextPageUrl = "";
+        }
+
+        return nextPageUrl;
+    }
+
+    public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
+        return getStreamsFromApi(collector, apiUrl, false);
+    }
+
+    @Nonnull
+    static String getUploaderUrl(JsonObject object) {
+        String url = object.getObject("user").getString("permalink_url", "");
+        return replaceHttpWithHttps(url);
+    }
+
+    @Nonnull
+    static String getAvatarUrl(JsonObject object) {
+        String url = object.getObject("user", new JsonObject()).getString("avatar_url", "");
+        return replaceHttpWithHttps(url);
+    }
+
+    public static String getUploaderName(JsonObject object) {
+        return object.getObject("user").getString("username", "");
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -0,0 +1,154 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+@SuppressWarnings("WeakerAccess")
+public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
+    private String playlistId;
+    private JsonObject playlist;
+
+    private StreamInfoItemsCollector streamInfoItemsCollector = null;
+    private String nextPageUrl = null;
+
+    public SoundcloudPlaylistExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+
+        playlistId = getUrlIdHandler().getId(getOriginalUrl());
+        String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
+                "?client_id=" + SoundcloudParsingHelper.clientId() +
+                "&representation=compact";
+
+        String response = downloader.download(apiUrl);
+        try {
+            playlist = JsonParser.object().from(response);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getCleanUrl() {
+        return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl();
+    }
+
+    @Nonnull
+    @Override
+    public String getId() {
+        return playlistId;
+    }
+
+    @Nonnull
+    @Override
+    public String getName() {
+        return playlist.getString("title");
+    }
+
+    @Override
+    public String getThumbnailUrl() {
+        String artworkUrl = playlist.getString("artwork_url");
+
+        if (artworkUrl == null) {
+            // If the thumbnail is null, traverse the items list and get a valid one,
+            // if it also fails, return null
+            try {
+                final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage();
+                if (infoItems.getItems().isEmpty()) return null;
+
+                for (StreamInfoItem item : infoItems.getItems()) {
+                    final String thumbnailUrl = item.getThumbnailUrl();
+                    if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
+
+                    return thumbnailUrl;
+                }
+            } catch (Exception ignored) {
+            }
+        }
+
+        return artworkUrl;
+    }
+
+    @Override
+    public String getBannerUrl() {
+        return null;
+    }
+
+    @Override
+    public String getUploaderUrl() {
+        return SoundcloudParsingHelper.getUploaderUrl(playlist);
+    }
+
+    @Override
+    public String getUploaderName() {
+        return SoundcloudParsingHelper.getUploaderName(playlist);
+    }
+
+    @Override
+    public String getUploaderAvatarUrl() {
+        return SoundcloudParsingHelper.getAvatarUrl(playlist);
+    }
+
+    @Override
+    public long getStreamCount() {
+        return playlist.getNumber("track_count", 0).longValue();
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
+        if (streamInfoItemsCollector == null) {
+            computeStreamsAndNextPageUrl();
+        }
+        return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
+    }
+
+    private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException {
+        streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
+
+        // Note the "api", NOT "api-v2"
+        String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
+                + "?client_id=" + SoundcloudParsingHelper.clientId()
+                + "&limit=20"
+                + "&linked_partitioning=1";
+
+        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
+    }
+
+    @Override
+    public String getNextPageUrl() throws IOException, ExtractionException {
+        if (nextPageUrl == null) {
+            computeStreamsAndNextPageUrl();
+        }
+        return nextPageUrl;
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
+        if (pageUrl == null || pageUrl.isEmpty()) {
+            throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
+        }
+
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
+
+        return new InfoItemsPage<>(collector, nextPageUrl);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java

@@ -0,0 +1,79 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
+    private static final String USER_KEY = "user";
+    private static final String AVATAR_URL_KEY = "avatar_url";
+    private static final String ARTWORK_URL_KEY = "artwork_url";
+
+    private final JsonObject itemObject;
+
+    public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) {
+        this.itemObject = itemObject;
+    }
+
+    @Override
+    public String getName() {
+        return itemObject.getString("title");
+    }
+
+    @Override
+    public String getUrl() {
+        return replaceHttpWithHttps(itemObject.getString("permalink_url"));
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        // Over-engineering at its finest
+        if (itemObject.isString(ARTWORK_URL_KEY)) {
+            final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
+            if (!artworkUrl.isEmpty()) return artworkUrl;
+        }
+
+        try {
+            // Look for artwork url inside the track list
+            for (Object track : itemObject.getArray("tracks")) {
+                final JsonObject trackObject = (JsonObject) track;
+
+                // First look for track artwork url
+                if (trackObject.isString(ARTWORK_URL_KEY)) {
+                    final String url = trackObject.getString(ARTWORK_URL_KEY, "");
+                    if (!url.isEmpty()) return url;
+                }
+
+                // Then look for track creator avatar url
+                final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject());
+                final String creatorAvatar = creator.getString(AVATAR_URL_KEY, "");
+                if (!creatorAvatar.isEmpty()) return creatorAvatar;
+            }
+        } catch (Exception ignored) {
+            // Try other method
+        }
+
+        try {
+            // Last resort, use user avatar url. If still not found, then throw exception.
+            return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist thumbnail url", e);
+        }
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        try {
+            return itemObject.getObject(USER_KEY).getString("username");
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist uploader", e);
+        }
+    }
+
+    @Override
+    public long getStreamCount() {
+        return itemObject.getNumber("track_count", 0).longValue();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java

@@ -0,0 +1,60 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
+    private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
+    private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
+            "/sets/[0-9a-z_-]+/?([#?].*)?$";
+
+    public static SoundcloudPlaylistUrlIdHandler getInstance() {
+        return instance;
+    }
+
+    @Override
+    public String getUrl(String id) throws ParsingException {
+        try {
+            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id);
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public String getId(String url) throws ParsingException {
+        Utils.checkUrl(URL_PATTERN, url);
+
+        try {
+            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public String cleanUrl(String complexUrl) throws ParsingException {
+        Utils.checkUrl(URL_PATTERN, complexUrl);
+
+        try {
+            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
+                    .select("meta[property=og:url]").first();
+
+            return replaceHttpWithHttps(ogElement.attr("content"));
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return Parser.isMatch(URL_PATTERN, url.toLowerCase());
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java

@@ -0,0 +1,84 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+
+public class SoundcloudSearchEngine extends SearchEngine {
+    public static final String CHARSET_UTF_8 = "UTF-8";
+
+    public SoundcloudSearchEngine(int serviceId) {
+        super(serviceId);
+    }
+
+    @Override
+    public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException {
+        InfoItemsSearchCollector collector = getInfoItemSearchCollector();
+
+        Downloader dl = NewPipe.getDownloader();
+
+        String url = "https://api-v2.soundcloud.com/search";
+
+        switch (filter) {
+            case STREAM:
+                url += "/tracks";
+                break;
+            case CHANNEL:
+                url += "/users";
+                break;
+            case PLAYLIST:
+                url += "/playlists";
+                break;
+            case ANY:
+                // Don't append any parameter to search for everything
+            default:
+                break;
+        }
+
+        url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+                + "&client_id=" + SoundcloudParsingHelper.clientId()
+                + "&limit=10"
+                + "&offset=" + Integer.toString(page * 10);
+
+        JsonArray searchCollection;
+        try {
+            searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection");
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+
+        if (searchCollection.size() == 0) {
+            throw new NothingFoundException("Nothing found");
+        }
+
+        for (Object result : searchCollection) {
+            if (!(result instanceof JsonObject)) continue;
+            //noinspection ConstantConditions
+            JsonObject searchResult = (JsonObject) result;
+            String kind = searchResult.getString("kind", "");
+            switch (kind) {
+                case "user":
+                    collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
+                    break;
+                case "track":
+                    collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
+                    break;
+                case "playlist":
+                    collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
+                    break;
+            }
+        }
+
+        return collector;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -0,0 +1,98 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.SuggestionExtractor;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.kiosk.KioskList;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
+
+import static java.util.Collections.singletonList;
+import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
+
+public class SoundcloudService extends StreamingService {
+
+    public SoundcloudService(int id) {
+        super(id, "SoundCloud", singletonList(AUDIO));
+    }
+
+    @Override
+    public SearchEngine getSearchEngine() {
+        return new SoundcloudSearchEngine(getServiceId());
+    }
+
+    @Override
+    public UrlIdHandler getStreamUrlIdHandler() {
+        return SoundcloudStreamUrlIdHandler.getInstance();
+    }
+
+    @Override
+    public UrlIdHandler getChannelUrlIdHandler() {
+        return SoundcloudChannelUrlIdHandler.getInstance();
+    }
+
+    @Override
+    public UrlIdHandler getPlaylistUrlIdHandler() {
+        return SoundcloudPlaylistUrlIdHandler.getInstance();
+    }
+
+
+    @Override
+    public StreamExtractor getStreamExtractor(String url) {
+        return new SoundcloudStreamExtractor(this, url);
+    }
+
+    @Override
+    public ChannelExtractor getChannelExtractor(String url) {
+        return new SoundcloudChannelExtractor(this, url);
+    }
+
+    @Override
+    public PlaylistExtractor getPlaylistExtractor(String url) {
+        return new SoundcloudPlaylistExtractor(this, url);
+    }
+
+    @Override
+    public SuggestionExtractor getSuggestionExtractor() {
+        return new SoundcloudSuggestionExtractor(getServiceId());
+    }
+
+    @Override
+    public KioskList getKioskList() throws ExtractionException {
+        KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() {
+            @Override
+            public KioskExtractor createNewKiosk(StreamingService streamingService,
+                                                 String url,
+                                                 String id)
+                    throws ExtractionException {
+                return new SoundcloudChartsExtractor(SoundcloudService.this,
+                        url,
+                        id);
+            }
+        };
+
+        KioskList list = new KioskList(getServiceId());
+
+        // add kiosks here e.g.:
+        final SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler();
+        try {
+            list.addKioskEntry(chartsFactory, h, "Top 50");
+            list.addKioskEntry(chartsFactory, h, "New & hot");
+        } catch (Exception e) {
+            throw new ExtractionException(e);
+        }
+
+        return list;
+    }
+
+
+    @Override
+    public SubscriptionExtractor getSubscriptionExtractor() {
+        return new SoundcloudSubscriptionExtractor(this);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -0,0 +1,217 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.schabi.newpipe.extractor.*;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.*;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudStreamExtractor extends StreamExtractor {
+    private JsonObject track;
+
+    public SoundcloudStreamExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
+
+        String policy = track.getString("policy", "");
+        if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
+            throw new ContentNotAvailableException("Content not available: policy " + policy);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getCleanUrl() {
+        return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl();
+    }
+
+    @Nonnull
+    @Override
+    public String getId() {
+        return track.getInt("id") + "";
+    }
+
+    @Nonnull
+    @Override
+    public String getName() {
+        return track.getString("title");
+    }
+
+    @Nonnull
+    @Override
+    public String getUploadDate() throws ParsingException {
+        return SoundcloudParsingHelper.toDateString(track.getString("created_at"));
+    }
+
+    @Nonnull
+    @Override
+    public String getThumbnailUrl() {
+        return track.getString("artwork_url", "");
+    }
+
+    @Nonnull
+    @Override
+    public String getDescription() {
+        return track.getString("description");
+    }
+
+    @Override
+    public int getAgeLimit() {
+        return NO_AGE_LIMIT;
+    }
+
+    @Override
+    public long getLength() {
+        return track.getNumber("duration", 0).longValue() / 1000L;
+    }
+
+    @Override
+    public long getTimeStamp() throws ParsingException {
+        return getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+    }
+
+    @Override
+    public long getViewCount() {
+        return track.getNumber("playback_count", 0).longValue();
+    }
+
+    @Override
+    public long getLikeCount() {
+        return track.getNumber("favoritings_count", -1).longValue();
+    }
+
+    @Override
+    public long getDislikeCount() {
+        return -1;
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderUrl() {
+        return SoundcloudParsingHelper.getUploaderUrl(track);
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderName() {
+        return SoundcloudParsingHelper.getUploaderName(track);
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderAvatarUrl() {
+        return SoundcloudParsingHelper.getAvatarUrl(track);
+    }
+
+    @Nonnull
+    @Override
+    public String getDashMpdUrl() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getHlsUrl() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
+        List<AudioStream> audioStreams = new ArrayList<>();
+        Downloader dl = NewPipe.getDownloader();
+
+        String apiUrl = "https://api.soundcloud.com/i1/tracks/" + urlEncode(getId()) + "/streams"
+                + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
+
+        String response = dl.download(apiUrl);
+        JsonObject responseObject;
+        try {
+            responseObject = JsonParser.object().from(response);
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+
+        String mp3Url = responseObject.getString("http_mp3_128_url");
+        if (mp3Url != null && !mp3Url.isEmpty()) {
+            audioStreams.add(new AudioStream(mp3Url, MediaFormat.MP3, 128));
+        } else {
+            throw new ExtractionException("Could not get SoundCloud's track audio url");
+        }
+
+        return audioStreams;
+    }
+
+    private static String urlEncode(String value) {
+        try {
+            return URLEncoder.encode(value, "UTF-8");
+        } catch (UnsupportedEncodingException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    @Override
+    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
+        return null;
+    }
+
+    @Override
+    public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
+        return null;
+    }
+
+    @Override
+    @Nonnull
+    public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException {
+        return Collections.emptyList();
+    }
+
+    @Override
+    @Nonnull
+    public List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public StreamType getStreamType() {
+        return StreamType.AUDIO_STREAM;
+    }
+
+    @Override
+    public StreamInfoItem getNextVideo() throws IOException, ExtractionException {
+        return null;
+    }
+
+    @Override
+    public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException {
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+
+        String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related"
+                + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
+
+        SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
+        return collector;
+    }
+
+
+    @Override
+    public String getErrorMessage() {
+        return null;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java

@@ -0,0 +1,67 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
+
+    protected final JsonObject itemObject;
+
+    public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) {
+        this.itemObject = itemObject;
+    }
+
+    @Override
+    public String getUrl() {
+        return replaceHttpWithHttps(itemObject.getString("permalink_url"));
+    }
+
+    @Override
+    public String getName() {
+        return itemObject.getString("title");
+    }
+
+    @Override
+    public long getDuration() {
+        return itemObject.getNumber("duration", 0).longValue() / 1000L;
+    }
+
+    @Override
+    public String getUploaderName() {
+        return itemObject.getObject("user").getString("username");
+    }
+
+    @Override
+    public String getUploaderUrl() {
+        return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url"));
+    }
+
+    @Override
+    public String getUploadDate() throws ParsingException {
+        return SoundcloudParsingHelper.toDateString(itemObject.getString("created_at"));
+    }
+
+    @Override
+    public long getViewCount() {
+        return itemObject.getNumber("playback_count", 0).longValue();
+    }
+
+    @Override
+    public String getThumbnailUrl() {
+        return itemObject.getString("artwork_url");
+    }
+
+    @Override
+    public StreamType getStreamType() {
+        return StreamType.AUDIO_STREAM;
+    }
+
+    @Override
+    public boolean isAd() {
+        return false;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java

@@ -0,0 +1,63 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
+    private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
+    private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
+            "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
+
+    private SoundcloudStreamUrlIdHandler() {
+    }
+
+    public static SoundcloudStreamUrlIdHandler getInstance() {
+        return instance;
+    }
+
+    @Override
+    public String getUrl(String id) throws ParsingException {
+        try {
+            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id);
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public String getId(String url) throws ParsingException {
+        Utils.checkUrl(URL_PATTERN, url);
+
+        try {
+            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public String cleanUrl(String complexUrl) throws ParsingException {
+        Utils.checkUrl(URL_PATTERN, complexUrl);
+
+        try {
+            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
+                    .select("meta[property=og:url]").first();
+
+            return replaceHttpWithHttps(ogElement.attr("content"));
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return Parser.isMatch(URL_PATTERN, url.toLowerCase());
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java

@@ -0,0 +1,74 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
+import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Extract the "followings" from a user in SoundCloud.
+ */
+public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
+
+    public SoundcloudSubscriptionExtractor(SoundcloudService service) {
+        super(service, Collections.singletonList(ContentSource.CHANNEL_URL));
+    }
+
+    @Override
+    public String getRelatedUrl() {
+        return "https://soundcloud.com/you";
+    }
+
+    @Override
+    public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
+        if (channelUrl == null) throw new InvalidSourceException("channel url is null");
+
+        String id;
+        try {
+            id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl));
+        } catch (ExtractionException e) {
+            throw new InvalidSourceException(e);
+        }
+
+        String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings"
+                + "?client_id=" + SoundcloudParsingHelper.clientId()
+                + "&limit=200";
+        ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId());
+        // ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough
+        SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl);
+
+        return toSubscriptionItems(collector.getItems());
+    }
+
+    private String getUrlFrom(String channelUrl) {
+        channelUrl = channelUrl.replace("http://", "https://").trim();
+
+        if (!channelUrl.startsWith("https://")) {
+            if (!channelUrl.contains("soundcloud.com/")) {
+                channelUrl = "https://soundcloud.com/" + channelUrl;
+            } else {
+                channelUrl = "https://" + channelUrl;
+            }
+        }
+
+        return channelUrl;
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Utils
+    //////////////////////////////////////////////////////////////////////////*/
+
+    private List<SubscriptionItem> toSubscriptionItems(List<ChannelInfoItem> items) {
+        List<SubscriptionItem> result = new ArrayList<>(items.size());
+        for (ChannelInfoItem item : items) {
+            result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName()));
+        }
+        return result;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java

@@ -0,0 +1,49 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.SuggestionExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
+
+    public static final String CHARSET_UTF_8 = "UTF-8";
+
+    public SoundcloudSuggestionExtractor(int serviceId) {
+        super(serviceId);
+    }
+
+    @Override
+    public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
+        List<String> suggestions = new ArrayList<>();
+
+        Downloader dl = NewPipe.getDownloader();
+
+        String url = "https://api-v2.soundcloud.com/search/queries"
+                + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+                + "&client_id=" + SoundcloudParsingHelper.clientId()
+                + "&limit=10";
+
+        String response = dl.download(url);
+        try {
+            JsonArray collection = JsonParser.object().from(response).getArray("collection");
+            for (Object suggestion : collection) {
+                if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query"));
+            }
+
+            return suggestions;
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java

@@ -0,0 +1,159 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import static org.schabi.newpipe.extractor.MediaFormat.*;
+import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.*;
+
+public class ItagItem {
+    /**
+     * List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
+     */
+    private static final ItagItem[] ITAG_LIST = {
+            /////////////////////////////////////////////////////
+            // VIDEO     ID  Type   Format  Resolution  FPS  ///
+            ///////////////////////////////////////////////////
+            new ItagItem(17, VIDEO, v3GPP, "144p"),
+            new ItagItem(36, VIDEO, v3GPP, "240p"),
+
+            new ItagItem(18, VIDEO, MPEG_4, "360p"),
+            new ItagItem(34, VIDEO, MPEG_4, "360p"),
+            new ItagItem(35, VIDEO, MPEG_4, "480p"),
+            new ItagItem(59, VIDEO, MPEG_4, "480p"),
+            new ItagItem(78, VIDEO, MPEG_4, "480p"),
+            new ItagItem(22, VIDEO, MPEG_4, "720p"),
+            new ItagItem(37, VIDEO, MPEG_4, "1080p"),
+            new ItagItem(38, VIDEO, MPEG_4, "1080p"),
+
+            new ItagItem(43, VIDEO, WEBM, "360p"),
+            new ItagItem(44, VIDEO, WEBM, "480p"),
+            new ItagItem(45, VIDEO, WEBM, "720p"),
+            new ItagItem(46, VIDEO, WEBM, "1080p"),
+
+            ////////////////////////////////////////////////////////////////////
+            // AUDIO     ID      ItagType          Format        Bitrate    ///
+            //////////////////////////////////////////////////////////////////
+            // Disable Opus codec as it's not well supported in older devices
+//          new ItagItem(249, AUDIO, WEBMA, 50),
+//          new ItagItem(250, AUDIO, WEBMA, 70),
+//          new ItagItem(251, AUDIO, WEBMA, 160),
+            new ItagItem(171, AUDIO, WEBMA, 128),
+            new ItagItem(172, AUDIO, WEBMA, 256),
+            new ItagItem(139, AUDIO, M4A, 48),
+            new ItagItem(140, AUDIO, M4A, 128),
+            new ItagItem(141, AUDIO, M4A, 256),
+
+            /// VIDEO ONLY ////////////////////////////////////////////
+            //           ID      Type     Format  Resolution  FPS  ///
+            /////////////////////////////////////////////////////////
+            // Don't add VideoOnly streams that have normal variants
+            new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
+            new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
+//          new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
+            new ItagItem(135, VIDEO_ONLY, MPEG_4, "480p"),
+            new ItagItem(212, VIDEO_ONLY, MPEG_4, "480p"),
+//          new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"),
+            new ItagItem(298, VIDEO_ONLY, MPEG_4, "720p60", 60),
+            new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"),
+            new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60),
+            new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"),
+
+            new ItagItem(278, VIDEO_ONLY, WEBM, "144p"),
+            new ItagItem(242, VIDEO_ONLY, WEBM, "240p"),
+//          new ItagItem(243, VIDEO_ONLY, WEBM, "360p"),
+            new ItagItem(244, VIDEO_ONLY, WEBM, "480p"),
+            new ItagItem(245, VIDEO_ONLY, WEBM, "480p"),
+            new ItagItem(246, VIDEO_ONLY, WEBM, "480p"),
+            new ItagItem(247, VIDEO_ONLY, WEBM, "720p"),
+            new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"),
+            new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"),
+            // #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
+            new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"),
+            new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60),
+            new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60),
+            new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60),
+            new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"),
+            new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60)
+    };
+    /*//////////////////////////////////////////////////////////////////////////
+    // Utils
+    //////////////////////////////////////////////////////////////////////////*/
+
+    public static boolean isSupported(int itag) {
+        for (ItagItem item : ITAG_LIST) {
+            if (itag == item.id) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static ItagItem getItag(int itagId) throws ParsingException {
+        for (ItagItem item : ITAG_LIST) {
+            if (itagId == item.id) {
+                return item;
+            }
+        }
+        throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported");
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Contructors and misc
+    //////////////////////////////////////////////////////////////////////////*/
+
+    public enum ItagType {
+        AUDIO,
+        VIDEO,
+        VIDEO_ONLY
+    }
+
+    /**
+     * Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30.
+     */
+    public ItagItem(int id, ItagType type, MediaFormat format, String resolution) {
+        this.id = id;
+        this.itagType = type;
+        this.mediaFormat = format;
+        this.resolutionString = resolution;
+        this.fps = 30;
+    }
+
+    /**
+     * Constructor for videos.
+     *
+     * @param resolution string that will be used in the frontend
+     */
+    public ItagItem(int id, ItagType type, MediaFormat format, String resolution, int fps) {
+        this.id = id;
+        this.itagType = type;
+        this.mediaFormat = format;
+        this.resolutionString = resolution;
+        this.fps = fps;
+    }
+
+    public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate) {
+        this.id = id;
+        this.itagType = type;
+        this.mediaFormat = format;
+        this.avgBitrate = avgBitrate;
+    }
+
+    private final MediaFormat mediaFormat;
+
+
+    public MediaFormat getMediaFormat() {
+        return mediaFormat;
+    }
+
+    public final int id;
+    public final ItagType itagType;
+
+    // Audio fields
+    public int avgBitrate = -1;
+
+    // Video fields
+    public String resolutionString;
+    public int fps = -1;
+
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java

@@ -0,0 +1,270 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+/*
+ * Created by Christian Schabesberger on 25.07.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeChannelExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+@SuppressWarnings("WeakerAccess")
+public class YoutubeChannelExtractor extends ChannelExtractor {
+    private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
+    private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
+
+    private Document doc;
+
+    public YoutubeChannelExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
+        String pageContent = downloader.download(channelUrl);
+        doc = Jsoup.parse(pageContent, channelUrl);
+
+    }
+
+    @Override
+    public String getNextPageUrl() throws ExtractionException {
+        return getNextPageUrlFrom(doc);
+    }
+
+    @Nonnull
+    @Override
+    public String getCleanUrl() {
+        try {
+            return "https://www.youtube.com/channel/" + getId();
+        } catch (ParsingException e) {
+            return super.getCleanUrl();
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getId() throws ParsingException {
+        try {
+            Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
+            if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
+
+            return element.attr("data-channel-external-id");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get channel id", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getName() throws ParsingException {
+        try {
+            return doc.select("meta[property=\"og:title\"]").first().attr("content");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get channel name", e);
+        }
+    }
+
+    @Override
+    public String getAvatarUrl() throws ParsingException {
+        try {
+            return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get avatar", e);
+        }
+    }
+
+    @Override
+    public String getBannerUrl() throws ParsingException {
+        try {
+            Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
+            String cssContent = el.html();
+            String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
+
+            return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get Banner", e);
+        }
+    }
+
+    @Override
+    public String getFeedUrl() throws ParsingException {
+        try {
+            return CHANNEL_FEED_BASE + getId();
+        } catch (Exception e) {
+            throw new ParsingException("Could not get feed url", e);
+        }
+    }
+
+    @Override
+    public long getSubscriberCount() throws ParsingException {
+        Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
+        if (el != null) {
+            return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
+        } else {
+            throw new ParsingException("Could not get subscriber count");
+        }
+    }
+
+    @Override
+    public String getDescription() throws ParsingException {
+        try {
+            return doc.select("meta[name=\"description\"]").first().attr("content");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get channel description", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
+        collectStreamsFrom(collector, ul);
+        return new InfoItemsPage<>(collector, getNextPageUrl());
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
+        if (pageUrl == null || pageUrl.isEmpty()) {
+            throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
+        }
+
+        // Unfortunately, we have to fetch the page even if we are only getting next streams,
+        // as they don't deliver enough information on their own (the channel name, for example).
+        fetchPage();
+
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        JsonObject ajaxJson;
+        try {
+            ajaxJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl));
+        } catch (JsonParserException pe) {
+            throw new ParsingException("Could not parse json data for next streams", pe);
+        }
+
+        final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl);
+        collectStreamsFrom(collector, ajaxHtml.select("body").first());
+
+        return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
+    }
+
+    private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
+        throws ParsingException {
+        String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
+        if (!loadMoreHtmlDataRaw.isEmpty()) {
+            return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl));
+        } else {
+            return "";
+        }
+    }
+
+    private String getNextPageUrlFrom(Document d) throws ParsingException {
+        try {
+            Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
+            if (button != null) {
+                return button.attr("abs:data-uix-load-more-href");
+            } else {
+                // Sometimes channels are simply so small, they don't have a more streams/videos
+                return "";
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not get next page url", e);
+        }
+    }
+
+    private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) throws ParsingException {
+        collector.reset();
+
+        final String uploaderName = getName();
+        final String uploaderUrl = getCleanUrl();
+        for (final Element li : element.children()) {
+            if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
+                collector.commit(new YoutubeStreamInfoItemExtractor(li) {
+                    @Override
+                    public String getUrl() throws ParsingException {
+                        try {
+                            Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
+                            Element dl = el.select("h3").first().select("a").first();
+                            return dl.attr("abs:href");
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get web page url for the video", e);
+                        }
+                    }
+
+                    @Override
+                    public String getName() throws ParsingException {
+                        try {
+                            Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
+                            Element dl = el.select("h3").first().select("a").first();
+                            return dl.text();
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get title", e);
+                        }
+                    }
+
+                    @Override
+                    public String getUploaderName() throws ParsingException {
+                        return uploaderName;
+                    }
+
+                    @Override
+                    public String getUploaderUrl() throws ParsingException {
+                        return uploaderUrl;
+                    }
+
+                    @Override
+                    public String getThumbnailUrl() throws ParsingException {
+                        try {
+                            String url;
+                            Element te = li.select("span[class=\"yt-thumb-clip\"]").first()
+                                    .select("img").first();
+                            url = te.attr("abs:src");
+                            // Sometimes youtube sends links to gif files which somehow seem to not exist
+                            // anymore. Items with such gif also offer a secondary image source. So we are going
+                            // to use that if we've caught such an item.
+                            if (url.contains(".gif")) {
+                                url = te.attr("abs:data-thumb");
+                            }
+                            return url;
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get thumbnail url", e);
+                        }
+                    }
+                });
+            }
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java

@@ -0,0 +1,89 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+/*
+ * Created by Christian Schabesberger on 12.02.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeChannelInfoItemExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor {
+    private final Element el;
+
+    public YoutubeChannelInfoItemExtractor(Element el) {
+        this.el = el;
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        Element img = el.select("span[class*=\"yt-thumb-simple\"]").first()
+                .select("img").first();
+
+        String url = img.attr("abs:src");
+
+        if (url.contains("gif")) {
+            url = img.attr("abs:data-thumb");
+        }
+        return url;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return el.select("a[class*=\"yt-uix-tile-link\"]").first()
+                .text();
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        return el.select("a[class*=\"yt-uix-tile-link\"]").first()
+                .attr("abs:href");
+    }
+
+    @Override
+    public long getSubscriberCount() throws ParsingException {
+        Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first();
+        if (subsEl == null) {
+            return 0;
+        } else {
+            return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text()));
+        }
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
+        if (metaEl == null) {
+            return 0;
+        } else {
+            return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text()));
+        }
+    }
+
+    @Override
+    public String getDescription() throws ParsingException {
+        Element desEl = el.select("div[class*=\"yt-lockup-description\"]").first();
+        if (desEl == null) {
+            return "";
+        } else {
+            return desEl.text();
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java

@@ -0,0 +1,56 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+/*
+ * Created by Christian Schabesberger on 25.07.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeChannelUrlIdHandler.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
+
+    private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler();
+    private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)";
+
+    public static YoutubeChannelUrlIdHandler getInstance() {
+        return instance;
+    }
+
+    @Override
+    public String getUrl(String id) {
+        return "https://www.youtube.com/" + id;
+    }
+
+    @Override
+    public String getId(String url) throws ParsingException {
+        return Parser.matchGroup1(ID_PATTERN, url);
+    }
+
+    @Override
+    public String cleanUrl(String complexUrl) throws ParsingException {
+        return getUrl(getId(complexUrl));
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return (url.contains("youtube") || url.contains("youtu.be") || url.contains("hooktube.com"))
+                && (url.contains("/user/") || url.contains("/channel/"));
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -0,0 +1,66 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+/*
+ * Created by Christian Schabesberger on 02.03.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeParsingHelper.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeParsingHelper {
+
+    private YoutubeParsingHelper() {
+    }
+
+    public static long parseDurationString(String input)
+            throws ParsingException, NumberFormatException {
+        String[] splitInput = input.split(":");
+        String days = "0";
+        String hours = "0";
+        String minutes = "0";
+        String seconds;
+
+        switch (splitInput.length) {
+            case 4:
+                days = splitInput[0];
+                hours = splitInput[1];
+                minutes = splitInput[2];
+                seconds = splitInput[3];
+                break;
+            case 3:
+                hours = splitInput[0];
+                minutes = splitInput[1];
+                seconds = splitInput[2];
+                break;
+            case 2:
+                minutes = splitInput[0];
+                seconds = splitInput[1];
+                break;
+            case 1:
+                seconds = splitInput[0];
+                break;
+            default:
+                throw new ParsingException("Error duration string with unknown format: " + input);
+        }
+        return ((((Long.parseLong(days) * 24)
+                + Long.parseLong(hours) * 60)
+                + Long.parseLong(minutes)) * 60)
+                + Long.parseLong(seconds);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java

@@ -0,0 +1,287 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+@SuppressWarnings("WeakerAccess")
+public class YoutubePlaylistExtractor extends PlaylistExtractor {
+
+    private Document doc;
+
+    public YoutubePlaylistExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        String pageContent = downloader.download(getCleanUrl());
+        doc = Jsoup.parse(pageContent, getCleanUrl());
+    }
+
+    @Override
+    public String getNextPageUrl() throws ExtractionException {
+        return getNextPageUrlFrom(doc);
+    }
+
+    @Nonnull
+    @Override
+    public String getId() throws ParsingException {
+        try {
+            return getUrlIdHandler().getId(getCleanUrl());
+        } catch (Exception e) {
+            throw new ParsingException("Could not get playlist id");
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getName() throws ParsingException {
+        try {
+            return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
+        } catch (Exception e) {
+            throw new ParsingException("Could not get playlist name");
+        }
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        try {
+            return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get playlist thumbnail");
+        }
+    }
+
+    @Override
+    public String getBannerUrl() {
+        return "";      // Banner can't be handled by frontend right now.
+                        // Whoever is willing to implement this should also implement this in the fornt end
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        try {
+            return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get playlist uploader name");
+        }
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        try {
+            return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
+        } catch (Exception e) {
+            throw new ParsingException("Could not get playlist uploader name");
+        }
+    }
+
+    @Override
+    public String getUploaderAvatarUrl() throws ParsingException {
+        try {
+            return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get playlist uploader avatar");
+        }
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        String input;
+
+        try {
+            input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text();
+        } catch (IndexOutOfBoundsException e) {
+            throw new ParsingException("Could not get video count from playlist", e);
+        }
+
+        try {
+            return Long.parseLong(Utils.removeNonDigitCharacters(input));
+        } catch (NumberFormatException e) {
+            // When there's no videos in a playlist, there's no number in the "innerHtml",
+            // all characters that is not a number is removed, so we try to parse a empty string
+            if (!input.isEmpty()) {
+                return 0;
+            } else {
+                throw new ParsingException("Could not handle input: " + input, e);
+            }
+        }
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
+        collectStreamsFrom(collector, tbody);
+        return new InfoItemsPage<>(collector, getNextPageUrl());
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
+        if (pageUrl == null || pageUrl.isEmpty()) {
+            throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
+        }
+
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        JsonObject pageJson;
+        try {
+            pageJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl));
+        } catch (JsonParserException pe) {
+            throw new ParsingException("Could not parse ajax json", pe);
+        }
+
+        final Document pageHtml = Jsoup.parse("<table><tbody id=\"pl-load-more-destination\">"
+                + pageJson.getString("content_html")
+                + "</tbody></table>", pageUrl);
+
+        collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first());
+
+        return new InfoItemsPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl));
+    }
+
+    private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl)
+            throws ParsingException{
+        String nextPageHtml = pageJson.getString("load_more_widget_html");
+        if (!nextPageHtml.isEmpty()) {
+            return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl));
+        } else {
+            return "";
+        }
+    }
+
+    private String getNextPageUrlFrom(Document d) throws ParsingException {
+        try {
+            Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
+            if (button != null) {
+                return button.attr("abs:data-uix-load-more-href");
+            } else {
+                // Sometimes playlists are simply so small, they don't have a more streams/videos
+                return "";
+            }
+        } catch (Exception e) {
+            throw new ParsingException("could not get next streams' url", e);
+        }
+    }
+
+    private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) throws ParsingException {
+        collector.reset();
+
+        final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();
+        for (final Element li : element.children()) {
+            if(isDeletedItem(li)) {
+                continue;
+            }
+
+            collector.commit(new YoutubeStreamInfoItemExtractor(li) {
+                public Element uploaderLink;
+
+                @Override
+                public boolean isAd() throws ParsingException {
+                    return false;
+                }
+
+                @Override
+                public String getUrl() throws ParsingException {
+                    try {
+                        return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get web page url for the video", e);
+                    }
+                }
+
+                @Override
+                public String getName() throws ParsingException {
+                    try {
+                        return li.attr("data-title");
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get title", e);
+                    }
+                }
+
+                @Override
+                public long getDuration() throws ParsingException {
+                    try {
+                        if (getStreamType() == StreamType.LIVE_STREAM) return -1;
+
+                        Element first = li.select("div[class=\"timestamp\"] span").first();
+                        if (first == null) {
+                            // Video unavailable (private, deleted, etc.), this is a thing that happens specifically with playlists,
+                            // because in other cases, those videos don't even show up
+                            return -1;
+                        }
+
+                        return YoutubeParsingHelper.parseDurationString(first.text());
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get duration" + getUrl(), e);
+                    }
+                }
+
+
+                private Element getUploaderLink() {
+                    // should always be present since we filter deleted items
+                    if(uploaderLink == null) {
+                        uploaderLink = li.select("div[class=pl-video-owner] a").first();
+                    }
+                    return uploaderLink;
+                }
+
+                @Override
+                public String getUploaderName() throws ParsingException {
+                    return getUploaderLink().text();
+                }
+
+                @Override
+                public String getUploaderUrl() throws ParsingException {
+                    return getUploaderLink().attr("abs:href");
+                }
+
+                @Override
+                public String getUploadDate() throws ParsingException {
+                    return "";
+                }
+
+                @Override
+                public long getViewCount() throws ParsingException {
+                    return -1;
+                }
+
+                @Override
+                public String getThumbnailUrl() throws ParsingException {
+                    try {
+                        return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getUrl()) + "/hqdefault.jpg";
+                    } catch (Exception e) {
+                        throw new ParsingException("Could not get thumbnail url", e);
+                    }
+                }
+            });
+        }
+    }
+
+    /**
+     * Check if the playlist item is deleted
+     * @param li the list item
+     * @return true if the item is deleted
+     */
+    private boolean isDeletedItem(Element li) {
+        return li.select("div[class=pl-video-owner] a").isEmpty();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java

@@ -0,0 +1,92 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
+    private final Element el;
+
+    public YoutubePlaylistInfoItemExtractor(Element el) {
+        this.el = el;
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        String url;
+
+        try {
+            Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first()
+                    .select("img").first();
+            url = te.attr("abs:src");
+
+            if (url.contains(".gif")) {
+                url = te.attr("abs:data-thumb");
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist thumbnail url", e);
+        }
+
+        return url;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        String name;
+        try {
+            final Element title = el.select("[class=\"yt-lockup-title\"]").first()
+                    .select("a").first();
+
+            name = title == null ? "" : title.text();
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist name", e);
+        }
+
+        return name;
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        String url;
+
+        try {
+            final Element href = el.select("div[class=\"yt-lockup-meta\"]").first()
+                    .select("a").first();
+
+            url = href.attr("abs:href");
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist url", e);
+        }
+
+        return url;
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        String name;
+
+        try {
+            final Element div = el.select("div[class=\"yt-lockup-byline\"]").first()
+                    .select("a").first();
+
+            name = div.text();
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist uploader", e);
+        }
+
+        return name;
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        try {
+            final Element count = el.select("span[class=\"formatted-video-count-label\"]").first()
+                    .select("b").first();
+
+            return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text()));
+        } catch (Exception e) {
+            throw new ParsingException("Failed to extract playlist stream count", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java

@@ -0,0 +1,42 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
+
+    private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
+    private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})";
+
+    public static YoutubePlaylistUrlIdHandler getInstance() {
+        return instance;
+    }
+
+    @Override
+    public String getUrl(String id) {
+        return "https://www.youtube.com/playlist?list=" + id;
+    }
+
+    @Override
+    public String getId(String url) throws ParsingException {
+        try {
+            return Parser.matchGroup1("list=" + ID_PATTERN, url);
+        } catch (final Exception exception) {
+            throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception);
+        }
+    }
+
+    @Override
+    public String cleanUrl(String complexUrl) throws ParsingException {
+        return getUrl(getId(complexUrl));
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        final boolean hasNotEmptyUrl = url != null && !url.isEmpty();
+        final boolean isYoutubeDomain = hasNotEmptyUrl && (url.contains("youtube") || url.contains("youtu.be"));
+        return isYoutubeDomain && url.contains("list=");
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java

@@ -0,0 +1,125 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+
+
+/*
+ * Created by Christian Schabesberger on 09.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeSearchEngine.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeSearchEngine extends SearchEngine {
+
+    private static final String TAG = YoutubeSearchEngine.class.toString();
+    public static final String CHARSET_UTF_8 = "UTF-8";
+
+    public YoutubeSearchEngine(int serviceId) {
+        super(serviceId);
+    }
+
+    @Override
+    public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter)
+            throws IOException, ExtractionException {
+        InfoItemsSearchCollector collector = getInfoItemSearchCollector();
+        Downloader downloader = NewPipe.getDownloader();
+
+        String url = "https://www.youtube.com/results"
+                + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+                + "&page=" + Integer.toString(page + 1);
+
+        switch (filter) {
+            case STREAM:
+                url += "&sp=EgIQAVAU";
+                break;
+            case CHANNEL:
+                url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
+                break;
+            case PLAYLIST:
+                url += "&sp=EgIQA1AU"; //EgIQA( one )AU
+                break;
+            case ANY:
+                // Don't append any parameter to search for everything
+            default:
+                break;
+        }
+
+        String site;
+        //String url = builder.build().toString();
+        //if we've been passed a valid language code, append it to the URL
+        if (!languageCode.isEmpty()) {
+            //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
+            site = downloader.download(url, languageCode);
+        } else {
+            site = downloader.download(url);
+        }
+
+        Document doc = Jsoup.parse(site, url);
+        Element list = doc.select("ol[class=\"item-section\"]").first();
+
+        for (Element item : list.children()) {
+            /* First we need to determine which kind of item we are working with.
+               Youtube depicts five different kinds of items on its search result page. These are
+               regular videos, playlists, channels, two types of video suggestions, and a "no video
+               found" item. Since we only want videos, we need to filter out all the others.
+               An example for this can be seen here:
+               https://www.youtube.com/results?search_query=asdf&page=1
+
+               We already applied a filter to the url, so we don't need to care about channels and
+               playlists now.
+            */
+
+            Element el;
+
+            // both types of spell correction item
+            if ((el = item.select("div[class*=\"spell-correction\"]").first()) != null) {
+                collector.setSuggestion(el.select("a").first().text());
+                if (list.children().size() == 1) {
+                    throw new NothingFoundException("Did you mean: " + el.select("a").first().text());
+                }
+                // search message item
+            } else if ((el = item.select("div[class*=\"search-message\"]").first()) != null) {
+                throw new NothingFoundException(el.text());
+
+                // video item type
+            } else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
+                collector.commit(new YoutubeStreamInfoItemExtractor(el));
+            } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
+                collector.commit(new YoutubeChannelInfoItemExtractor(el));
+            } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null &&
+                    item.select(".yt-pl-icon-mix").isEmpty()) {
+                collector.commit(new YoutubePlaylistInfoItemExtractor(el));
+            } else {
+                // noinspection ConstantConditions
+                // simply ignore not known items
+                // throw new ExtractionException("unexpected element found: \"" + item + "\"");
+            }
+        }
+
+        return collector;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -0,0 +1,111 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.SuggestionExtractor;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.kiosk.KioskList;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.search.SearchEngine;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
+
+import static java.util.Arrays.asList;
+import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*;
+
+
+/*
+ * Created by Christian Schabesberger on 23.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeService.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeService extends StreamingService {
+
+    public YoutubeService(int id) {
+        super(id, "YouTube", asList(AUDIO, VIDEO, LIVE));
+    }
+
+    @Override
+    public SearchEngine getSearchEngine() {
+        return new YoutubeSearchEngine(getServiceId());
+    }
+
+    @Override
+    public UrlIdHandler getStreamUrlIdHandler() {
+        return YoutubeStreamUrlIdHandler.getInstance();
+    }
+
+    @Override
+    public UrlIdHandler getChannelUrlIdHandler() {
+        return YoutubeChannelUrlIdHandler.getInstance();
+    }
+
+    @Override
+    public UrlIdHandler getPlaylistUrlIdHandler() {
+        return YoutubePlaylistUrlIdHandler.getInstance();
+    }
+
+    @Override
+    public StreamExtractor getStreamExtractor(String url) {
+        return new YoutubeStreamExtractor(this, url);
+    }
+
+    @Override
+    public ChannelExtractor getChannelExtractor(String url) {
+        return new YoutubeChannelExtractor(this, url);
+    }
+
+    @Override
+    public PlaylistExtractor getPlaylistExtractor(String url) {
+        return new YoutubePlaylistExtractor(this, url);
+    }
+
+    @Override
+    public SuggestionExtractor getSuggestionExtractor() {
+        return new YoutubeSuggestionExtractor(getServiceId());
+    }
+
+    @Override
+    public KioskList getKioskList() throws ExtractionException {
+        KioskList list = new KioskList(getServiceId());
+
+        // add kiosks here e.g.:
+        try {
+            list.addKioskEntry(new KioskList.KioskExtractorFactory() {
+                @Override
+                public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id)
+                throws ExtractionException {
+                    return new YoutubeTrendingExtractor(YoutubeService.this, url, id);
+                }
+            }, new YoutubeTrendingUrlIdHandler(), "Trending");
+            list.setDefaultKiosk("Trending");
+        } catch (Exception e) {
+            throw new ExtractionException(e);
+        }
+
+        return list;
+    }
+
+    @Override
+    public SubscriptionExtractor getSubscriptionExtractor() {
+        return new YoutubeSubscriptionExtractor(this);
+    }
+
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -0,0 +1,927 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.mozilla.javascript.Context;
+import org.mozilla.javascript.Function;
+import org.mozilla.javascript.ScriptableObject;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.Subtitles;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.*;
+
+/*
+ * Created by Christian Schabesberger on 06.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeStreamExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeStreamExtractor extends StreamExtractor {
+    private static final String TAG = YoutubeStreamExtractor.class.getSimpleName();
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Exceptions
+    //////////////////////////////////////////////////////////////////////////*/
+
+    public class DecryptException extends ParsingException {
+        DecryptException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    public class GemaException extends ContentNotAvailableException {
+        GemaException(String message) {
+            super(message);
+        }
+    }
+
+    public class SubtitlesException extends ContentNotAvailableException {
+        SubtitlesException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////*/
+
+    private Document doc;
+    @Nullable
+    private JsonObject playerArgs;
+    @Nonnull
+    private final Map<String, String> videoInfoPage = new HashMap<>();
+
+    @Nonnull
+    private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
+
+    private boolean isAgeRestricted;
+
+    public YoutubeStreamExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Impl
+    //////////////////////////////////////////////////////////////////////////*/
+
+    @Nonnull
+    @Override
+    public String getId() throws ParsingException {
+        try {
+            return getUrlIdHandler().getId(getCleanUrl());
+        } catch (Exception e) {
+            throw new ParsingException("Could not get stream id");
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getName() throws ParsingException {
+        assertPageFetched();
+        String name = getStringFromMetaData("title");
+        if(name == null) {
+            // Fallback to HTML method
+            try {
+                name = doc.select("meta[name=title]").attr(CONTENT);
+            } catch (Exception e) {
+                throw new ParsingException("Could not get the title", e);
+            }
+        }
+        if(name == null || name.isEmpty()) {
+            throw new ParsingException("Could not get the title");
+        }
+        return name;
+    }
+
+    @Nonnull
+    @Override
+    public String getUploadDate() throws ParsingException {
+        assertPageFetched();
+        try {
+            return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
+        } catch (Exception e) {//todo: add fallback method
+            throw new ParsingException("Could not get upload date", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        assertPageFetched();
+        // Try to get high resolution thumbnail first, if it fails, use low res from the player instead
+        try {
+            return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
+        } catch (Exception ignored) {
+            // Try other method...
+        }
+
+        try {
+            if (playerArgs != null && playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
+        } catch (Exception ignored) {
+            // Try other method...
+        }
+
+        try {
+            return videoInfoPage.get("thumbnail_url");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get thumbnail url", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getDescription() throws ParsingException {
+        assertPageFetched();
+        try {
+            return doc.select("p[id=\"eow-description\"]").first().html();
+        } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know
+            throw new ParsingException("Could not get the description", e);
+        }
+    }
+
+    @Override
+    public int getAgeLimit() throws ParsingException {
+        assertPageFetched();
+        if (!isAgeRestricted) {
+            return NO_AGE_LIMIT;
+        }
+        try {
+            return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]")
+                    .attr(CONTENT).replace("+", ""));
+        } catch (Exception e) {
+            throw new ParsingException("Could not get age restriction");
+        }
+    }
+
+    @Override
+    public long getLength() throws ParsingException {
+        assertPageFetched();
+        if(playerArgs != null) {
+            try {
+                long returnValue = Long.parseLong(playerArgs.get("length_seconds") + "");
+                if (returnValue >= 0) return returnValue;
+            } catch (Exception ignored) {
+                // Try other method...
+            }
+        }
+
+        String lengthString = videoInfoPage.get("length_seconds");
+        try {
+            return Long.parseLong(lengthString);
+        } catch (Exception ignored) {
+            // Try other method...
+        }
+
+        // TODO: 25.11.17 Implement a way to get the length for age restricted videos #44
+        try {
+            // Fallback to HTML method
+            return Long.parseLong(doc.select("div[class~=\"ytp-progress-bar\"][role=\"slider\"]").first()
+                    .attr("aria-valuemax"));
+        } catch (Exception e) {
+            throw new ParsingException("Could not get video length", e);
+        }
+    }
+
+    /**
+     * Attempts to parse (and return) the offset to start playing the video from.
+     *
+     * @return the offset (in seconds), or 0 if no timestamp is found.
+     */
+    @Override
+    public long getTimeStamp() throws ParsingException {
+        return getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+    }
+
+    @Override
+    public long getViewCount() throws ParsingException {
+        assertPageFetched();
+        try {
+            return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
+        } catch (Exception e) {//todo: find fallback method
+            throw new ParsingException("Could not get number of views", e);
+        }
+    }
+
+    @Override
+    public long getLikeCount() throws ParsingException {
+        assertPageFetched();
+        String likesString = "";
+        try {
+            Element button = doc.select("button.like-button-renderer-like-button").first();
+            try {
+                likesString = button.select("span.yt-uix-button-content").first().text();
+            } catch (NullPointerException e) {
+                //if this kicks in our button has no content and therefore likes/dislikes are disabled
+                return -1;
+            }
+            return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
+        } catch (NumberFormatException nfe) {
+            throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe);
+        } catch (Exception e) {
+            throw new ParsingException("Could not get like count", e);
+        }
+    }
+
+    @Override
+    public long getDislikeCount() throws ParsingException {
+        assertPageFetched();
+        String dislikesString = "";
+        try {
+            Element button = doc.select("button.like-button-renderer-dislike-button").first();
+            try {
+                dislikesString = button.select("span.yt-uix-button-content").first().text();
+            } catch (NullPointerException e) {
+                //if this kicks in our button has no content and therefore likes/dislikes are disabled
+                return -1;
+            }
+            return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
+        } catch (NumberFormatException nfe) {
+            throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe);
+        } catch (Exception e) {
+            throw new ParsingException("Could not get dislike count", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        assertPageFetched();
+        try {
+            return doc.select("div[class=\"yt-user-info\"]").first().children()
+                    .select("a").first().attr("abs:href");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get channel link", e);
+        }
+    }
+
+
+    @Nullable
+    private String getStringFromMetaData(String field) {
+        assertPageFetched();
+        String value = null;
+        if(playerArgs != null) {
+            // This can not fail
+            value = playerArgs.getString(field);
+        }
+        if(value == null) {
+            // This can not fail too
+            value = videoInfoPage.get(field);
+        }
+        return value;
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderName() throws ParsingException {
+        assertPageFetched();
+        String name = getStringFromMetaData("author");
+
+        if(name == null) {
+            try {
+                // Fallback to HTML method
+                name = doc.select("div.yt-user-info").first().text();
+            } catch (Exception e) {
+                throw new ParsingException("Could not get uploader name", e);
+            }
+        }
+        if(name == null || name.isEmpty()) {
+            throw new ParsingException("Could not get uploader name");
+        }
+        return name;
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderAvatarUrl() throws ParsingException {
+        assertPageFetched();
+        try {
+            return doc.select("a[class*=\"yt-user-photo\"]").first()
+                    .select("img").first()
+                    .attr("abs:data-thumb");
+        } catch (Exception e) {//todo: add fallback method
+            throw new ParsingException("Could not get uploader thumbnail URL.", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getDashMpdUrl() throws ParsingException {
+        assertPageFetched();
+        try {
+            String dashManifestUrl;
+            if (videoInfoPage.containsKey("dashmpd")) {
+                dashManifestUrl = videoInfoPage.get("dashmpd");
+            } else if (playerArgs != null && playerArgs.isString("dashmpd")) {
+                dashManifestUrl = playerArgs.getString("dashmpd", "");
+            } else {
+                return "";
+            }
+
+            if (!dashManifestUrl.contains("/signature/")) {
+                String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
+                String decryptedSig;
+
+                decryptedSig = decryptSignature(encryptedSig, decryptionCode);
+                dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
+            }
+
+            return dashManifestUrl;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get dash manifest url", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public String getHlsUrl() throws ParsingException {
+        assertPageFetched();
+        try {
+            String hlsvp;
+            if (playerArgs != null && playerArgs.isString("hlsvp")) {
+                hlsvp = playerArgs.getString("hlsvp", "");
+            } else {
+                return "";
+            }
+
+            return hlsvp;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get hls manifest url", e);
+        }
+    }
+
+    @Override
+    public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
+        assertPageFetched();
+        List<AudioStream> audioStreams = new ArrayList<>();
+        try {
+            for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) {
+                ItagItem itag = entry.getValue();
+
+                AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
+                if (!Stream.containSimilarStream(audioStream, audioStreams)) {
+                    audioStreams.add(audioStream);
+                }
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not get audio streams", e);
+        }
+
+        return audioStreams;
+    }
+
+    @Override
+    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
+        assertPageFetched();
+        List<VideoStream> videoStreams = new ArrayList<>();
+        try {
+            for (Map.Entry<String, ItagItem> entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) {
+                ItagItem itag = entry.getValue();
+
+                VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
+                if (!Stream.containSimilarStream(videoStream, videoStreams)) {
+                    videoStreams.add(videoStream);
+                }
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not get video streams", e);
+        }
+
+        return videoStreams;
+    }
+
+    @Override
+    public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
+        assertPageFetched();
+        List<VideoStream> videoOnlyStreams = new ArrayList<>();
+        try {
+            for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
+                ItagItem itag = entry.getValue();
+
+                VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
+                if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
+                    videoOnlyStreams.add(videoStream);
+                }
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not get video only streams", e);
+        }
+
+        return videoOnlyStreams;
+    }
+
+    @Override
+    @Nonnull
+    public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException {
+        return getSubtitles(SubtitlesFormat.TTML);
+    }
+
+    @Override
+    @Nonnull
+    public List<Subtitles> getSubtitles(final SubtitlesFormat format) throws IOException, ExtractionException {
+        assertPageFetched();
+        List<Subtitles> subtitles = new ArrayList<>();
+        for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) {
+            subtitles.add(subtitlesInfo.getSubtitle(format));
+        }
+        return subtitles;
+    }
+
+    @Override
+    public StreamType getStreamType() throws ParsingException {
+        assertPageFetched();
+        try {
+            if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
+                    playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
+                return StreamType.LIVE_STREAM;
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not get hls manifest url", e);
+        }
+        return StreamType.VIDEO_STREAM;
+    }
+
+    @Override
+    public StreamInfoItem getNextVideo() throws IOException, ExtractionException {
+        assertPageFetched();
+        try {
+            StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+            collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]")
+                    .first().select("li").first()));
+
+            return collector.getItems().get(0);
+        } catch (Exception e) {
+            throw new ParsingException("Could not get next video", e);
+        }
+    }
+
+    @Override
+    public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException {
+        assertPageFetched();
+        try {
+            StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+            Element ul = doc.select("ul[id=\"watch-related\"]").first();
+            if (ul != null) {
+                for (Element li : ul.children()) {
+                    // first check if we have a playlist. If so leave them out
+                    if (li.select("a[class*=\"content-link\"]").first() != null) {
+                        collector.commit(extractVideoPreviewInfo(li));
+                    }
+                }
+            }
+            return collector;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get related videos", e);
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public String getErrorMessage() {
+        String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
+        StringBuilder errorReason;
+
+        if (errorMessage == null || errorMessage.isEmpty()) {
+            errorReason = null;
+        } else if (errorMessage.contains("GEMA")) {
+            // Gema sometimes blocks youtube music content in germany:
+            // https://www.gema.de/en/
+            // Detailed description:
+            // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
+            errorReason = new StringBuilder("GEMA");
+        } else {
+            errorReason = new StringBuilder(errorMessage);
+            errorReason.append("  ");
+            errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
+        }
+
+        return errorReason != null ? errorReason.toString() : null;
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Fetch page
+    //////////////////////////////////////////////////////////////////////////*/
+
+    private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
+    private static final String ADAPTIVE_FMTS = "adaptive_fmts";
+    private static final String HTTPS = "https:";
+    private static final String CONTENT = "content";
+    private static final String DECRYPTION_FUNC_NAME = "decrypt";
+
+    private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
+
+    private volatile String decryptionCode = "";
+
+    private String pageHtml = null;
+
+    private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
+        final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS;
+        if (pageHtml == null) {
+            pageHtml = downloader.download(verifiedUrl);
+        }
+        return pageHtml;
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        final String pageContent = getPageHtml(downloader);
+        doc = Jsoup.parse(pageContent, getCleanUrl());
+
+        final String playerUrl;
+        // Check if the video is age restricted
+        if (pageContent.contains("<meta property=\"og:restrictions:age")) {
+            final EmbeddedInfo info = getEmbeddedInfo();
+            final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
+            final String infoPageResponse = downloader.download(videoInfoUrl);
+            videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
+            playerUrl = info.url;
+            isAgeRestricted = true;
+        } else {
+            final JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
+            playerArgs = getPlayerArgs(ytPlayerConfig);
+            playerUrl = getPlayerUrl(ytPlayerConfig);
+            isAgeRestricted = false;
+        }
+
+        if (decryptionCode.isEmpty()) {
+            decryptionCode = loadDecryptionCode(playerUrl);
+        }
+
+        if (subtitlesInfos.isEmpty()) {
+            subtitlesInfos.addAll(getAvailableSubtitlesInfo());
+        }
+    }
+
+    private JsonObject getPlayerConfig(String pageContent) throws ParsingException {
+        try {
+            String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
+            return JsonParser.object().from(ytPlayerConfigRaw);
+        } catch (Parser.RegexException e) {
+            String errorReason = getErrorMessage();
+            switch (errorReason) {
+                case "GEMA":
+                    throw new GemaException(errorReason);
+                case "":
+                    throw new ContentNotAvailableException("Content not available: player config empty", e);
+                default:
+                    throw new ContentNotAvailableException("Content not available", e);
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not parse yt player config", e);
+        }
+    }
+
+    private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
+        JsonObject playerArgs;
+
+        //attempt to load the youtube js player JSON arguments
+        try {
+            playerArgs = playerConfig.getObject("args");
+        } catch (Exception e) {
+            throw new ParsingException("Could not parse yt player config", e);
+        }
+
+        return playerArgs;
+    }
+
+    private String getPlayerUrl(JsonObject playerConfig) throws ParsingException {
+        try {
+            // The Youtube service needs to be initialized by downloading the
+            // js-Youtube-player. This is done in order to get the algorithm
+            // for decrypting cryptic signatures inside certain stream urls.
+            String playerUrl;
+
+            JsonObject ytAssets = playerConfig.getObject("assets");
+            playerUrl = ytAssets.getString("js");
+
+            if (playerUrl.startsWith("//")) {
+                playerUrl = HTTPS + playerUrl;
+            }
+            return playerUrl;
+        } catch (Exception e) {
+            throw new ParsingException("Could not load decryption code for the Youtube service.", e);
+        }
+    }
+
+    @Nonnull
+    private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
+        try {
+            final Downloader downloader = NewPipe.getDownloader();
+            final String embedUrl = "https://www.youtube.com/embed/" + getId();
+            final String embedPageContent = downloader.download(embedUrl);
+
+            // Get player url
+            final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
+            String playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
+                    .replace("\\", "").replace("\"", "");
+            if (playerUrl.startsWith("//")) {
+                playerUrl = HTTPS + playerUrl;
+            }
+
+            // Get embed sts
+            final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
+            final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
+
+            return new EmbeddedInfo(playerUrl, sts);
+        } catch (IOException e) {
+            throw new ParsingException(
+                    "Could load decryption code form restricted video for the Youtube service.", e);
+        } catch (ReCaptchaException e) {
+            throw new ReCaptchaException("reCaptcha Challenge requested");
+        }
+    }
+
+    private String loadDecryptionCode(String playerUrl) throws DecryptException {
+        String decryptionFuncName;
+        String decryptionFunc;
+        String helperObjectName;
+        String helperObject;
+        String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}";
+        String decryptionCode;
+
+        try {
+            Downloader downloader = NewPipe.getDownloader();
+            if (!playerUrl.contains("https://youtube.com")) {
+                //sometimes the https://youtube.com part does not get send with
+                //than we have to add it by hand
+                playerUrl = "https://youtube.com" + playerUrl;
+            }
+            String playerCode = downloader.download(playerUrl);
+
+            decryptionFuncName =
+                    Parser.matchGroup("([\"\\'])signature\\1\\s*,\\s*([a-zA-Z0-9$]+)\\(", playerCode, 2);
+
+            String functionPattern = "("
+                    + decryptionFuncName.replace("$", "\\$")
+                    + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
+            decryptionFunc = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
+
+            helperObjectName = Parser
+                    .matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc);
+
+            String helperPattern = "(var "
+                    + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
+            helperObject = Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
+
+
+            callerFunc = callerFunc.replace("%%", decryptionFuncName);
+            decryptionCode = helperObject + decryptionFunc + callerFunc;
+        } catch (IOException ioe) {
+            throw new DecryptException("Could not load decrypt function", ioe);
+        } catch (Exception e) {
+            throw new DecryptException("Could not parse decrypt function ", e);
+        }
+
+        return decryptionCode;
+    }
+
+    private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
+        Context context = Context.enter();
+        context.setOptimizationLevel(-1);
+        Object result;
+        try {
+            ScriptableObject scope = context.initStandardObjects();
+            context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
+            Function decryptionFunc = (Function) scope.get("decrypt", scope);
+            result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
+        } catch (Exception e) {
+            throw new DecryptException("could not get decrypt signature", e);
+        } finally {
+            Context.exit();
+        }
+        return result == null ? "" : result.toString();
+    }
+
+    @Nonnull
+    private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
+        // If the video is age restricted getPlayerConfig will fail
+        if(isAgeRestricted) return Collections.emptyList();
+
+        final JsonObject playerConfig;
+        try {
+            playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
+        } catch (IOException | ExtractionException e) {
+            throw new SubtitlesException("Unable to download player configs", e);
+        }
+        final String playerResponse = playerConfig.getObject("args", new JsonObject())
+                .getString("player_response");
+
+        final JsonObject captions;
+        try {
+            if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) {
+                // Captions does not exist
+                return Collections.emptyList();
+            }
+            captions = JsonParser.object().from(playerResponse).getObject("captions");
+        } catch (JsonParserException e) {
+            throw new SubtitlesException("Unable to parse subtitles listing", e);
+        }
+
+        final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
+        final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
+        // todo: use this to apply auto translation to different language from a source language
+        final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray());
+
+        // This check is necessary since there may be cases where subtitles metadata do not contain caption track info
+        // e.g. https://www.youtube.com/watch?v=-Vpwatutnko
+        final int captionsSize = captionsArray.size();
+        if(captionsSize == 0) return Collections.emptyList();
+
+        List<SubtitlesInfo> result = new ArrayList<>();
+        for (int i = 0; i < captionsSize; i++) {
+            final String languageCode = captionsArray.getObject(i).getString("languageCode");
+            final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
+            final String vssId = captionsArray.getObject(i).getString("vssId");
+
+            if (languageCode != null && baseUrl != null && vssId != null) {
+                final boolean isAutoGenerated = vssId.startsWith("a.");
+                result.add(new SubtitlesInfo(baseUrl, languageCode, isAutoGenerated));
+            }
+        }
+
+        return result;
+    }
+    /*//////////////////////////////////////////////////////////////////////////
+    // Data Class
+    //////////////////////////////////////////////////////////////////////////*/
+
+    private class EmbeddedInfo {
+        final String url;
+        final String sts;
+
+        EmbeddedInfo(final String url, final String sts) {
+            this.url = url;
+            this.sts = sts;
+        }
+    }
+
+    private class SubtitlesInfo {
+        final String cleanUrl;
+        final String languageCode;
+        final boolean isGenerated;
+
+        final Locale locale;
+
+        public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) {
+            this.cleanUrl = baseUrl
+                    .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
+                    .replaceAll("&tlang=[^&]*", ""); // Remove translation language
+            this.languageCode = languageCode;
+            this.isGenerated = isGenerated;
+
+            final String[] splits = languageCode.split("-");
+            this.locale = splits.length == 2 ? new Locale(splits[0], splits[1]) : new Locale(languageCode);
+        }
+
+        public Subtitles getSubtitle(final SubtitlesFormat format) {
+            return new Subtitles(format, locale, cleanUrl + "&fmt=" + format.getExtension(), isGenerated);
+        }
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Utils
+    //////////////////////////////////////////////////////////////////////////*/
+
+    @Nonnull
+    private static String getVideoInfoUrl(final String id, final String sts) {
+        return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
+                "&eurl=https://youtube.googleapis.com/v/" + id +
+                "&sts=" + sts + "&ps=default&gl=US&hl=en";
+    }
+
+    private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
+        Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
+
+        String encodedUrlMap = "";
+        if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
+            encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
+        } else if (videoInfoPage.containsKey(encodedUrlMapKey)) {
+            encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
+        }
+
+        for (String url_data_str : encodedUrlMap.split(",")) {
+            try {
+                // This loop iterates through multiple streams, therefore tags
+                // is related to one and the same stream at a time.
+                Map<String, String> tags = Parser.compatParseMap(
+                        org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
+
+                int itag = Integer.parseInt(tags.get("itag"));
+
+                if (ItagItem.isSupported(itag)) {
+                    ItagItem itagItem = ItagItem.getItag(itag);
+                    if (itagItem.itagType == itagTypeWanted) {
+                        String streamUrl = tags.get("url");
+                        // if video has a signature: decrypt it and add it to the url
+                        if (tags.get("s") != null) {
+                            streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
+                        }
+                        urlAndItags.put(streamUrl, itagItem);
+                    }
+                }
+            } catch (DecryptException e) {
+                throw e;
+            } catch (Exception ignored) {
+            }
+        }
+
+        return urlAndItags;
+    }
+
+    /**
+     * Provides information about links to other videos on the video page, such as related videos.
+     * This is encapsulated in a StreamInfoItem object, which is a subset of the fields in a full StreamInfo.
+     */
+    private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
+        return new YoutubeStreamInfoItemExtractor(li) {
+
+            @Override
+            public String getUrl() throws ParsingException {
+                return li.select("a.content-link").first().attr("abs:href");
+            }
+
+            @Override
+            public String getName() throws ParsingException {
+                //todo: check NullPointerException causing
+                return li.select("span.title").first().text();
+                //this page causes the NullPointerException, after finding it by searching for "tjvg":
+                //https://www.youtube.com/watch?v=Uqg0aEhLFAg
+            }
+
+            @Override
+            public String getUploaderName() throws ParsingException {
+                return li.select("span[class*=\"attribution\"").first()
+                        .select("span").first().text();
+            }
+
+            @Override
+            public String getUploaderUrl() throws ParsingException {
+                return ""; // The uploader is not linked
+            }
+
+            @Override
+            public String getUploadDate() throws ParsingException {
+                return "";
+            }
+
+            @Override
+            public long getViewCount() throws ParsingException {
+                try {
+                    if (getStreamType() == StreamType.LIVE_STREAM) return -1;
+
+                    return Long.parseLong(Utils.removeNonDigitCharacters(
+                            li.select("span.view-count").first().text()));
+                } catch (Exception e) {
+                    //related videos sometimes have no view count
+                    return 0;
+                }
+            }
+
+            @Override
+            public String getThumbnailUrl() throws ParsingException {
+                Element img = li.select("img").first();
+                String thumbnailUrl = img.attr("abs:src");
+                // Sometimes youtube sends links to gif files which somehow seem to not exist
+                // anymore. Items with such gif also offer a secondary image source. So we are going
+                // to use that if we caught such an item.
+                if (thumbnailUrl.contains(".gif")) {
+                    thumbnailUrl = img.attr("data-thumb");
+                }
+                if (thumbnailUrl.startsWith("//")) {
+                    thumbnailUrl = HTTPS + thumbnailUrl;
+                }
+                return thumbnailUrl;
+            }
+        };
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java

@@ -0,0 +1,181 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+/*
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeStreamInfoItemExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
+
+    private final Element item;
+
+    public YoutubeStreamInfoItemExtractor(Element item) {
+        this.item = item;
+    }
+
+    @Override
+    public StreamType getStreamType() throws ParsingException {
+        if (isLiveStream(item)) {
+            return StreamType.LIVE_STREAM;
+        } else {
+            return StreamType.VIDEO_STREAM;
+        }
+    }
+
+    @Override
+    public boolean isAd() throws ParsingException {
+        return !item.select("span[class*=\"icon-not-available\"]").isEmpty()
+                || !item.select("span[class*=\"yt-badge-ad\"]").isEmpty();
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        try {
+            Element el = item.select("div[class*=\"yt-lockup-video\"").first();
+            Element dl = el.select("h3").first().select("a").first();
+            return dl.attr("abs:href");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get web page url for the video", e);
+        }
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        try {
+            Element el = item.select("div[class*=\"yt-lockup-video\"").first();
+            Element dl = el.select("h3").first().select("a").first();
+            return dl.text();
+        } catch (Exception e) {
+            throw new ParsingException("Could not get title", e);
+        }
+    }
+
+    @Override
+    public long getDuration() throws ParsingException {
+        try {
+            if (getStreamType() == StreamType.LIVE_STREAM) return -1;
+
+            final Element duration = item.select("span[class*=\"video-time\"]").first();
+            // apparently on youtube, video-time element will not show up if the video has a duration of 00:00
+            // see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf
+            return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text());
+        } catch (Exception e) {
+            throw new ParsingException("Could not get Duration: " + getUrl(), e);
+        }
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        try {
+            return item.select("div[class=\"yt-lockup-byline\"]").first()
+                    .select("a").first()
+                    .text();
+        } catch (Exception e) {
+            throw new ParsingException("Could not get uploader", e);
+        }
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        try {
+            try {
+                return item.select("div[class=\"yt-lockup-byline\"]").first()
+                        .select("a").first()
+                        .attr("abs:href");
+            } catch (Exception e){}
+
+            // try this if the first didn't work
+            return item.select("span[class=\"title\"")
+                    .text().split(" - ")[0];
+        } catch (Exception e) {
+            System.out.println(item.html());
+            throw new ParsingException("Could not get uploader", e);
+        }
+    }
+
+    @Override
+    public String getUploadDate() throws ParsingException {
+        try {
+            Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
+            if (meta == null) return "";
+
+            return meta.select("li").first().text();
+        } catch (Exception e) {
+            throw new ParsingException("Could not get upload date", e);
+        }
+    }
+
+    @Override
+    public long getViewCount() throws ParsingException {
+        String input;
+        try {
+            // TODO: Return the actual live stream's watcher count
+            // -1 for no view count
+            if (getStreamType() == StreamType.LIVE_STREAM) return -1;
+
+            Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
+            if (meta == null) return -1;
+
+            input = meta.select("li").get(1).text();
+        } catch (IndexOutOfBoundsException e) {
+            throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
+        }
+
+        try {
+            return Long.parseLong(Utils.removeNonDigitCharacters(input));
+        } catch (NumberFormatException e) {
+            // if this happens the video probably has no views
+            if (!input.isEmpty()){
+                return 0;
+            }
+
+            throw new ParsingException("Could not handle input: " + input, e);
+        }
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        try {
+            String url;
+            Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
+                    .select("img").first();
+            url = te.attr("abs:src");
+            // Sometimes youtube sends links to gif files which somehow seem to not exist
+            // anymore. Items with such gif also offer a secondary image source. So we are going
+            // to use that if we've caught such an item.
+            if (url.contains(".gif")) {
+                url = te.attr("abs:data-thumb");
+            }
+            return url;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get thumbnail url", e);
+        }
+    }
+
+    /**
+     * Generic method that checks if the element contains any clues that it's a livestream item
+     */
+    protected static boolean isLiveStream(Element item) {
+        return !item.select("span[class*=\"yt-badge-live\"]").isEmpty()
+                || !item.select("span[class*=\"video-time-overlay-live\"]").isEmpty();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java

@@ -0,0 +1,191 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.FoundAdException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLDecoder;
+
+/*
+ * Created by Christian Schabesberger on 02.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeStreamUrlIdHandler.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
+
+    private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler();
+    private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})";
+
+    private YoutubeStreamUrlIdHandler() {
+    }
+
+    public static YoutubeStreamUrlIdHandler getInstance() {
+        return instance;
+    }
+
+    @Override
+    public String getUrl(String id) {
+        return "https://www.youtube.com/watch?v=" + id;
+    }
+
+    @Override
+    public String getId(String url) throws ParsingException, IllegalArgumentException {
+        if (url.isEmpty()) {
+            throw new IllegalArgumentException("The url parameter should not be empty");
+        }
+
+        String id;
+        String lowercaseUrl = url.toLowerCase();
+        if (lowercaseUrl.contains("youtube")) {
+            if (url.contains("attribution_link")) {
+                try {
+                    String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url);
+                    String query = URLDecoder.decode(escapedQuery, "UTF-8");
+                    id = Parser.matchGroup1("v=" + ID_PATTERN, query);
+                } catch (UnsupportedEncodingException uee) {
+                    throw new ParsingException("Could not parse attribution_link", uee);
+                }
+            } else if (lowercaseUrl.contains("youtube.com/shared?ci=")) {
+                return getRealIdFromSharedLink(url);
+            } else if (url.contains("vnd.youtube")) {
+                id = Parser.matchGroup1(ID_PATTERN, url);
+            } else if (url.contains("embed")) {
+                id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
+            } else if (url.contains("googleads")) {
+                throw new FoundAdException("Error found add: " + url);
+            } else {
+                id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
+            }
+        } else if (lowercaseUrl.contains("youtu.be")) {
+            if (url.contains("v=")) {
+                id = Parser.matchGroup1("v=" + ID_PATTERN, url);
+            } else {
+                id = Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url);
+            }
+        } else if(lowercaseUrl.contains("hooktube")) {
+            if(lowercaseUrl.contains("&v=")
+                    || lowercaseUrl.contains("?v=")) {
+                id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
+            } else if (url.contains("/embed/")) {
+                id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
+            } else if (url.contains("/v/")) {
+                id = Parser.matchGroup1("v/" + ID_PATTERN, url);
+            } else if (url.contains("/watch/")) {
+                id = Parser.matchGroup1("watch/" + ID_PATTERN, url);
+            } else {
+                throw new ParsingException("Error no suitable url: " + url);
+            }
+        } else {
+            throw new ParsingException("Error no suitable url: " + url);
+        }
+
+
+        if (!id.isEmpty()) {
+            return id;
+        } else {
+            throw new ParsingException("Error could not parse url: " + url);
+        }
+    }
+
+    /**
+     * Get the real url from a shared uri.
+     * <p>
+     * Shared URI's look like this:
+     * <pre>
+     *     * https://www.youtube.com/shared?ci=PJICrTByb3E
+     *     * vnd.youtube://www.youtube.com/shared?ci=PJICrTByb3E&feature=twitter-deep-link
+     * </pre>
+     *
+     * @param url The shared url
+     * @return the id of the stream
+     * @throws ParsingException
+     */
+    private String getRealIdFromSharedLink(String url) throws ParsingException {
+        URI uri;
+        try {
+            uri = new URI(url);
+        } catch (URISyntaxException e) {
+            throw new ParsingException("Invalid shared link", e);
+        }
+        String sharedId = getSharedId(uri);
+        Downloader downloader = NewPipe.getDownloader();
+        String content;
+        try {
+            content = downloader.download("https://www.youtube.com/shared?ci=" + sharedId);
+        } catch (IOException | ReCaptchaException e) {
+            throw new ParsingException("Unable to resolve shared link", e);
+        }
+        Document document = Jsoup.parse(content);
+        String urlWithRealId;
+
+        Element element = document.select("link[rel=\"canonical\"]").first();
+        if (element != null) {
+            urlWithRealId = element.attr("abs:href");
+        } else {
+            urlWithRealId = document.select("meta[property=\"og:url\"]").first()
+                    .attr("abs:content");
+        }
+
+        String realId = Parser.matchGroup1(ID_PATTERN, urlWithRealId);
+        if (sharedId.equals(realId)) {
+            throw new ParsingException("Got same id for as shared info_id: " + sharedId);
+        }
+        return realId;
+    }
+
+    private String getSharedId(URI uri) throws ParsingException {
+        if (!"/shared".equals(uri.getPath())) {
+            throw new ParsingException("Not a shared link: " + uri.toString() + " (path != " + uri.getPath() + ")");
+        }
+        return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
+    }
+
+    @Override
+    public String cleanUrl(String complexUrl) throws ParsingException {
+        return getUrl(getId(complexUrl));
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        String lowercaseUrl = url.toLowerCase();
+        if (lowercaseUrl.contains("youtube")
+                || lowercaseUrl.contains("youtu.be")
+                || lowercaseUrl.contains("hooktube")) {
+            // bad programming I know
+            try {
+                getId(url);
+                return true;
+            } catch (Exception e) {
+                return false;
+            }
+        } else {
+            return false;
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java

@@ -0,0 +1,131 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
+import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM;
+
+/**
+ * Extract subscriptions from a YouTube export (OPML format supported)
+ */
+public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
+
+    public YoutubeSubscriptionExtractor(YoutubeService service) {
+        super(service, Collections.singletonList(INPUT_STREAM));
+    }
+
+    @Override
+    public String getRelatedUrl() {
+        return "https://www.youtube.com/subscription_manager?action_takeout=1";
+    }
+
+    @Override
+    public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws ExtractionException {
+        if (contentInputStream == null) throw new InvalidSourceException("input stream is null");
+
+        return getItemsFromOPML(contentInputStream);
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // OPML implementation
+    //////////////////////////////////////////////////////////////////////////*/
+
+    private static final String ID_PATTERN = "/videos.xml\\?channel_id=([A-Za-z0-9_-]*)";
+    private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/";
+
+    private List<SubscriptionItem> getItemsFromOPML(InputStream contentInputStream) throws ExtractionException {
+        final List<SubscriptionItem> result = new ArrayList<>();
+
+        final String contentString = readFromInputStream(contentInputStream);
+        Document document = Jsoup.parse(contentString, "", org.jsoup.parser.Parser.xmlParser());
+
+        if (document.select("opml").isEmpty()) {
+            throw new InvalidSourceException("document does not have OPML tag");
+        }
+
+        if (document.select("outline").isEmpty()) {
+            throw new InvalidSourceException("document does not have at least one outline tag");
+        }
+
+        for (Element outline : document.select("outline[type=rss]")) {
+            String title = outline.attr("title");
+            String xmlUrl = outline.attr("abs:xmlUrl");
+
+            if (title.isEmpty() || xmlUrl.isEmpty()) {
+                throw new InvalidSourceException("document has invalid entries");
+            }
+
+            try {
+                String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
+                result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
+            } catch (Parser.RegexException e) {
+                throw new InvalidSourceException("document has invalid entries", e);
+            }
+        }
+
+        return result;
+    }
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Utils
+    //////////////////////////////////////////////////////////////////////////*/
+
+    /**
+     * Throws an exception if the string does not have the right tag/string from a valid export.
+     */
+    private void throwIfTagIsNotFound(String content) throws InvalidSourceException {
+        if (!content.trim().contains("<opml")) {
+            throw new InvalidSourceException("input stream does not have OPML tag");
+        }
+    }
+
+    private String readFromInputStream(InputStream inputStream) throws InvalidSourceException {
+        StringBuilder contentBuilder = new StringBuilder();
+        boolean hasTag = false;
+        try {
+            byte[] buffer = new byte[16 * 1024];
+            int read;
+            while ((read = inputStream.read(buffer)) != -1) {
+                String currentPartOfContent = new String(buffer, 0, read, "UTF-8");
+                contentBuilder.append(currentPartOfContent);
+
+                // Fail-fast in case of reading a long unsupported input stream
+                if (!hasTag && contentBuilder.length() > 128) {
+                    throwIfTagIsNotFound(contentBuilder.toString());
+                    hasTag = true;
+                }
+            }
+        } catch (InvalidSourceException e) {
+            throw e;
+        } catch (Throwable e) {
+            throw new InvalidSourceException(e);
+        } finally {
+            try {
+                inputStream.close();
+            } catch (IOException ignored) {
+            }
+        }
+
+        final String fileContent = contentBuilder.toString().trim();
+        if (fileContent.isEmpty()) {
+            throw new InvalidSourceException("Empty input stream");
+        }
+
+        if (!hasTag) {
+            throwIfTagIsNotFound(fileContent);
+        }
+
+        return fileContent;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java

@@ -0,0 +1,74 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.SuggestionExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
+
+/*
+ * Created by Christian Schabesberger on 28.09.16.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeSuggestionExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class YoutubeSuggestionExtractor extends SuggestionExtractor {
+
+    public static final String CHARSET_UTF_8 = "UTF-8";
+
+    public YoutubeSuggestionExtractor(int serviceId) {
+        super(serviceId);
+    }
+
+    @Override
+    public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
+        Downloader dl = NewPipe.getDownloader();
+        List<String> suggestions = new ArrayList<>();
+
+        String url = "https://suggestqueries.google.com/complete/search"
+                + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+                + "&jsonp=" + "JP"
+                + "&ds=" + "yt"
+                + "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8)
+                + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
+
+        String response = dl.download(url);
+        // trim JSONP part "JP(...)"
+        response = response.substring(3, response.length()-1);
+        try {
+            JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray());
+            for (Object suggestion : collection) {
+                if (!(suggestion instanceof JsonArray)) continue;
+                String suggestionStr = ((JsonArray)suggestion).getString(0);
+                if (suggestionStr == null) continue;
+                suggestions.add(suggestionStr);
+            }
+
+            return suggestions;
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java

@@ -0,0 +1,169 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+public class YoutubeTrendingExtractor extends KioskExtractor {
+
+    private Document doc;
+
+    public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId)
+            throws ExtractionException {
+        super(service, url, kioskId);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        final String contentCountry = getContentCountry();
+        String url = getCleanUrl();
+        if(contentCountry != null && !contentCountry.isEmpty()) {
+            url += "?gl=" + contentCountry;
+        }
+
+        String pageContent = downloader.download(url);
+        doc = Jsoup.parse(pageContent, url);
+    }
+
+    @Nonnull
+    @Override
+    public UrlIdHandler getUrlIdHandler() {
+        return new YoutubeTrendingUrlIdHandler();
+    }
+
+    @Override
+    public String getNextPageUrl() {
+        return "";
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public String getName() throws ParsingException {
+        try {
+            Element a = doc.select("a[href*=\"/feed/trending\"]").first();
+            Element span = a.select("span[class*=\"display-name\"]").first();
+            Element nameSpan = span.select("span").first();
+            return nameSpan.text();
+        } catch (Exception e) {
+            throw new ParsingException("Could not get Trending name", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
+        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
+        for(Element ul : uls) {
+            for(final Element li : ul.children()) {
+                final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
+                collector.commit(new YoutubeStreamInfoItemExtractor(li) {
+                    @Override
+                    public String getUrl() throws ParsingException {
+                        try {
+                            Element dl = el.select("h3").first().select("a").first();
+                            return dl.attr("abs:href");
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get web page url for the video", e);
+                        }
+                    }
+
+                    @Override
+                    public String getName() throws ParsingException {
+                        try {
+                            Element dl = el.select("h3").first().select("a").first();
+                            return dl.text();
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get web page url for the video", e);
+                        }
+                    }
+
+                    @Override
+                    public String getUploaderUrl() throws ParsingException {
+                        try {
+                            String link = getUploaderLink().attr("abs:href");
+                            if (link.isEmpty()) {
+                                throw new IllegalArgumentException("is empty");
+                            }
+                            return link;
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get Uploader name");
+                        }
+                    }
+
+                    private Element getUploaderLink() {
+                        Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
+                        return uploaderEl.select("a").first();
+                    }
+
+                    @Override
+                    public String getUploaderName() throws ParsingException {
+                        try {
+                            return getUploaderLink().text();
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get Uploader name");
+                        }
+                    }
+
+                    @Override
+                    public String getThumbnailUrl() throws ParsingException {
+                        try {
+                            String url;
+                            Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
+                                    .select("img").first();
+                            url = te.attr("abs:src");
+                            // Sometimes youtube sends links to gif files which somehow seem to not exist
+                            // anymore. Items with such gif also offer a secondary image source. So we are going
+                            // to use that if we've caught such an item.
+                            if (url.contains(".gif")) {
+                                url = te.attr("abs:data-thumb");
+                            }
+                            return url;
+                        } catch (Exception e) {
+                            throw new ParsingException("Could not get thumbnail url", e);
+                        }
+                    }
+                });
+            }
+        }
+
+        return new InfoItemsPage<>(collector, getNextPageUrl());
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java

@@ -0,0 +1,46 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingUrlIdHandler.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+public class YoutubeTrendingUrlIdHandler implements UrlIdHandler {
+
+    public String getUrl(String id) {
+        return "https://www.youtube.com/feed/trending";
+    }
+
+    @Override
+    public String getId(String url) {
+        return "Trending";
+    }
+
+    @Override
+    public String cleanUrl(String url) {
+        return getUrl("");
+    }
+
+    @Override
+    public boolean acceptUrl(String url) {
+        return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java

@@ -0,0 +1,52 @@
+package org.schabi.newpipe.extractor.stream;
+
+/*
+ * Created by Christian Schabesberger on 04.03.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * AudioStream.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.MediaFormat;
+
+public class AudioStream extends Stream {
+    public int average_bitrate = -1;
+
+    /**
+     * Create a new audio stream
+     * @param url the url
+     * @param format the format
+     * @param averageBitrate the average bitrate
+     */
+    public AudioStream(String url, MediaFormat format, int averageBitrate) {
+        super(url, format);
+        this.average_bitrate = averageBitrate;
+    }
+
+    @Override
+    public boolean equalStats(Stream cmp) {
+        return super.equalStats(cmp) && cmp instanceof AudioStream &&
+                average_bitrate == ((AudioStream) cmp).average_bitrate;
+    }
+
+    /**
+     * Get the average bitrate
+     * @return the average bitrate or -1
+     */
+    public int getAverageBitrate() {
+        return average_bitrate;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java

@@ -0,0 +1,60 @@
+package org.schabi.newpipe.extractor.stream;
+
+import org.schabi.newpipe.extractor.MediaFormat;
+
+import java.io.Serializable;
+import java.util.List;
+
+public abstract class Stream implements Serializable {
+    private final MediaFormat mediaFormat;
+    public final String url;
+
+    /**
+     * @deprecated Use {@link #getFormat()}  or {@link #getFormatId()}
+     */
+    @Deprecated
+    public final int format;
+
+    public Stream(String url, MediaFormat format) {
+        this.url = url;
+        this.format = format.id;
+        this.mediaFormat = format;
+    }
+
+    /**
+     * Reveals whether two streams have the same stats (format and bitrate, for example)
+     */
+    public boolean equalStats(Stream cmp) {
+        return cmp != null && getFormatId() == cmp.getFormatId();
+    }
+
+    /**
+     * Reveals whether two Streams are equal
+     */
+    public boolean equals(Stream cmp) {
+        return equalStats(cmp) && url.equals(cmp.url);
+    }
+
+    /**
+     * Check if the list already contains one stream with equals stats
+     */
+    public static boolean containSimilarStream(Stream stream, List<? extends Stream> streamList) {
+        if (stream == null || streamList == null) return false;
+        for (Stream cmpStream : streamList) {
+            if (stream.equalStats(cmpStream)) return true;
+        }
+        return false;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public MediaFormat getFormat() {
+        return mediaFormat;
+    }
+
+    public int getFormatId() {
+        return mediaFormat.id;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -0,0 +1,150 @@
+package org.schabi.newpipe.extractor.stream;
+
+/*
+ * Created by Christian Schabesberger on 10.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * StreamExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.Extractor;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.Subtitles;
+import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Scrapes information from a video streaming service (eg, YouTube).
+ */
+public abstract class StreamExtractor extends Extractor {
+
+    public static final int NO_AGE_LIMIT = 0;
+
+    public StreamExtractor(StreamingService service, String url) {
+        super(service, url);
+    }
+
+    @Nonnull
+    @Override
+    protected UrlIdHandler getUrlIdHandler()  {
+        return getService().getStreamUrlIdHandler();
+    }
+
+    @Nonnull
+    public abstract String getUploadDate() throws ParsingException;
+    @Nonnull
+    public abstract String getThumbnailUrl() throws ParsingException;
+    @Nonnull
+    public abstract String getDescription() throws ParsingException;
+
+    /**
+     * Get the age limit
+     * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit
+     * @throws ParsingException if an error occurs while parsing
+     */
+    public abstract int getAgeLimit() throws ParsingException;
+
+    public abstract long getLength() throws ParsingException;
+    public abstract long getTimeStamp() throws ParsingException;
+    protected long getTimestampSeconds(String regexPattern) throws ParsingException {
+        String timeStamp;
+        try {
+            timeStamp = Parser.matchGroup1(regexPattern, getOriginalUrl());
+        } catch (Parser.RegexException e) {
+            // catch this instantly since an url does not necessarily have to have a time stamp
+
+            // -2 because well the testing system will then know its the regex that failed :/
+            // not good i know
+            return -2;
+        }
+
+        if (!timeStamp.isEmpty()) {
+            try {
+                String secondsString = "";
+                String minutesString = "";
+                String hoursString = "";
+                try {
+                    secondsString = Parser.matchGroup1("(\\d{1,3})s", timeStamp);
+                    minutesString = Parser.matchGroup1("(\\d{1,3})m", timeStamp);
+                    hoursString = Parser.matchGroup1("(\\d{1,3})h", timeStamp);
+                } catch (Exception e) {
+                    //it could be that time is given in another method
+                    if (secondsString.isEmpty() //if nothing was got,
+                            && minutesString.isEmpty()//treat as unlabelled seconds
+                            && hoursString.isEmpty()) {
+                        secondsString = Parser.matchGroup1("t=(\\d+)", timeStamp);
+                    }
+                }
+
+                int seconds = secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString);
+                int minutes = minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString);
+                int hours = hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString);
+
+                //don't trust BODMAS!
+                return seconds + (60 * minutes) + (3600 * hours);
+                //Log.d(TAG, "derived timestamp value:"+ret);
+                //the ordering varies internationally
+            } catch (ParsingException e) {
+                throw new ParsingException("Could not get timestamp.", e);
+            }
+        } else {
+            return 0;
+        }};
+
+    public abstract long getViewCount() throws ParsingException;
+    public abstract long getLikeCount() throws ParsingException;
+    public abstract long getDislikeCount() throws ParsingException;
+
+    @Nonnull
+    public abstract String getUploaderUrl() throws ParsingException;
+    @Nonnull
+    public abstract String getUploaderName() throws ParsingException;
+    @Nonnull
+    public abstract String getUploaderAvatarUrl() throws ParsingException;
+
+    /**
+     * Get the dash mpd url
+     * @return the url as a string or an empty string
+     * @throws ParsingException if an error occurs while reading
+     */
+    @Nonnull public abstract String getDashMpdUrl() throws ParsingException;
+    @Nonnull public abstract String getHlsUrl() throws ParsingException;
+    public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
+    public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
+    public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
+
+    @Nonnull
+    public abstract List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException;
+    @Nonnull
+    public abstract List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException;
+
+    public abstract StreamType getStreamType() throws ParsingException;
+    public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException;
+    public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
+
+    /**
+     * Analyses the webpage's document and extracts any error message there might be.
+     *
+     * @return Error message; null if there is no error message.
+     */
+    public abstract String getErrorMessage();
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -0,0 +1,468 @@
+package org.schabi.newpipe.extractor.stream;
+
+import org.schabi.newpipe.extractor.*;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.utils.DashMpdParser;
+import org.schabi.newpipe.extractor.utils.ExtractorHelper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/*
+ * Created by Christian Schabesberger on 26.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * StreamInfo.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Info object for opened videos, ie the video ready to play.
+ */
+public class StreamInfo extends Info {
+
+    public static class StreamExtractException extends ExtractionException {
+        StreamExtractException(String message) {
+            super(message);
+        }
+    }
+
+    public StreamInfo(int serviceId, String url, StreamType streamType, String id, String name, int ageLimit) {
+        super(serviceId, id, url, name);
+        this.streamType = streamType;
+        this.ageLimit = ageLimit;
+    }
+
+    public static StreamInfo getInfo(String url) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url);
+    }
+
+    public static StreamInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
+        return getInfo(service.getStreamExtractor(url));
+    }
+
+    private static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
+        extractor.fetchPage();
+        StreamInfo streamInfo;
+        try {
+            streamInfo = extractImportantData(extractor);
+            streamInfo = extractStreams(streamInfo, extractor);
+            streamInfo = extractOptionalData(streamInfo, extractor);
+        } catch (ExtractionException e) {
+            // Currently YouTube does not distinguish between age restricted videos and videos blocked
+            // by country.  This means that during the initialisation of the extractor, the extractor
+            // will assume that a video is age restricted while in reality it it blocked by country.
+            //
+            // We will now detect whether the video is blocked by country or not.
+            String errorMsg = extractor.getErrorMessage();
+
+            if (errorMsg != null) {
+                throw new ContentNotAvailableException(errorMsg);
+            } else {
+                throw e;
+            }
+        }
+
+        return streamInfo;
+    }
+
+    private static StreamInfo extractImportantData(StreamExtractor extractor) throws ExtractionException {
+        /* ---- important data, without the video can't be displayed goes here: ---- */
+        // if one of these is not available an exception is meant to be thrown directly into the frontend.
+
+        int serviceId = extractor.getServiceId();
+        String url = extractor.getCleanUrl();
+        StreamType streamType = extractor.getStreamType();
+        String id = extractor.getId();
+        String name = extractor.getName();
+        int ageLimit = extractor.getAgeLimit();
+
+        if ((streamType == StreamType.NONE)
+                || (url == null || url.isEmpty())
+                || (id == null || id.isEmpty())
+                || (name == null /* streamInfo.title can be empty of course */)
+                || (ageLimit == -1)) {
+            throw new ExtractionException("Some important stream information was not given.");
+        }
+
+        return new StreamInfo(serviceId, url, streamType, id, name, ageLimit);
+    }
+
+    private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
+        /* ---- stream extraction goes here ---- */
+        // At least one type of stream has to be available,
+        // otherwise an exception will be thrown directly into the frontend.
+
+        try {
+            streamInfo.setDashMpdUrl(extractor.getDashMpdUrl());
+        } catch (Exception e) {
+            streamInfo.addError(new ExtractionException("Couldn't get Dash manifest", e));
+        }
+
+        try {
+            streamInfo.setHlsUrl(extractor.getHlsUrl());
+        } catch (Exception e) {
+            streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e));
+        }
+
+        /*  Load and extract audio */
+        try {
+            streamInfo.setAudioStreams(extractor.getAudioStreams());
+        } catch (Exception e) {
+            streamInfo.addError(new ExtractionException("Couldn't get audio streams", e));
+        }
+        /* Extract video stream url*/
+        try {
+            streamInfo.setVideoStreams(extractor.getVideoStreams());
+        } catch (Exception e) {
+            streamInfo.addError(new ExtractionException("Couldn't get video streams", e));
+        }
+        /* Extract video only stream url*/
+        try {
+            streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams());
+        } catch (Exception e) {
+            streamInfo.addError(new ExtractionException("Couldn't get video only streams", e));
+        }
+
+        // Lists can be null if a exception was thrown during extraction
+        if (streamInfo.getVideoStreams() == null) streamInfo.setVideoStreams(new ArrayList<VideoStream>());
+        if (streamInfo.getVideoOnlyStreams() == null) streamInfo.setVideoOnlyStreams(new ArrayList<VideoStream>());
+        if (streamInfo.getAudioStreams() == null) streamInfo.setAudioStreams(new ArrayList<AudioStream>());
+
+        Exception dashMpdError = null;
+        if (streamInfo.getDashMpdUrl() != null && !streamInfo.getDashMpdUrl().isEmpty()) {
+            try {
+                DashMpdParser.getStreams(streamInfo);
+            } catch (Exception e) {
+                // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl),
+                // just skip the exception (but store it somewhere), as we later check if we have streams anyway.
+                dashMpdError = e;
+            }
+        }
+
+        // Either audio or video has to be available, otherwise we didn't get a stream (since videoOnly are optional, they don't count).
+        if ((streamInfo.videoStreams.isEmpty())
+                && (streamInfo.audioStreams.isEmpty())) {
+
+            if (dashMpdError != null) {
+                // If we don't have any video or audio and the dashMpd 'errored', add it to the error list
+                // (it's optional and it don't get added automatically, but it's good to have some additional error context)
+                streamInfo.addError(dashMpdError);
+            }
+
+            throw new StreamExtractException("Could not get any stream. See error variable to get further details.");
+        }
+
+        return streamInfo;
+    }
+
+    private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
+        /*  ---- optional data goes here: ---- */
+        // If one of these fails, the frontend needs to handle that they are not available.
+        // Exceptions are therefore not thrown into the frontend, but stored into the error List,
+        // so the frontend can afterwards check where errors happened.
+
+        try {
+            streamInfo.setThumbnailUrl(extractor.getThumbnailUrl());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setDuration(extractor.getLength());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setUploaderName(extractor.getUploaderName());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setUploaderUrl(extractor.getUploaderUrl());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setDescription(extractor.getDescription());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setViewCount(extractor.getViewCount());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setUploadDate(extractor.getUploadDate());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setStartPosition(extractor.getTimeStamp());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setLikeCount(extractor.getLikeCount());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setDislikeCount(extractor.getDislikeCount());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setNextVideo(extractor.getNextVideo());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setSubtitles(extractor.getSubtitlesDefault());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+
+        streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
+        return streamInfo;
+    }
+
+    private StreamType streamType;
+    private String thumbnailUrl;
+    private String uploadDate;
+    private long duration = -1;
+    private int ageLimit = -1;
+    private String description;
+
+    private long viewCount = -1;
+    private long likeCount = -1;
+    private long dislikeCount = -1;
+
+    private String uploaderName;
+    private String uploaderUrl;
+    private String uploaderAvatarUrl;
+
+    private List<VideoStream> videoStreams;
+    private List<AudioStream> audioStreams;
+    private List<VideoStream> videoOnlyStreams;
+
+    private String dashMpdUrl;
+    private String hlsUrl;
+    private StreamInfoItem nextVideo;
+    private List<InfoItem> relatedStreams;
+
+    private long startPosition = 0;
+    private List<Subtitles> subtitles;
+
+    /**
+     * Get the stream type
+     *
+     * @return the stream type
+     */
+    public StreamType getStreamType() {
+        return streamType;
+    }
+
+    public void setStreamType(StreamType streamType) {
+        this.streamType = streamType;
+    }
+
+    /**
+     * Get the thumbnail url
+     *
+     * @return the thumbnail url as a string
+     */
+    public String getThumbnailUrl() {
+        return thumbnailUrl;
+    }
+
+    public void setThumbnailUrl(String thumbnailUrl) {
+        this.thumbnailUrl = thumbnailUrl;
+    }
+
+    public String getUploadDate() {
+        return uploadDate;
+    }
+
+    public void setUploadDate(String uploadDate) {
+        this.uploadDate = uploadDate;
+    }
+
+    /**
+     * Get the duration in seconds
+     *
+     * @return the duration in seconds
+     */
+    public long getDuration() {
+        return duration;
+    }
+
+    public void setDuration(long duration) {
+        this.duration = duration;
+    }
+
+    public int getAgeLimit() {
+        return ageLimit;
+    }
+
+    public void setAgeLimit(int ageLimit) {
+        this.ageLimit = ageLimit;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public long getViewCount() {
+        return viewCount;
+    }
+
+    public void setViewCount(long viewCount) {
+        this.viewCount = viewCount;
+    }
+
+    /**
+     * Get the number of likes.
+     *
+     * @return The number of likes or -1 if this information is not available
+     */
+    public long getLikeCount() {
+        return likeCount;
+    }
+
+    public void setLikeCount(long likeCount) {
+        this.likeCount = likeCount;
+    }
+
+    /**
+     * Get the number of dislikes.
+     *
+     * @return The number of likes or -1 if this information is not available
+     */
+    public long getDislikeCount() {
+        return dislikeCount;
+    }
+
+    public void setDislikeCount(long dislikeCount) {
+        this.dislikeCount = dislikeCount;
+    }
+
+    public String getUploaderName() {
+        return uploaderName;
+    }
+
+    public void setUploaderName(String uploaderName) {
+        this.uploaderName = uploaderName;
+    }
+
+    public String getUploaderUrl() {
+        return uploaderUrl;
+    }
+
+    public void setUploaderUrl(String uploaderUrl) {
+        this.uploaderUrl = uploaderUrl;
+    }
+
+    public String getUploaderAvatarUrl() {
+        return uploaderAvatarUrl;
+    }
+
+    public void setUploaderAvatarUrl(String uploaderAvatarUrl) {
+        this.uploaderAvatarUrl = uploaderAvatarUrl;
+    }
+
+    public List<VideoStream> getVideoStreams() {
+        return videoStreams;
+    }
+
+    public void setVideoStreams(List<VideoStream> videoStreams) {
+        this.videoStreams = videoStreams;
+    }
+
+    public List<AudioStream> getAudioStreams() {
+        return audioStreams;
+    }
+
+    public void setAudioStreams(List<AudioStream> audioStreams) {
+        this.audioStreams = audioStreams;
+    }
+
+    public List<VideoStream> getVideoOnlyStreams() {
+        return videoOnlyStreams;
+    }
+
+    public void setVideoOnlyStreams(List<VideoStream> videoOnlyStreams) {
+        this.videoOnlyStreams = videoOnlyStreams;
+    }
+
+    public String getDashMpdUrl() {
+        return dashMpdUrl;
+    }
+
+    public void setDashMpdUrl(String dashMpdUrl) {
+        this.dashMpdUrl = dashMpdUrl;
+    }
+
+    public String getHlsUrl() {
+        return hlsUrl;
+    }
+
+    public void setHlsUrl(String hlsUrl) {
+        this.hlsUrl = hlsUrl;
+    }
+
+    public StreamInfoItem getNextVideo() {
+        return nextVideo;
+    }
+
+    public void setNextVideo(StreamInfoItem nextVideo) {
+        this.nextVideo = nextVideo;
+    }
+
+    public List<InfoItem> getRelatedStreams() {
+        return relatedStreams;
+    }
+
+    public void setRelatedStreams(List<InfoItem> relatedStreams) {
+        this.relatedStreams = relatedStreams;
+    }
+
+    public long getStartPosition() {
+        return startPosition;
+    }
+
+    public void setStartPosition(long startPosition) {
+        this.startPosition = startPosition;
+    }
+
+    public List<Subtitles> getSubtitles() {
+        return subtitles;
+    }
+
+    public void setSubtitles(List<Subtitles> subtitles) {
+        this.subtitles = subtitles;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java

@@ -0,0 +1,103 @@
+package org.schabi.newpipe.extractor.stream;
+
+/*
+ * Created by Christian Schabesberger on 26.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * StreamInfoItem.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.InfoItem;
+
+/**
+ * Info object for previews of unopened videos, eg search results, related videos
+ */
+public class StreamInfoItem extends InfoItem {
+    private final StreamType streamType;
+
+    private String uploaderName;
+    private String uploadDate;
+    private long viewCount = -1;
+    private long duration = -1;
+
+    private String uploaderUrl = null;
+
+    public StreamInfoItem(int serviceId, String url, String name, StreamType streamType) {
+        super(InfoType.STREAM, serviceId, url, name);
+        this.streamType = streamType;
+    }
+
+    public StreamType getStreamType() {
+        return streamType;
+    }
+
+    public String getUploaderName() {
+        return uploaderName;
+    }
+
+    public void setUploaderName(String uploader_name) {
+        this.uploaderName = uploader_name;
+    }
+
+    public String getUploadDate() {
+        return uploadDate;
+    }
+
+    public void setUploadDate(String upload_date) {
+        this.uploadDate = upload_date;
+    }
+
+    public long getViewCount() {
+        return viewCount;
+    }
+
+    public void setViewCount(long view_count) {
+        this.viewCount = view_count;
+    }
+
+    public long getDuration() {
+        return duration;
+    }
+
+    public void setDuration(long duration) {
+        this.duration = duration;
+    }
+
+    public String getUploaderUrl() {
+        return uploaderUrl;
+    }
+
+    public void setUploaderUrl(String uploaderUrl) {
+        this.uploaderUrl = uploaderUrl;
+    }
+
+    @Override
+    public String toString() {
+        return "StreamInfoItem{" +
+                "streamType=" + streamType +
+                ", uploaderName='" + uploaderName + '\'' +
+                ", uploadDate='" + uploadDate + '\'' +
+                ", viewCount=" + viewCount +
+                ", duration=" + duration +
+                ", uploaderUrl='" + uploaderUrl + '\'' +
+                ", infoType=" + getInfoType() +
+                ", serviceId=" + getServiceId() +
+                ", url='" + getUrl() + '\'' +
+                ", name='" + getName() + '\'' +
+                ", thumbnailUrl='" + getThumbnailUrl() + '\'' +
+                '}';
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java

@@ -0,0 +1,73 @@
+package org.schabi.newpipe.extractor.stream;
+
+import org.schabi.newpipe.extractor.InfoItemExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+/*
+ * Created by Christian Schabesberger on 28.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * StreamInfoItemExtractor.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public interface StreamInfoItemExtractor extends InfoItemExtractor {
+
+
+    /**
+     * Get the stream type
+     * @return the stream type
+     * @throws ParsingException thrown if there is an error in the extraction
+     */
+    StreamType getStreamType() throws ParsingException;
+
+    /**
+     * Check if the stream is an ad.
+     * @return {@code true} if the stream is an ad.
+     * @throws ParsingException thrown if there is an error in the extraction
+     */
+    boolean isAd() throws ParsingException;
+
+    /**
+     * Get the stream duration in seconds
+     * @return the stream duration in seconds
+     * @throws ParsingException thrown if there is an error in the extraction
+     */
+    long getDuration() throws ParsingException;
+
+    /**
+     * Parses the number of views
+     * @return the number of views or -1 for live streams
+     * @throws ParsingException thrown if there is an error in the extraction
+     */
+    long getViewCount() throws ParsingException;
+
+    /**
+     * Get the uploader name
+     * @return the uploader name
+     * @throws ParsingException if parsing fails
+     */
+    String getUploaderName() throws ParsingException;
+
+    String getUploaderUrl() throws ParsingException;
+
+    /**
+     * Extract the uploader name
+     * @return the uploader name
+     * @throws ParsingException thrown if there is an error in the extraction
+     */
+    String getUploadDate() throws ParsingException;
+
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java

@@ -0,0 +1,106 @@
+package org.schabi.newpipe.extractor.stream;
+
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.InfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.FoundAdException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.util.List;
+import java.util.Vector;
+
+/*
+ * Created by Christian Schabesberger on 28.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * StreamInfoItemsCollector.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class StreamInfoItemsCollector extends InfoItemsCollector<StreamInfoItem, StreamInfoItemExtractor> {
+
+    public StreamInfoItemsCollector(int serviceId) {
+        super(serviceId);
+    }
+
+    @Override
+    public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws ParsingException {
+        if (extractor.isAd()) {
+            throw new FoundAdException("Found ad");
+        }
+
+        // important information
+        int serviceId = getServiceId();
+        String url = extractor.getUrl();
+        String name = extractor.getName();
+        StreamType streamType = extractor.getStreamType();
+
+        StreamInfoItem resultItem = new StreamInfoItem(serviceId, url, name, streamType);
+
+
+        // optional information
+        try {
+            resultItem.setDuration(extractor.getDuration());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setUploaderName(extractor.getUploaderName());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setUploadDate(extractor.getUploadDate());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setViewCount(extractor.getViewCount());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setThumbnailUrl(extractor.getThumbnailUrl());
+        } catch (Exception e) {
+            addError(e);
+        }
+        try {
+            resultItem.setUploaderUrl(extractor.getUploaderUrl());
+        } catch (Exception e) {
+            addError(e);
+        }
+        return resultItem;
+    }
+
+    @Override
+    public void commit(StreamInfoItemExtractor extractor) {
+        try {
+            addItem(extract(extractor));
+        } catch (FoundAdException ae) {
+            //System.out.println("AD_WARNING: " + ae.getMessage());
+        } catch (Exception e) {
+            addError(e);
+        }
+    }
+
+    public List<StreamInfoItem> getStreamInfoItemList() {
+        List<StreamInfoItem> siiList = new Vector<>();
+        for(InfoItem ii : super.getItems()) {
+            if(ii instanceof StreamInfoItem) {
+                siiList.add((StreamInfoItem) ii);
+            }
+        }
+        return siiList;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamType.java

@@ -0,0 +1,10 @@
+package org.schabi.newpipe.extractor.stream;
+
+public enum StreamType {
+    NONE,   // placeholder to check if stream type was checked or not
+    VIDEO_STREAM,
+    AUDIO_STREAM,
+    LIVE_STREAM,
+    AUDIO_LIVE_STREAM,
+    FILE
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java

@@ -0,0 +1,25 @@
+package org.schabi.newpipe.extractor.stream;
+
+public enum SubtitlesFormat {
+    // YouTube subtitles formats
+    // TRANSCRIPT(3) is default YT format based on TTML,
+    // but unlike VTT or TTML, it is NOT W3 standard
+    // TRANSCRIPT subtitles are NOT supported by ExoPlayer, only VTT and TTML
+    VTT (0x0, "vtt"),
+    TTML (0x1, "ttml"),
+    TRANSCRIPT1 (0x2, "srv1"),
+    TRANSCRIPT2 (0x3, "srv2"),
+    TRANSCRIPT3 (0x4, "srv3");
+
+    private final int id;
+    private final String extension;
+
+    SubtitlesFormat(int id, String extension) {
+        this.id = id;
+        this.extension = extension;
+    }
+
+    public String getExtension() {
+        return extension;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java

@@ -0,0 +1,64 @@
+package org.schabi.newpipe.extractor.stream;
+
+/*
+ * Created by Christian Schabesberger on 04.03.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * VideoStream.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.schabi.newpipe.extractor.MediaFormat;
+
+public class VideoStream extends Stream {
+    public final String resolution;
+    public final boolean isVideoOnly;
+
+
+    public VideoStream(String url, MediaFormat format, String resolution) {
+        this(url, format, resolution, false);
+    }
+
+    public VideoStream(String url, MediaFormat format, String resolution, boolean isVideoOnly) {
+        super(url, format);
+        this.resolution = resolution;
+        this.isVideoOnly = isVideoOnly;
+    }
+
+    @Override
+    public boolean equalStats(Stream cmp) {
+        return super.equalStats(cmp) && cmp instanceof VideoStream &&
+                resolution.equals(((VideoStream) cmp).resolution) &&
+                isVideoOnly == ((VideoStream) cmp).isVideoOnly;
+    }
+
+    /**
+     * Get the video resolution
+     * @return the video resolution
+     */
+    public String getResolution() {
+        return resolution;
+    }
+
+    /**
+     * Check if the video is video only.
+     *
+     * Video only streams have no audio
+     * @return {@code true} if this stream is vid
+     */
+    public boolean isVideoOnly() {
+        return isVideoOnly;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java

@@ -0,0 +1,78 @@
+package org.schabi.newpipe.extractor.subscription;
+
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+
+public abstract class SubscriptionExtractor {
+
+    /**
+     * Exception that should be thrown when the input <b>do not</b> contain valid content that the
+     * extractor can parse (e.g. nonexistent user in case of a url extraction).
+     */
+    public static class InvalidSourceException extends ParsingException {
+        public InvalidSourceException() {
+            this(null, null);
+        }
+
+        public InvalidSourceException(String detailMessage) {
+            this(detailMessage, null);
+        }
+
+        public InvalidSourceException(Throwable cause) {
+            this(null, cause);
+        }
+
+        public InvalidSourceException(String detailMessage, Throwable cause) {
+            super(detailMessage == null ? "Not a valid source" : "Not a valid source (" + detailMessage + ")", cause);
+        }
+    }
+
+    public enum ContentSource {
+        CHANNEL_URL, INPUT_STREAM
+    }
+
+    private final List<ContentSource> supportedSources;
+    protected final StreamingService service;
+
+    public SubscriptionExtractor(StreamingService service, List<ContentSource> supportedSources) {
+        this.service = service;
+        this.supportedSources = Collections.unmodifiableList(supportedSources);
+    }
+
+    public List<ContentSource> getSupportedSources() {
+        return supportedSources;
+    }
+
+    /**
+     * Returns an url that can help/guide the user to the file (or channel url) to extract the subscriptions.
+     * <p>For example, in YouTube, the export subscriptions url is a good choice to return here.</p>
+     */
+    @Nullable
+    public abstract String getRelatedUrl();
+
+    /**
+     * Reads and parse a list of {@link SubscriptionItem} from the given channel url.
+     *
+     * @throws InvalidSourceException when the channelUrl doesn't exist or is invalid
+     */
+    public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
+        throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from a channel url");
+    }
+
+    /**
+     * Reads and parse a list of {@link SubscriptionItem} from the given InputStream.
+     *
+     * @throws InvalidSourceException when the content read from the InputStream is invalid and can not be parsed
+     */
+    @SuppressWarnings("RedundantThrows")
+    public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws IOException, ExtractionException {
+        throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from an InputStream");
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionItem.java

@@ -0,0 +1,32 @@
+package org.schabi.newpipe.extractor.subscription;
+
+import java.io.Serializable;
+
+public class SubscriptionItem implements Serializable {
+    private final int serviceId;
+    private final String url, name;
+
+    public SubscriptionItem(int serviceId, String url, String name) {
+        this.serviceId = serviceId;
+        this.url = url;
+        this.name = name;
+    }
+
+    public int getServiceId() {
+        return serviceId;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "@" + Integer.toHexString(hashCode()) +
+                "[name=" + name + " > " + serviceId + ":" + url + "]";
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java

@@ -0,0 +1,118 @@
+package org.schabi.newpipe.extractor.utils;
+
+import org.schabi.newpipe.extractor.Downloader;
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.Stream;
+import org.schabi.newpipe.extractor.stream.StreamInfo;
+import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/*
+ * Created by Christian Schabesberger on 02.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * DashMpdParser.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class DashMpdParser {
+
+    private DashMpdParser() {
+    }
+
+    public static class DashMpdParsingException extends ParsingException {
+        DashMpdParsingException(String message, Exception e) {
+            super(message, e);
+        }
+    }
+
+    /**
+     * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
+     * then it will search for any stream that the ItagItem has (by the id).
+     * <p>
+     * It has video, video only and audio streams and will only add to the list if it don't
+     * find a similar stream in the respective lists (calling {@link Stream#equalStats}).
+     *
+     * @param streamInfo where the parsed streams will be added
+     */
+    public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
+        String dashDoc;
+        Downloader downloader = NewPipe.getDownloader();
+        try {
+            dashDoc = downloader.download(streamInfo.getDashMpdUrl());
+        } catch (IOException ioe) {
+            throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
+        } catch (ReCaptchaException e) {
+            throw new ReCaptchaException("reCaptcha Challenge needed");
+        }
+
+        try {
+            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+            DocumentBuilder builder = factory.newDocumentBuilder();
+            InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
+
+            Document doc = builder.parse(stream);
+            NodeList representationList = doc.getElementsByTagName("Representation");
+
+            for (int i = 0; i < representationList.getLength(); i++) {
+                Element representation = ((Element) representationList.item(i));
+                try {
+                    String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType");
+                    String id = representation.getAttribute("id");
+                    String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
+                    ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
+                    if (itag != null) {
+                        MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
+
+                        if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
+                            AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
+
+                            if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
+                                streamInfo.getAudioStreams().add(audioStream);
+                            }
+                        } else {
+                            boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
+                            VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly);
+
+                            if (isVideoOnly) {
+                                if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
+                                    streamInfo.getVideoOnlyStreams().add(videoStream);
+                                }
+                            } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
+                                streamInfo.getVideoStreams().add(videoStream);
+                            }
+                        }
+                    }
+                } catch (Exception ignored) {
+                }
+            }
+        } catch (Exception e) {
+            throw new DashMpdParsingException("Could not parse Dash mpd", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java

@@ -0,0 +1,42 @@
+package org.schabi.newpipe.extractor.utils;
+
+import org.schabi.newpipe.extractor.Info;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.InfoItemsCollector;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfo;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ExtractorHelper {
+    private ExtractorHelper() {}
+
+    public static <T extends InfoItem> InfoItemsPage<T> getItemsPageOrLogError(Info info, ListExtractor<T> extractor) {
+        try {
+            InfoItemsPage<T> page = extractor.getInitialPage();
+            info.addAllErrors(page.getErrors());
+
+            return page;
+        } catch (Exception e) {
+            info.addError(e);
+            return InfoItemsPage.emptyPage();
+        }
+    }
+
+
+    public static List<InfoItem> getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) {
+        try {
+            InfoItemsCollector<? extends InfoItem, ?> collector = extractor.getRelatedVideos();
+            info.addAllErrors(collector.getErrors());
+
+            //noinspection unchecked
+            return (List<InfoItem>) collector.getItems();
+        } catch (Exception e) {
+            info.addError(e);
+            return Collections.emptyList();
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -0,0 +1,83 @@
+package org.schabi.newpipe.extractor.utils;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/*
+ * Created by Christian Schabesberger on 02.02.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * Parser.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * avoid using regex !!!
+ */
+public class Parser {
+
+    private Parser() {
+    }
+
+    public static class RegexException extends ParsingException {
+        public RegexException(String message) {
+            super(message);
+        }
+    }
+
+    public static String matchGroup1(String pattern, String input) throws RegexException {
+        return matchGroup(pattern, input, 1);
+    }
+
+    public static String matchGroup(String pattern, String input, int group) throws RegexException {
+        Pattern pat = Pattern.compile(pattern);
+        Matcher mat = pat.matcher(input);
+        boolean foundMatch = mat.find();
+        if (foundMatch) {
+            return mat.group(group);
+        } else {
+            if (input.length() > 1024) {
+                throw new RegexException("failed to find pattern \"" + pattern);
+            } else {
+                throw new RegexException("failed to find pattern \"" + pattern + " inside of " + input + "\"");
+            }
+        }
+    }
+
+    public static boolean isMatch(String pattern, String input) {
+        Pattern pat = Pattern.compile(pattern);
+        Matcher mat = pat.matcher(input);
+        return mat.find();
+    }
+
+    public static Map<String, String> compatParseMap(final String input) throws UnsupportedEncodingException {
+        Map<String, String> map = new HashMap<>();
+        for (String arg : input.split("&")) {
+            String[] splitArg = arg.split("=");
+            if (splitArg.length > 1) {
+                map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
+            } else {
+                map.put(splitArg[0], "");
+            }
+        }
+        return map;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -0,0 +1,61 @@
+package org.schabi.newpipe.extractor.utils;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.util.List;
+
+public class Utils {
+
+    private Utils() {
+        //no instance
+    }
+
+    /**
+     * Remove all non-digit characters from a string.<p>
+     * Examples:<p>
+     * <ul><li>1 234 567 views -&gt; 1234567</li>
+     * <li>$31,133.124 -&gt; 31133124</li></ul>
+     *
+     * @param toRemove string to remove non-digit chars
+     * @return a string that contains only digits
+     */
+    public static String removeNonDigitCharacters(String toRemove) {
+        return toRemove.replaceAll("\\D+", "");
+    }
+
+    /**
+     * Check if the url matches the pattern.
+     *
+     * @param pattern the pattern that will be used to check the url
+     * @param url     the url to be tested
+     */
+    public static void checkUrl(String pattern, String url) throws ParsingException {
+        if (url == null || url.isEmpty()) {
+            throw new IllegalArgumentException("Url can't be null or empty");
+        }
+
+        if (!Parser.isMatch(pattern, url.toLowerCase())) {
+            throw new ParsingException("Url don't match the pattern");
+        }
+    }
+
+    public static void printErrors(List<Throwable> errors) {
+        for(Throwable e : errors) {
+            e.printStackTrace();
+            System.err.println("----------------");
+        }
+    }
+
+    private static final String HTTP = "http://";
+    private static final String HTTPS = "https://";
+
+    public static String replaceHttpWithHttps(final String url) {
+        if (url == null) return null;
+
+        if(!url.isEmpty() && url.startsWith(HTTP)) {
+            return HTTPS + url.substring(HTTP.length());
+        }
+        return url;
+    }
+}
+

extractor/src/test/java/org/schabi/newpipe/Downloader.java

@@ -0,0 +1,157 @@
+package org.schabi.newpipe;
+
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+
+import javax.net.ssl.HttpsURLConnection;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/*
+ * Created by Christian Schabesberger on 28.01.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * Downloader.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class Downloader implements org.schabi.newpipe.extractor.Downloader {
+
+    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
+    private static String mCookies = "";
+
+    private static Downloader instance = null;
+
+    private Downloader() {
+    }
+
+    public static Downloader getInstance() {
+        if (instance == null) {
+            synchronized (Downloader.class) {
+                if (instance == null) {
+                    instance = new Downloader();
+                }
+            }
+        }
+        return instance;
+    }
+
+    public static synchronized void setCookies(String cookies) {
+        Downloader.mCookies = cookies;
+    }
+
+    public static synchronized String getCookies() {
+        return Downloader.mCookies;
+    }
+
+    /**
+     * Download the text file at the supplied URL as in download(String),
+     * but set the HTTP header field "Accept-Language" to the supplied string.
+     *
+     * @param siteUrl  the URL of the text file to return the contents of
+     * @param language the language (usually a 2-character code) to set as the preferred language
+     * @return the contents of the specified text file
+     */
+    public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
+        Map<String, String> requestProperties = new HashMap<>();
+        requestProperties.put("Accept-Language", language);
+        return download(siteUrl, requestProperties);
+    }
+
+
+    /**
+     * Download the text file at the supplied URL as in download(String),
+     * but set the HTTP header field "Accept-Language" to the supplied string.
+     *
+     * @param siteUrl          the URL of the text file to return the contents of
+     * @param customProperties set request header properties
+     * @return the contents of the specified text file
+     * @throws IOException
+     */
+    public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
+        URL url = new URL(siteUrl);
+        HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
+        for (Map.Entry<String, String> pair: customProperties.entrySet()) {
+            con.setRequestProperty(pair.getKey(), pair.getValue());
+        }
+        return dl(con);
+    }
+
+    /**
+     * Common functionality between download(String url) and download(String url, String language)
+     */
+    private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
+        StringBuilder response = new StringBuilder();
+        BufferedReader in = null;
+
+        try {
+            con.setConnectTimeout(30 * 1000);// 30s
+            con.setReadTimeout(30 * 1000);// 30s
+            con.setRequestMethod("GET");
+            con.setRequestProperty("User-Agent", USER_AGENT);
+
+            if (getCookies().length() > 0) {
+                con.setRequestProperty("Cookie", getCookies());
+            }
+
+            in = new BufferedReader(
+                    new InputStreamReader(con.getInputStream()));
+            String inputLine;
+
+            while ((inputLine = in.readLine()) != null) {
+                response.append(inputLine);
+            }
+        } catch (UnknownHostException uhe) {//thrown when there's no internet connection
+            throw new IOException("unknown host or no network", uhe);
+            //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
+        } catch (Exception e) {
+            /*
+             * HTTP 429 == Too Many Request
+             * Receive from Youtube.com = ReCaptcha challenge request
+             * See : https://github.com/rg3/youtube-dl/issues/5138
+             */
+            if (con.getResponseCode() == 429) {
+                throw new ReCaptchaException("reCaptcha Challenge requested");
+            }
+
+            throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
+        } finally {
+            if (in != null) {
+                in.close();
+            }
+        }
+
+        return response.toString();
+    }
+
+    /**
+     * Download (via HTTP) the text file located at the supplied URL, and return its contents.
+     * Primarily intended for downloading web pages.
+     *
+     * @param siteUrl the URL of the text file to download
+     * @return the contents of the specified text file
+     */
+    public String download(String siteUrl) throws IOException, ReCaptchaException {
+        URL url = new URL(siteUrl);
+        HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
+        //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
+        return dl(con);
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java

@@ -0,0 +1,59 @@
+package org.schabi.newpipe.extractor;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class ExtractorAsserts {
+    public static void assertEmptyErrors(String message, List<Throwable> errors) {
+        if (!errors.isEmpty()) {
+            StringBuilder messageBuilder = new StringBuilder(message);
+            for (Throwable e : errors) {
+                messageBuilder.append("\n  * ").append(e.getMessage());
+            }
+            messageBuilder.append(" ");
+            throw new AssertionError(messageBuilder.toString(), errors.get(0));
+        }
+    }
+
+    @Nonnull
+    private static URL urlFromString(String url) {
+        try {
+            return new URL(url);
+        } catch (MalformedURLException e) {
+            throw new AssertionError("Invalid url: " + "\"" + url + "\"", e);
+        }
+    }
+
+    public static void assertIsValidUrl(String url) {
+        urlFromString(url);
+    }
+
+    public static void assertIsSecureUrl(String urlToCheck) {
+        URL url = urlFromString(urlToCheck);
+        assertEquals("Protocol of URL is not secure", "https", url.getProtocol());
+    }
+
+    public static void assertNotEmpty(String stringToCheck) {
+        assertNotEmpty(null, stringToCheck);
+    }
+
+    public static void assertNotEmpty(@Nullable String message, String stringToCheck) {
+        assertNotNull(message, stringToCheck);
+        assertFalse(message, stringToCheck.isEmpty());
+    }
+
+    public static void assertEmpty(String stringToCheck) {
+        assertEmpty(null, stringToCheck);
+    }
+
+    public static void assertEmpty(@Nullable String message, String stringToCheck) {
+        if (stringToCheck != null) {
+            assertTrue(message, stringToCheck.isEmpty());
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java

@@ -0,0 +1,72 @@
+package org.schabi.newpipe.extractor;
+
+import org.junit.Test;
+
+import java.util.HashSet;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+public class NewPipeTest {
+    @Test
+    public void getAllServicesTest() throws Exception {
+        assertEquals(NewPipe.getServices().size(), ServiceList.all().size());
+    }
+
+    @Test
+    public void testAllServicesHaveDifferentId() throws Exception {
+        HashSet<Integer> servicesId = new HashSet<>();
+        for (StreamingService streamingService : NewPipe.getServices()) {
+            String errorMsg = "There are services with the same id = " + streamingService.getServiceId() + " (current service > " + streamingService.getServiceInfo().getName() + ")";
+
+            assertTrue(errorMsg, servicesId.add(streamingService.getServiceId()));
+        }
+    }
+
+    @Test
+    public void getServiceWithId() throws Exception {
+        assertEquals(NewPipe.getService(YouTube.getServiceId()), YouTube);
+        assertEquals(NewPipe.getService(SoundCloud.getServiceId()), SoundCloud);
+
+        assertNotEquals(NewPipe.getService(SoundCloud.getServiceId()), YouTube);
+    }
+
+    @Test
+    public void getServiceWithName() throws Exception {
+        assertEquals(NewPipe.getService(YouTube.getServiceInfo().getName()), YouTube);
+        assertEquals(NewPipe.getService(SoundCloud.getServiceInfo().getName()), SoundCloud);
+
+        assertNotEquals(NewPipe.getService(YouTube.getServiceInfo().getName()), SoundCloud);
+    }
+
+    @Test
+    public void getServiceWithUrl() throws Exception {
+        assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), YouTube);
+        assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), YouTube);
+        assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), YouTube);
+        assertEquals(getServiceByUrl("https://soundcloud.com/shupemoosic/pegboard-nerds-try-this"), SoundCloud);
+        assertEquals(getServiceByUrl("https://soundcloud.com/deluxe314/sets/pegboard-nerds"), SoundCloud);
+        assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud);
+
+        assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), YouTube);
+        assertNotEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), SoundCloud);
+    }
+
+    @Test
+    public void getIdWithServiceName() throws Exception {
+        assertEquals(NewPipe.getIdOfService(YouTube.getServiceInfo().getName()), YouTube.getServiceId());
+        assertEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().getName()), SoundCloud.getServiceId());
+
+        assertNotEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().getName()), YouTube.getServiceId());
+    }
+
+    @Test
+    public void getServiceNameWithId() throws Exception {
+        assertEquals(NewPipe.getNameOfService(YouTube.getServiceId()), YouTube.getServiceInfo().getName());
+        assertEquals(NewPipe.getNameOfService(SoundCloud.getServiceId()), SoundCloud.getServiceInfo().getName());
+
+        assertNotEquals(NewPipe.getNameOfService(YouTube.getServiceId()), SoundCloud.getServiceInfo().getName());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java

@@ -0,0 +1,10 @@
+package org.schabi.newpipe.extractor.services;
+
+@SuppressWarnings("unused")
+public interface BaseChannelExtractorTest extends BaseListExtractorTest {
+    void testDescription() throws Exception;
+    void testAvatarUrl() throws Exception;
+    void testBannerUrl() throws Exception;
+    void testFeedUrl() throws Exception;
+    void testSubscriberCount() throws Exception;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java

@@ -0,0 +1,10 @@
+package org.schabi.newpipe.extractor.services;
+
+@SuppressWarnings("unused")
+public interface BaseExtractorTest {
+    void testServiceId() throws Exception;
+    void testName() throws Exception;
+    void testId() throws Exception;
+    void testCleanUrl() throws Exception;
+    void testOriginalUrl() throws Exception;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseListExtractorTest.java

@@ -0,0 +1,7 @@
+package org.schabi.newpipe.extractor.services;
+
+@SuppressWarnings("unused")
+public interface BaseListExtractorTest extends BaseExtractorTest {
+    void testRelatedItems() throws Exception;
+    void testMoreRelatedItems() throws Exception;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java

@@ -0,0 +1,11 @@
+package org.schabi.newpipe.extractor.services;
+
+@SuppressWarnings("unused")
+public interface BasePlaylistExtractorTest extends BaseListExtractorTest {
+    void testThumbnailUrl() throws Exception;
+    void testBannerUrl() throws Exception;
+    void testUploaderUrl() throws Exception;
+    void testUploaderName() throws Exception;
+    void testUploaderAvatarUrl() throws Exception;
+    void testStreamCount() throws Exception;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java

@@ -0,0 +1,60 @@
+package org.schabi.newpipe.extractor.services;
+
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.*;
+
+public final class DefaultTests {
+    public static void defaultTestListOfItems(int expectedServiceId, List<? extends InfoItem> itemsList, List<Throwable> errors) {
+        assertTrue("List of items is empty", !itemsList.isEmpty());
+        assertFalse("List of items contains a null element", itemsList.contains(null));
+        assertEmptyErrors("Errors during stream list extraction", errors);
+
+        for (InfoItem item : itemsList) {
+            assertIsSecureUrl(item.getUrl());
+            if (item.getThumbnailUrl() != null && !item.getThumbnailUrl().isEmpty()) {
+                assertIsSecureUrl(item.getThumbnailUrl());
+            }
+            assertNotNull("InfoItem type not set: " + item, item.getInfoType());
+            assertEquals("Service id doesn't match: " + item, expectedServiceId, item.getServiceId());
+
+            if (item instanceof StreamInfoItem) {
+                StreamInfoItem streamInfoItem = (StreamInfoItem) item;
+                assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
+                assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
+            }
+        }
+    }
+
+    public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestRelatedItems(ListExtractor<T> extractor, int expectedServiceId) throws Exception {
+        final ListExtractor.InfoItemsPage<T> page = extractor.getInitialPage();
+        final List<T> itemsList = page.getItems();
+        List<Throwable> errors = page.getErrors();
+
+        defaultTestListOfItems(expectedServiceId, itemsList, errors);
+        return page;
+    }
+
+    public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor, int expectedServiceId) throws Exception {
+        assertTrue("Doesn't have more items", extractor.hasNextPage());
+        ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(extractor.getNextPageUrl());
+        final List<T> items = nextPage.getItems();
+        assertTrue("Next page is empty", !items.isEmpty());
+        assertEmptyErrors("Next page have errors", nextPage.getErrors());
+
+        defaultTestListOfItems(expectedServiceId, nextPage.getItems(), nextPage.getErrors());
+        return nextPage;
+    }
+
+    public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor, int expectedServiceId) throws Exception {
+        final String nextPageUrl = extractor.getNextPageUrl();
+
+        final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPageUrl);
+        defaultTestListOfItems(expectedServiceId, page.getItems(), page.getErrors());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/BaseSoundcloudSearchTest.java

@@ -0,0 +1,57 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.junit.Test;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
+import org.schabi.newpipe.extractor.search.SearchResult;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamType;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+
+public abstract class BaseSoundcloudSearchTest {
+
+    protected static SearchResult result;
+
+    @Test
+    public void testResultList() {
+        assertFalse("Got empty result list", result.resultList.isEmpty());
+        for(InfoItem infoItem: result.resultList) {
+            assertIsSecureUrl(infoItem.getUrl());
+            assertIsSecureUrl(infoItem.getThumbnailUrl());
+            assertFalse(infoItem.getName().isEmpty());
+            assertFalse("Name is probably a URI: " + infoItem.getName(),
+                    infoItem.getName().contains("://"));
+            if(infoItem instanceof StreamInfoItem) {
+                // test stream item
+                StreamInfoItem streamInfoItem = (StreamInfoItem) infoItem;
+                assertIsSecureUrl(streamInfoItem.getUploaderUrl());
+                assertFalse(streamInfoItem.getUploadDate().isEmpty());
+                assertFalse(streamInfoItem.getUploaderName().isEmpty());
+                assertEquals(StreamType.AUDIO_STREAM, streamInfoItem.getStreamType());
+            } else if(infoItem instanceof ChannelInfoItem) {
+                // Nothing special to check?
+            } else if(infoItem instanceof PlaylistInfoItem)  {
+                // test playlist item
+                assertTrue(infoItem.getUrl().contains("/sets/"));
+                long streamCount = ((PlaylistInfoItem) infoItem).getStreamCount();
+                assertTrue(streamCount > 0);
+            } else {
+                fail("Unknown infoItem type: " + infoItem);
+            }
+        }
+    }
+
+    @Test
+    public void testResultErrors() {
+        assertNotNull(result.errors);
+        if (!result.errors.isEmpty()) {
+            for (Throwable error : result.errors) {
+                error.printStackTrace();
+            }
+        }
+        assertTrue(result.errors.isEmpty());
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java

@@ -0,0 +1,197 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+import static org.schabi.newpipe.extractor.services.DefaultTests.*;
+
+/**
+ * Test for {@link SoundcloudChannelExtractor}
+ */
+public class SoundcloudChannelExtractorTest {
+    public static class LilUzi implements BaseChannelExtractorTest {
+        private static SoundcloudChannelExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(Downloader.getInstance());
+            extractor = (SoundcloudChannelExtractor) SoundCloud
+                    .getChannelExtractor("http://soundcloud.com/liluzivert/sets");
+            extractor.fetchPage();
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testServiceId() {
+            assertEquals(SoundCloud.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        public void testName() {
+            assertEquals("LIL UZI VERT", extractor.getName());
+        }
+
+        @Test
+        public void testId() {
+            assertEquals("10494998", extractor.getId());
+        }
+
+        @Test
+        public void testCleanUrl() {
+            assertEquals("https://soundcloud.com/liluzivert", extractor.getCleanUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() {
+            assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
+        }
+
+        @Test
+        public void testMoreRelatedItems() throws Exception {
+            defaultTestMoreItems(extractor, SoundCloud.getServiceId());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ChannelExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testDescription() {
+            assertNotNull(extractor.getDescription());
+        }
+
+        @Test
+        public void testAvatarUrl() {
+            assertIsSecureUrl(extractor.getAvatarUrl());
+        }
+
+        @Test
+        public void testBannerUrl() {
+            assertIsSecureUrl(extractor.getBannerUrl());
+        }
+
+        @Test
+        public void testFeedUrl() {
+            assertEmpty(extractor.getFeedUrl());
+        }
+
+        @Test
+        public void testSubscriberCount() {
+            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1e6);
+        }
+    }
+
+    public static class DubMatix implements BaseChannelExtractorTest {
+        private static SoundcloudChannelExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(Downloader.getInstance());
+            extractor = (SoundcloudChannelExtractor) SoundCloud
+                    .getChannelExtractor("https://soundcloud.com/dubmatix");
+            extractor.fetchPage();
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Additional Testing
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testGetPageInNewExtractor() throws Exception {
+            final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getCleanUrl());
+            defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testServiceId() {
+            assertEquals(SoundCloud.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        public void testName() {
+            assertEquals("dubmatix", extractor.getName());
+        }
+
+        @Test
+        public void testId() {
+            assertEquals("542134", extractor.getId());
+        }
+
+        @Test
+        public void testCleanUrl() {
+            assertEquals("https://soundcloud.com/dubmatix", extractor.getCleanUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() {
+            assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
+        }
+
+        @Test
+        public void testMoreRelatedItems() throws Exception {
+            defaultTestMoreItems(extractor, SoundCloud.getServiceId());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ChannelExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testDescription() {
+            assertNotNull(extractor.getDescription());
+        }
+
+        @Test
+        public void testAvatarUrl() {
+            assertIsSecureUrl(extractor.getAvatarUrl());
+        }
+
+        @Test
+        public void testBannerUrl() {
+            assertIsSecureUrl(extractor.getBannerUrl());
+        }
+
+        @Test
+        public void testFeedUrl() {
+            assertEmpty(extractor.getFeedUrl());
+        }
+
+        @Test
+        public void testSubscriberCount() {
+            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2e6);
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -0,0 +1,93 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+
+/**
+ * Test for {@link SoundcloudChartsUrlIdHandler}
+ */
+public class SoundcloudChartsExtractorTest {
+
+    static KioskExtractor extractor;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = SoundCloud
+                .getKioskList()
+                .getExtractorById("Top 50", null);
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void testGetDownloader() throws Exception {
+        assertNotNull(NewPipe.getDownloader());
+    }
+
+    @Ignore
+    @Test
+    public void testGetName() throws Exception {
+        assertEquals(extractor.getName(), "Top 50");
+    }
+
+    @Test
+    public void testId() {
+        assertEquals(extractor.getId(), "Top 50");
+    }
+
+    @Test
+    public void testGetStreams() throws Exception {
+        ListExtractor.InfoItemsPage<StreamInfoItem> page = extractor.getInitialPage();
+        if(!page.getErrors().isEmpty()) {
+            System.err.println("----------");
+            List<Throwable> errors = page.getErrors();
+            for(Throwable e: errors) {
+                e.printStackTrace();
+                System.err.println("----------");
+            }
+        }
+        assertTrue("no streams are received",
+                !page.getItems().isEmpty()
+                        && page.getErrors().isEmpty());
+    }
+
+    @Test
+    public void testGetStreamsErrors() throws Exception {
+        assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty());
+    }
+
+    @Test
+    public void testHasMoreStreams() throws Exception {
+        // Setup the streams
+        extractor.getInitialPage();
+        assertTrue("has more streams", extractor.hasNextPage());
+    }
+
+    @Test
+    public void testGetNextPageUrl() throws Exception {
+        assertTrue(extractor.hasNextPage());
+    }
+
+    @Test
+    public void testGetNextPage() throws Exception {
+        extractor.getInitialPage().getItems();
+        assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
+                || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty());
+    }
+
+    @Test
+    public void testGetCleanUrl() throws Exception {
+        assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top");
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java

@@ -0,0 +1,49 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for {@link SoundcloudChartsUrlIdHandler}
+ */
+public class SoundcloudChartsUrlIdHandlerTest {
+    private static SoundcloudChartsUrlIdHandler urlIdHandler;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        urlIdHandler = new SoundcloudChartsUrlIdHandler();
+        NewPipe.init(Downloader.getInstance());
+    }
+
+    @Test
+    public void getUrl() {
+        assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top");
+        assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new");
+    }
+
+    @Test
+    public void getId() {
+        assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50");
+        assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot");
+    }
+
+    @Test
+    public void acceptUrl() {
+        assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts"));
+        assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/"));
+        assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new"));
+        assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/charts/top?genre=all-music"));
+        assertTrue(urlIdHandler.acceptUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"));
+
+        assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia"));
+        assertFalse(urlIdHandler.acceptUrl("soundcloud.com/charts askjkf"));
+        assertFalse(urlIdHandler.acceptUrl("    soundcloud.com/charts"));
+        assertFalse(urlIdHandler.acceptUrl(""));
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java

@@ -0,0 +1,28 @@
+package org.schabi.newpipe.extractor.services.soundcloud;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+
+public class SoundcloudParsingHelperTest {
+    @BeforeClass
+    public static void setUp() {
+        NewPipe.init(Downloader.getInstance());
+    }
+
+    @Test
+    public void resolveUrlWithEmbedPlayerTest() throws Exception {
+        Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));
+        Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159"));
+    }
+
+    @Test
+    public void resolveIdWithEmbedPlayerTest() throws Exception {
+        Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity"));
+        Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds"));
+
+    }
+
+}