CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add SearchExtractor

Browse source
This commit is contained in:
Christian Schabesberger 2018-05-13 21:28:51 +02:00
parent 0501a2f543
commit b4544a67e8
21 changed files with 189 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java
View file

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor;
import edu.umd.cs.findbugs.annotations.NonNull;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -34,7 +33,7 @@ public abstract class Extractor {
* @return The {@link UrlIdHandler} of the current extractor object (e.g. a ChannelExtractor should return a channel url handler).
*/
@Nonnull
protected UrlIdHandler getUrlIdHandler() {
public UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/Info.java
View file

@ -1,5 +1,7 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
@ -16,8 +18,8 @@ public abstract class Info implements Serializable {
/**
* Different than the {@link #originalUrl} in the sense that it <i>may</i> be set as a cleaned url.
*
* @see UrlIdHandler#cleanUrl(String)
* @see Extractor#getCleanUrl()
* @see UrlIdHandler#getUrl()
* @see Extractor#getOriginalUrl()
*/
private final String url;
/**
@ -46,6 +48,14 @@ public abstract class Info implements Serializable {
this.name = name;
}
public Info(int serviceId, UrlIdHandler urlIdHandler, String name) throws ParsingException {
this(serviceId,
urlIdHandler.getId(),
urlIdHandler.getUrl(),
urlIdHandler.getOriginalUrl(),
name);
}
@Override
public String toString() {
final String ifDifferentString = !url.equals(originalUrl) ? " (originalUrl=\"" + originalUrl + "\")" : "";

5
extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View file

@ -49,6 +49,11 @@ public abstract class ListExtractor<R extends InfoItem> extends Extractor {
return nextPageUrl != null && !nextPageUrl.isEmpty();
}
@Override
public ListUrlIdHandler getUrlIdHandler() {
return (ListUrlIdHandler) super.getUrlIdHandler();
}
/*//////////////////////////////////////////////////////////////////////////
// Inner
//////////////////////////////////////////////////////////////////////////*/

28
extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java
View file

@ -1,13 +1,31 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.List;
public abstract class ListInfo<T extends InfoItem> extends Info {
private List<T> relatedItems;
private String nextPageUrl = null;
private String[] contentFilter = {};
private String sortFilter = "";
public ListInfo(int serviceId, String id, String url, String originalUrl, String name) {
public ListInfo(int serviceId,
String id,
String url,
String originalUrl,
String name,
String[] contentFilter,
String sortFilter) {
super(serviceId, id, url, originalUrl, name);
this.contentFilter = contentFilter;
this.sortFilter = sortFilter;
}
public ListInfo(int serviceId, ListUrlIdHandler listUrlIdHandler, String name) throws ParsingException {
super(serviceId, listUrlIdHandler, name);
this.contentFilter = listUrlIdHandler.getContentFilter();
this.sortFilter = listUrlIdHandler.getSortFilter();
}
public List<T> getRelatedItems() {
@ -29,4 +47,12 @@ public abstract class ListInfo<T extends InfoItem> extends Info {
public void setNextPageUrl(String pageUrl) {
this.nextPageUrl = pageUrl;
}
public String[] getContentFilter() {
return contentFilter;
}
public String getSortFilter() {
return sortFilter;
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java
View file

@ -39,4 +39,13 @@ public abstract class ListUrlIdHandler extends UrlIdHandler {
public String[] getAvailableSortFilter() {
return new String[0];
}
public String[] getContentFilter() {
return contentFilter;
}
public String getSortFilter() {
return sortFilter;
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View file

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
@ -75,6 +76,7 @@ public abstract class StreamingService {
// Extractor
////////////////////////////////////////////
public abstract SearchEngine getSearchEngine();
public abstract SearchExtractor getSearchExtractor();
public abstract SuggestionExtractor getSuggestionExtractor();
public abstract SubscriptionExtractor getSubscriptionExtractor();
public abstract KioskList getKioskList() throws ExtractionException;

6
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View file

@ -36,12 +36,6 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
super(service, urlIdHandler);
}
@Nonnull
@Override
protected UrlIdHandler getUrlIdHandler() {
return getService().getChannelUrlIdHandler();
}
public abstract String getAvatarUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException;
public abstract String getFeedUrl() throws ParsingException;

17
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View file

@ -2,9 +2,11 @@ package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
@ -32,8 +34,8 @@ import java.io.IOException;
public class ChannelInfo extends ListInfo<StreamInfoItem> {
public ChannelInfo(int serviceId, String id, String url, String originalUrl, String name) {
super(serviceId, id, url, originalUrl, name);
public ChannelInfo(int serviceId, ListUrlIdHandler urlIdHandler, String name) throws ParsingException {
super(serviceId, urlIdHandler, name);
}
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@ -52,14 +54,9 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
// important data
int serviceId = extractor.getServiceId();
String url = extractor.getUrl();
String originalUrl = extractor.getOriginalUrl();
String id = extractor.getId();
String name = extractor.getName();
ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name);
ChannelInfo info = new ChannelInfo(extractor.getServiceId(),
extractor.getUrlIdHandler(),
extractor.getName());
try {

20
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View file

@ -20,11 +20,9 @@ package org.schabi.newpipe.extractor.kiosk;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
@ -32,8 +30,8 @@ import java.io.IOException;
public class KioskInfo extends ListInfo<StreamInfoItem> {
private KioskInfo(int serviceId, String id, String url, String originalUrl, String name) {
super(serviceId, id, url, originalUrl, name);
private KioskInfo(int serviceId, ListUrlIdHandler urlIdHandler, String name) throws ParsingException {
super(serviceId, urlIdHandler, name);
}
public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
@ -68,13 +66,9 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
*/
public static KioskInfo getInfo(KioskExtractor extractor) throws ExtractionException {
int serviceId = extractor.getServiceId();
String name = extractor.getName();
String id = extractor.getId();
String url = extractor.getUrl();
String originalUrl = extractor.getOriginalUrl();
KioskInfo info = new KioskInfo(serviceId, id, url, originalUrl, name);
final KioskInfo info = new KioskInfo(extractor.getServiceId(),
extractor.getUrlIdHandler(),
extractor.getName());
final ListExtractor.InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());

6
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View file

@ -16,12 +16,6 @@ public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
super(service, urlIdHandler);
}
@Nonnull
@Override
protected UrlIdHandler getUrlIdHandler() {
return getService().getPlaylistUrlIdHandler();
}
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException;

18
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View file

@ -2,9 +2,11 @@ package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
@ -12,8 +14,8 @@ import java.io.IOException;
public class PlaylistInfo extends ListInfo<StreamInfoItem> {
public PlaylistInfo(int serviceId, String id, String url, String originalUrl, String name) {
super(serviceId, id, url, originalUrl, name);
public PlaylistInfo(int serviceId, ListUrlIdHandler urlIdHandler, String name) throws ParsingException {
super(serviceId, urlIdHandler, name);
}
public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
@ -35,14 +37,12 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
*
* @param extractor an extractor where fetchPage() was already got called on.
*/
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException {
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ExtractionException {
int serviceId = extractor.getServiceId();
String url = extractor.getUrl();
String originalUrl = extractor.getOriginalUrl();
String id = extractor.getId();
String name = extractor.getName();
PlaylistInfo info = new PlaylistInfo(serviceId, id, url, originalUrl, name);
final PlaylistInfo info = new PlaylistInfo(
extractor.getServiceId(),
extractor.getUrlIdHandler(),
extractor.getName());
try {
info.setStreamCount(extractor.getStreamCount());

2
extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java
View file

@ -61,7 +61,7 @@ public class InfoItemsSearchCollector extends InfoItemsCollector<InfoItem, InfoI
this.suggestion = suggestion;
}
public SearchResult getSearchResult() throws ExtractionException {
public SearchResult getSearchResult() {
return new SearchResult(getServiceId(), suggestion, getItems(), getErrors());
}

38
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java Normal file
View file

@ -0,0 +1,38 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public abstract class SearchExtractor extends ListExtractor<InfoItem> {
public static class NothingFoundException extends ExtractionException {
public NothingFoundException(String message) {
super(message);
}
}
private final InfoItemsSearchCollector collector;
public SearchExtractor(StreamingService service, SearchQuerryUrlHandler urlIdHandler) {
super(service, urlIdHandler);
collector = new InfoItemsSearchCollector(service.getServiceId());
}
public String getSearchQuerry() {
return getUrlIdHandler().getSearchQuerry();
}
public abstract String getSearchSuggestion() throws ParsingException;
protected InfoItemsSearchCollector getInfoItemSearchCollector() {
return collector;
}
@Override
public SearchQuerryUrlHandler getUrlIdHandler() {
return (SearchQuerryUrlHandler) super.getUrlIdHandler();
}
}

46
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java Normal file
View file

@ -0,0 +1,46 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class SearchInfo extends ListInfo<InfoItem> {
private String searchQuerry = "";
private String searchSuggestion = "";
public SearchInfo(int serviceId,
ListUrlIdHandler urlIdHandler,
String searchQuerry) throws ParsingException {
super(serviceId, urlIdHandler, "Search");
this.searchQuerry = searchQuerry;
}
public static SearchInfo getInfo(SearchExtractor extractor) throws ExtractionException {
final SearchInfo info = new SearchInfo(
extractor.getServiceId(),
extractor.getUrlIdHandler(),
extractor.getSearchQuerry());
try {
info.searchSuggestion = extractor.getSearchSuggestion();
} catch (Exception e) {
info.addError(e);
}
return info;
}
// Getter
public String getSearchQuerry() {
return searchQuerry;
}
public String getSearchSuggestion() {
return searchSuggestion;
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchQuerryUrlHandler.java Normal file
View file

@ -0,0 +1,11 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
public abstract class SearchQuerryUrlHandler extends ListUrlIdHandler {
String searchQuerry;
public String getSearchQuerry() {
return searchQuerry;
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View file

@ -32,12 +32,6 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
return getId();
}
@Nonnull
@Override
public UrlIdHandler getUrlIdHandler() {
return new SoundcloudChartsUrlIdHandler();
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
if (pageUrl == null || pageUrl.isEmpty()) {

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View file

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
@ -27,6 +28,12 @@ public class SoundcloudService extends StreamingService {
return new SoundcloudSearchEngine(getServiceId());
}
@Override
public SearchExtractor getSearchExtractor() {
return null;
}
@Override
public UrlIdHandler getStreamUrlIdHandler() {
return SoundcloudStreamUrlIdHandler.getInstance();

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View file

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.*;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubePlaylistUrlIdHandler;
@ -53,6 +54,11 @@ public class YoutubeService extends StreamingService {
return new YoutubeSearchEngine(getServiceId());
}
@Override
public SearchExtractor getSearchExtractor() {
return null;
}
@Override
public UrlIdHandler getStreamUrlIdHandler() {
return YoutubeStreamUrlIdHandler.getInstance();

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View file

@ -59,12 +59,6 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
doc = Jsoup.parse(pageContent, url);
}
@Nonnull
@Override
public UrlIdHandler getUrlIdHandler() {
return new YoutubeTrendingUrlIdHandler();
}
@Override
public String getNextPageUrl() {
return "";

6
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View file

@ -43,12 +43,6 @@ public abstract class StreamExtractor extends Extractor {
super(service, urlIdHandler);
}
@Nonnull
@Override
protected UrlIdHandler getUrlIdHandler() {
return getService().getStreamUrlIdHandler();
}
@Nonnull
public abstract String getUploadDate() throws ParsingException;
@Nonnull

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View file

@ -46,7 +46,7 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testName() throws Exception {
String name = extractor.getName();
assertTrue(name, name.startsWith("Pop Music Playlist: Timeless Pop Hits"));
assertTrue(name, name.startsWith("Pop Music Playlist"));
}
@Test