CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
Ritvik Saraf 2018-10-19 20:09:51 +05:30
commit 05ef80c2eb
15 changed files with 53 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java
View file

@ -103,7 +103,7 @@ public class NewPipe {
NewPipe.localization = localization;
}
public static Localization getLocalization() {
public static Localization getPreferredLocalization() {
return localization;
}
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View file

@ -87,7 +87,7 @@ public abstract class StreamingService {
public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization);
public abstract SuggestionExtractor getSuggestionExtractor(Localization localization);
public abstract SubscriptionExtractor getSubscriptionExtractor();
public abstract KioskList getKioskList(Localization localization) throws ExtractionException;
public abstract KioskList getKioskList() throws ExtractionException;
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException;
@ -102,27 +102,23 @@ public abstract class StreamingService {
////////////////////////////////////////////
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return getSearchExtractor(queryHandler, NewPipe.getLocalization());
return getSearchExtractor(queryHandler, NewPipe.getPreferredLocalization());
}
public SuggestionExtractor getSuggestionExtractor() {
return getSuggestionExtractor(NewPipe.getLocalization());
}
public KioskList getKioskList() throws ExtractionException {
return getKioskList(NewPipe.getLocalization());
return getSuggestionExtractor(NewPipe.getPreferredLocalization());
}
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getChannelExtractor(linkHandler, NewPipe.getLocalization());
return getChannelExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getPlaylistExtractor(linkHandler, NewPipe.getLocalization());
return getPlaylistExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException {
return getStreamExtractor(linkHandler, NewPipe.getLocalization());
return getStreamExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException {
@ -167,19 +163,19 @@ public abstract class StreamingService {
////////////////////////////////////////////
public SearchExtractor getSearchExtractor(String query) throws ExtractionException {
return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getLocalization());
return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getPreferredLocalization());
}
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getLocalization());
return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
}
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getLocalization());
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
}
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getLocalization());
return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
}
public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {

3
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View file

@ -51,8 +51,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url,
String pageUrl,
Localization localization) throws IOException, ExtractionException {
String pageUrl) throws IOException, ExtractionException {
return service.getChannelExtractor(url).getPage(pageUrl);
}

17
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View file

@ -30,7 +30,6 @@ import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
private String contentCountry = null;
private final String id;
public KioskExtractor(StreamingService streamingService,
@ -41,17 +40,6 @@ public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
this.id = kioskId;
}
/**
* For certain Websites the content of a kiosk will be different depending
* on the country you want to poen the website in. Therefore you should
* set the contentCountry.
* @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm
*/
public void setContentCountry(String contentCountry) {
this.contentCountry = contentCountry;
}
@Nonnull
@Override
public String getId() {
@ -69,9 +57,4 @@ public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
@Nonnull
@Override
public abstract String getName() throws ParsingException;
public String getContentCountry() {
return contentCountry;
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View file

@ -37,25 +37,20 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url,
String pageUrl,
String contentCountry) throws IOException, ExtractionException {
String pageUrl) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
extractor.setContentCountry(contentCountry);
return extractor.getPage(pageUrl);
}
public static KioskInfo getInfo(String url,
String contentCountry) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry);
public static KioskInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static KioskInfo getInfo(StreamingService service,
String url,
String contentCountry) throws IOException, ExtractionException {
String url) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, null);
extractor.setContentCountry(contentCountry);
extractor.fetchPage();
return getInfo(extractor);
}

25
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View file

@ -23,7 +23,6 @@ public class KioskList {
private final int service_id;
private final HashMap<String, KioskEntry> kioskList = new HashMap<>();
private String defaultKiosk = null;
private final Localization localization;
private class KioskEntry {
public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) {
@ -34,9 +33,8 @@ public class KioskList {
final ListLinkHandlerFactory handlerFactory;
}
public KioskList(int service_id, Localization localization) {
public KioskList(int service_id) {
this.service_id = service_id;
this.localization = localization;
}
public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id)
@ -53,13 +51,18 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor(String nextPageUrl)
throws ExtractionException, IOException {
return getDefaultKioskExtractor(nextPageUrl, NewPipe.getPreferredLocalization());
}
public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
if(defaultKiosk != null && !defaultKiosk.equals("")) {
return getExtractorById(defaultKiosk, nextPageUrl);
return getExtractorById(defaultKiosk, nextPageUrl, localization);
} else {
if(!kioskList.isEmpty()) {
// if not set get any entry
Object[] keySet = kioskList.keySet().toArray();
return getExtractorById(keySet[0].toString(), nextPageUrl);
return getExtractorById(keySet[0].toString(), nextPageUrl, localization);
} else {
return null;
}
@ -72,6 +75,11 @@ public class KioskList {
public KioskExtractor getExtractorById(String kioskId, String nextPageUrl)
throws ExtractionException, IOException {
return getExtractorById(kioskId, nextPageUrl, NewPipe.getPreferredLocalization());
}
public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
KioskEntry ke = kioskList.get(kioskId);
if(ke == null) {
throw new ExtractionException("No kiosk found with the type: " + kioskId);
@ -86,11 +94,16 @@ public class KioskList {
}
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl)
throws ExtractionException, IOException{
return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization());
}
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
KioskEntry ke = e.getValue();
if(ke.handlerFactory.acceptUrl(url)) {
return getExtractorById(e.getKey(), nextPageUrl);
return getExtractorById(e.getKey(), nextPageUrl, localization);
}
}
throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View file

@ -69,7 +69,7 @@ public class SoundcloudService extends StreamingService {
}
@Override
public KioskList getKioskList(Localization localization) throws ExtractionException {
public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
@ -82,7 +82,7 @@ public class SoundcloudService extends StreamingService {
}
};
KioskList list = new KioskList(getServiceId(), localization);
KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View file

@ -111,8 +111,8 @@ public class YoutubeService extends StreamingService {
}
@Override
public KioskList getKioskList(final Localization localization) throws ExtractionException {
KioskList list = new KioskList(getServiceId(), localization);
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.:
try {

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View file

@ -53,22 +53,16 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String site;
final String url = getUrl();
final String contentCountry = getLocalization().getCountry();
final String url = getUrl() + "?gl="+ getLocalization().getCountry();
//String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if (!contentCountry.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, getLocalization());
} else {
site = downloader.download(url);
}
site = downloader.download(url, getLocalization());
doc = Jsoup.parse(site, url);
}
@Override
public String getSearchSuggestion() throws ParsingException {
public String getSearchSuggestion() {
final Element el = doc.select("div[class*=\"spell-correction\"]").first();
if (el != null) {
return el.select("a").first().text();
@ -79,13 +73,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
return new InfoItemsPage<>(collectItems(doc), getNextPageUrl());
}
@Override
public String getNextPageUrl() throws ExtractionException {
return getUrl() + "&page=" + Integer.toString( 2);
return getUrl() + "&page=" + 2 + "&gl=" + getLocalization().getCountry();
}
@Override
@ -104,7 +98,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getQuery())
.get("page"));
return currentUrl.replace("&page=" + Integer.toString( pageNr),
return currentUrl.replace("&page=" + pageNr,
"&page=" + Integer.toString(pageNr + 1));
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View file

@ -50,7 +50,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String contentCountry = getContentCountry();
final String contentCountry = getLocalization().getCountry();
String url = getUrl();
if(contentCountry != null && !contentCountry.isEmpty()) {
url += "?gl=" + contentCountry;

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View file

@ -177,7 +177,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=LzR8Sf5PK2Q");
.getStreamExtractor("https://www.youtube.com/watch?v=fBc4Q_htqPg");
extractor.fetchPage();
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View file

@ -51,7 +51,6 @@ public class YoutubeTrendingExtractorTest {
.getKioskList()
.getExtractorById("Trending", null);
extractor.fetchPage();
extractor.setContentCountry("de");
}
@Test

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java
View file

@ -46,7 +46,7 @@ public class YoutubeTrendingKioskInfoTest {
StreamingService service = YouTube;
LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending");
kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl(), null);
kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl());
}
@Test

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java
View file

@ -45,12 +45,12 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
}
assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=3", secondPage.getNextPageUrl());
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=3&gl=GB", secondPage.getNextPageUrl());
}
@Test
public void testGetSecondPageUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=2", extractor.getNextPageUrl());
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=2&gl=GB", extractor.getNextPageUrl());
}
@Test

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java
View file

@ -53,12 +53,12 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
@Test
public void testGetSecondPageUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&page=2", extractor.getNextPageUrl());
assertEquals("https://www.youtube.com/results?q=pewdiepie&page=2&gl=GB", extractor.getNextPageUrl());
}
@Test
public void testResultList_FirstElement() {
InfoItem firstInfoItem = itemsPage.getItems().get(0);
InfoItem firstInfoItem = itemsPage.getItems().get(1);
// THe channel should be the first item
assertTrue(firstInfoItem instanceof ChannelInfoItem);
@ -96,7 +96,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
}
assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&page=3", secondPage.getNextPageUrl());
assertEquals("https://www.youtube.com/results?q=pewdiepie&page=3&gl=GB", secondPage.getNextPageUrl());
}
@Test