Add SoundcloudChartsExtractor

This commit is contained in:
wb9688 2017-08-20 10:03:41 +02:00
parent 5119dabb63
commit e2b7cb9c69
6 changed files with 254 additions and 8 deletions

View File

@ -0,0 +1,63 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
public class SoundcloudChartsExtractor extends KioskExtractor {
private String url;
public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
this.url = url;
}
@Override
public void fetchPage() {
}
@Override
public String getType() throws ParsingException {
return getUrlIdHandler().getId(url);
}
@Override
public UrlIdHandler getUrlIdHandler() {
return new SoundcloudChartsUrlIdHandler();
}
@Override
public NextItemsResult getNextStreams() throws IOException, ExtractionException {
if (!hasMoreStreams()) {
throw new ExtractionException("Chart doesn't have more streams");
}
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
}
@Override
public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" +
"&client_id=" + SoundcloudParsingHelper.clientId();
if (getType().equals("Top 50")) {
apiUrl += "&kind=top";
} else {
apiUrl += "&kind=new";
}
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
return collector;
}
}

View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
public String getUrl(String id) {
if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top";
} else {
return "https://soundcloud.com/charts/new";
}
}
@Override
public String getId(String url) {
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
return "Top 50";
} else {
return "New & hot";
}
}
@Override
public String cleanUrl(String url) {
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
return "https://soundcloud.com/charts/top";
} else {
return "https://soundcloud.com/charts/new";
}
}
@Override
public boolean acceptUrl(String url) {
return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase());
}
}

View File

@ -121,13 +121,17 @@ public class SoundcloudParsingHelper {
*
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
if (charts) {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i).getJSONObject("track")));
} else {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
}
}
String nextStreamsUrl;
@ -140,4 +144,8 @@ public class SoundcloudParsingHelper {
return nextStreamsUrl;
}
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
return getStreamsFromApi(collector, apiUrl, false);
}
}

View File

@ -1,18 +1,17 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import java.io.IOException;
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id, String name) {
@ -61,12 +60,17 @@ public class SoundcloudService extends StreamingService {
}
@Override
public KioskList getKioskList() {
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.:
//list.addKioskEntry("trinding", new TrendingKiosk(), new TrendingUrlIdHandler());
SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler();
try {
list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("Top 50"), null), h);
list.addKioskEntry(new SoundcloudChartsExtractor(this, h.getUrl("New & hot"), null), h);
} catch (Exception e) {
throw new ExtractionException(e);
}
return list;
}

View File

@ -0,0 +1,85 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
/**
* Test for {@link SoundcloudChartsUrlIdHandler}
*/
public class SoundcloudChartsExtractorTest {
KioskExtractor extractor;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = SoundCloud.getService()
.getKioskList()
.getExtractorByType("Top 50");
extractor.fetchPage();
}
@Test
public void testGetDownloader() throws Exception {
assertNotNull(NewPipe.getDownloader());
}
@Test
public void testGetName() throws Exception {
assertEquals(extractor.getName(), "Top 50");
}
@Test
public void testId() throws Exception {
assertEquals(extractor.getId(), "Top 50");
}
@Test
public void testGetStreams() throws Exception {
InfoItemCollector collector = extractor.getStreams();
if(!collector.getErrors().isEmpty()) {
System.err.println("----------");
for(Throwable e : collector.getErrors()) {
e.printStackTrace();
System.err.println("----------");
}
}
assertTrue("no streams are received",
!collector.getItemList().isEmpty()
&& collector.getErrors().isEmpty());
}
@Test
public void testGetStreamsErrors() throws Exception {
assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
}
@Test
public void testHasMoreStreams() throws Exception {
// Setup the streams
extractor.getStreams();
assertTrue("has more streams", extractor.hasMoreStreams());
}
@Test
public void testGetNextStreams() throws Exception {
extractor.getStreams();
assertFalse("extractor has next streams", extractor.getNextStreams() == null
|| extractor.getNextStreams().nextItemsList.isEmpty());
}
@Test
public void testGetCleanUrl() throws Exception {
assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top");
}
}

View File

@ -0,0 +1,49 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
/**
* Test for {@link SoundcloudChartsUrlIdHandler}
*/
public class SoundcloudChartsUrlIdHandlerTest {
private SoundcloudChartsUrlIdHandler urlIdHandler;
@Before
public void setUp() throws Exception {
urlIdHandler = new SoundcloudChartsUrlIdHandler();
NewPipe.init(Downloader.getInstance());
}
@Test
public void getUrl() {
assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top");
assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new");
}
@Test
public void getId() {
assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50");
assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot");
}
@Test
public void acceptUrl() {
assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts"));
assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/"));
assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new"));
assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/charts/top?genre=all-music"));
assertTrue(urlIdHandler.acceptUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"));
assertFalse(urlIdHandler.acceptUrl("kdskjfiiejfia"));
assertFalse(urlIdHandler.acceptUrl("soundcloud.com/charts askjkf"));
assertFalse(urlIdHandler.acceptUrl(" soundcloud.com/charts"));
assertFalse(urlIdHandler.acceptUrl(""));
}
}