CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

-Added playlist info item extraction for Youtube and Soundcloud. 

-Added playlist collection into search engine.
-Fixed stream info duration exception when parsing 0 second video.
This commit is contained in:
John Zhen M 2017-08-15 17:11:38 -07:00
parent 9184fc509c
commit 645ee5a5ff
9 changed files with 378 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java
View File

@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
@ -32,6 +34,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
private String suggestion = "";
private StreamInfoItemCollector streamCollector;
private ChannelInfoItemCollector userCollector;
private PlaylistInfoItemCollector playlistCollector;
private SearchResult result = new SearchResult();
@ -39,6 +42,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
super(serviceId);
streamCollector = new StreamInfoItemCollector(serviceId);
userCollector = new ChannelInfoItemCollector(serviceId);
playlistCollector = new PlaylistInfoItemCollector(serviceId);
}
public void setSuggestion(String suggestion) {
@ -49,6 +53,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
addFromCollector(userCollector);
addFromCollector(streamCollector);
addFromCollector(playlistCollector);
result.suggestion = suggestion;
result.errors = getErrors();
@ -74,4 +79,14 @@ public class InfoItemSearchCollector extends InfoItemCollector {
addError(e);
}
}
public void commit(PlaylistInfoItemExtractor extractor) {
try {
result.resultList.add(playlistCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
}
}
}

89
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java Normal file
View File

@ -0,0 +1,89 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray;
import com.github.openjson.JSONObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private static final String USER_KEY = "user";
private static final String AVATAR_URL_KEY = "avatar_url";
private static final String ARTWORK_URL_KEY = "artwork_url";
private static final String NULL_VALUE = "null";
private JSONObject searchResult;
public SoundcloudPlaylistInfoItemExtractor(JSONObject searchResult) {
this.searchResult = searchResult;
}
@Override
public String getName() throws ParsingException {
try {
return searchResult.getString("title");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist name", e);
}
}
@Override
public String getUrl() throws ParsingException {
try {
return searchResult.getString("permalink_url");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist name", e);
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
try {
final String artworkUrl = searchResult.optString(ARTWORK_URL_KEY);
if (!artworkUrl.isEmpty() && !artworkUrl.equals(NULL_VALUE)) return artworkUrl;
// Look for artwork url inside the track list
final JSONArray tracks = searchResult.optJSONArray("tracks");
if (tracks == null) return null;
for (int i = 0; i < tracks.length(); i++) {
if (tracks.isNull(i)) continue;
final JSONObject track = tracks.optJSONObject(i);
if (track == null) continue;
// First look for track artwork url
final String url = track.optString(ARTWORK_URL_KEY);
if (!url.isEmpty() && !url.equals(NULL_VALUE)) return url;
// Then look for track creator avatar url
final JSONObject creator = track.getJSONObject(USER_KEY);
final String creatorAvatar = creator.optString(AVATAR_URL_KEY);
if (!creatorAvatar.isEmpty() && !creatorAvatar.equals(NULL_VALUE)) return creatorAvatar;
}
// Last resort, use user avatar url. If still not found, then throw exception.
final JSONObject user = searchResult.getJSONObject(USER_KEY);
return user.getString(AVATAR_URL_KEY);
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
}
@Override
public String getUploaderName() throws ParsingException {
try {
final JSONObject user = searchResult.getJSONObject(USER_KEY);
return user.optString("username");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
}
@Override
public long getStreamCount() throws ParsingException {
try {
return Long.parseLong(searchResult.optString("track_count"));
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist stream count", e);
}
}
}

8
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java
View File

@ -27,10 +27,12 @@ public class SoundcloudSearchEngine extends SearchEngine {
String url = "https://api-v2.soundcloud.com/search";
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
if (filter.contains(Filter.STREAM) && filter.size() == 1) {
url += "/tracks";
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
} else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
url += "/users";
} else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
url += "/playlists";
}
url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
@ -53,6 +55,8 @@ public class SoundcloudSearchEngine extends SearchEngine {
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
} else if (kind.equals("track")) {
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
} else if (kind.equals("playlist")) {
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
}
}

92
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java Normal file
View File

@ -0,0 +1,92 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private Element el;
public YoutubePlaylistInfoItemExtractor(Element el) {
this.el = el;
}
@Override
public String getThumbnailUrl() throws ParsingException {
String url;
try {
Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
url = te.attr("abs:src");
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
return url;
}
@Override
public String getName() throws ParsingException {
String name;
try {
final Element title = el.select("[class=\"yt-lockup-title\"]").first()
.select("a").first();
name = title == null ? "" : title.text();
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist name", e);
}
return name;
}
@Override
public String getUrl() throws ParsingException {
String url;
try {
final Element href = el.select("div[class=\"yt-lockup-meta\"]").first()
.select("a").first();
url = href.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist url", e);
}
return url;
}
@Override
public String getUploaderName() throws ParsingException {
String name;
try {
final Element div = el.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first();
name = div.text();
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
return name;
}
@Override
public long getStreamCount() throws ParsingException {
try {
final Element count = el.select("span[class=\"formatted-video-count-label\"]").first()
.select("b").first();
return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text()));
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist stream count", e);
}
}
}

13
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java
View File

@ -57,10 +57,13 @@ public class YoutubeSearchEngine extends SearchEngine {
String url = "https://www.youtube.com/results"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&page=" + Integer.toString(page + 1);
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
url += "&sp=EgIQAQ%253D%253D";
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
url += "&sp=EgIQAg%253D%253D";
if (filter.contains(Filter.STREAM) && filter.size() == 1) {
url += "&sp=EgIQAVAU";
} else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
} else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
url += "&sp=EgIQA1AU"; //EgIQA( one )AU
}
String site;
@ -105,6 +108,8 @@ public class YoutubeSearchEngine extends SearchEngine {
collector.commit(new YoutubeStreamInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
collector.commit(new YoutubeChannelInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null) {
collector.commit(new YoutubePlaylistInfoItemExtractor(el));
} else {
// noinspection ConstantConditions
// simply ignore not known items

5
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@ -74,7 +74,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
try {
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
return YoutubeParsingHelper.parseDurationString(item.select("span[class*=\"video-time\"]").first().text());
final Element duration = item.select("span[class*=\"video-time\"]").first();
// apparently on youtube, video-time element will not show up if the video has a duration of 00:00
// see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf
return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text());
} catch (Exception e) {
throw new ParsingException("Could not get Duration: " + getUrl(), e);
}

80
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java Normal file
View File

@ -0,0 +1,80 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/*
* Created by Christian Schabesberger on 29.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngineStreamTest.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Test for {@link SearchEngine}
*/
public class SoundcloudSearchEnginePlaylistTest {
private SearchResult result;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = SoundCloud.getService().getSearchEngine();
// Search by country not yet implemented
result = engine.search("parkmemme", 0, "", EnumSet.of(SearchEngine.Filter.PLAYLIST))
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testUserItemType() {
for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
}
}
@Test
public void testResultErrors() {
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors == null || result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {
//todo write a real test
assertTrue(result.suggestion != null);
}
}

3
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java
View File

@ -50,7 +50,8 @@ public class YoutubeSearchEngineAllTest {
// keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("asdgff", 0, "de",
EnumSet.of(SearchEngine.Filter.CHANNEL,
SearchEngine.Filter.STREAM)).getSearchResult();
SearchEngine.Filter.STREAM,
SearchEngine.Filter.PLAYLIST)).getSearchResult();
}
@Test

81
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java Normal file
View File

@ -0,0 +1,81 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/*
* Created by Christian Schabesberger on 29.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngineStreamTest.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Test for {@link SearchEngine}
*/
public class YoutubeSearchEnginePlaylistTest {
private SearchResult result;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = YouTube.getService().getSearchEngine();
// Youtube will suggest "gronkh" instead of "grrunkh"
// keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("grrunkh", 0, "de",
EnumSet.of(SearchEngine.Filter.PLAYLIST)).getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testUserItemType() {
for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
}
}
@Test
public void testResultErrors() {
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors == null || result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {
//todo write a real test
assertTrue(result.suggestion != null);
}
}