commit
b1130629bb
6 changed files with 187 additions and 44 deletions
|
@ -44,23 +44,10 @@ public class SearchResult {
|
|||
this.errors = Collections.unmodifiableList(new ArrayList<>(errors));
|
||||
}
|
||||
|
||||
public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter)
|
||||
public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page,
|
||||
final String languageCode, final SearchEngine.Filter filter)
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
SearchResult result = engine
|
||||
.search(query, page, languageCode, filter)
|
||||
.getSearchResult();
|
||||
if (result.resultList.isEmpty()) {
|
||||
if (result.suggestion.isEmpty()) {
|
||||
if (result.errors.isEmpty()) {
|
||||
throw new ExtractionException("Empty result despite no error");
|
||||
}
|
||||
} else {
|
||||
// This is used as a fallback. Do not relay on it !!!
|
||||
throw new SearchEngine.NothingFoundException(result.suggestion);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
return engine.search(query, page, languageCode, filter).getSearchResult();
|
||||
}
|
||||
|
||||
public String getSuggestion() {
|
||||
|
|
|
@ -116,9 +116,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
return SoundcloudParsingHelper.getAvatarUrl(track);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getDashMpdUrl() {
|
||||
return null;
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getHlsUrl() throws ParsingException {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -66,12 +66,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
public class LiveStreamException extends ContentNotAvailableException {
|
||||
LiveStreamException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
public class SubtitlesException extends ContentNotAvailableException {
|
||||
SubtitlesException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
|
@ -338,6 +332,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getDashMpdUrl() throws ParsingException {
|
||||
assertPageFetched();
|
||||
|
@ -365,6 +360,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getHlsUrl() throws ParsingException {
|
||||
assertPageFetched();
|
||||
try {
|
||||
String hlsvp;
|
||||
if (playerArgs != null && playerArgs.isString("hlsvp")) {
|
||||
hlsvp = playerArgs.getString("hlsvp", "");
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
|
||||
return hlsvp;
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get hls manifest url", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
||||
assertPageFetched();
|
||||
|
@ -428,7 +441,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
@Nonnull
|
||||
public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException {
|
||||
return getSubtitles(SubtitlesFormat.VTT);
|
||||
return getSubtitles(SubtitlesFormat.TTML);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -444,7 +457,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
//todo: if implementing livestream support this value should be generated dynamically
|
||||
assertPageFetched();
|
||||
try {
|
||||
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
|
||||
playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
||||
return StreamType.LIVE_STREAM;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get hls manifest url", e);
|
||||
}
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
|
||||
|
@ -517,13 +538,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
private static final String CONTENT = "content";
|
||||
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
||||
|
||||
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
|
||||
|
||||
private volatile String decryptionCode = "";
|
||||
|
||||
private String pageHtml = null;
|
||||
|
||||
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{
|
||||
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
|
||||
final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS;
|
||||
if (pageHtml == null) {
|
||||
pageHtml = downloader.download(getCleanUrl());
|
||||
pageHtml = downloader.download(verifiedUrl);
|
||||
}
|
||||
return pageHtml;
|
||||
}
|
||||
|
@ -534,7 +558,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
doc = Jsoup.parse(pageContent, getCleanUrl());
|
||||
|
||||
final String playerUrl;
|
||||
// TODO: use embedded videos to fetch DASH manifest for all videos
|
||||
// Check if the video is age restricted
|
||||
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
|
||||
final EmbeddedInfo info = getEmbeddedInfo();
|
||||
|
@ -582,21 +605,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
JsonObject playerArgs;
|
||||
|
||||
//attempt to load the youtube js player JSON arguments
|
||||
boolean isLiveStream = false; //used to determine if this is a livestream or not
|
||||
try {
|
||||
playerArgs = playerConfig.getObject("args");
|
||||
|
||||
// check if we have a live stream. We need to filter it, since its not yet supported.
|
||||
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|
||||
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
||||
isLiveStream = true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not parse yt player config", e);
|
||||
}
|
||||
if (isLiveStream) {
|
||||
throw new LiveStreamException("This is a Live stream. Can't use those right now.");
|
||||
}
|
||||
|
||||
return playerArgs;
|
||||
}
|
||||
|
@ -806,11 +819,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
"&sts=" + sts + "&ps=default&gl=US&hl=en";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private static String getSubtitleFormatUrl(final String baseUrl, final SubtitlesFormat format) {
|
||||
return baseUrl.replaceAll("&fmt=[^&]*", "") + "&fmt=" + format.getExtension();
|
||||
}
|
||||
|
||||
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
|
||||
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
|
||||
|
||||
|
|
|
@ -126,7 +126,8 @@ public abstract class StreamExtractor extends Extractor {
|
|||
* @return the url as a string or an empty string
|
||||
* @throws ParsingException if an error occurs while reading
|
||||
*/
|
||||
public abstract String getDashMpdUrl() throws ParsingException;
|
||||
@Nonnull public abstract String getDashMpdUrl() throws ParsingException;
|
||||
@Nonnull public abstract String getHlsUrl() throws ParsingException;
|
||||
public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
|
||||
public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
|
||||
public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
|
||||
|
|
|
@ -126,6 +126,10 @@ public class StreamInfo extends Info {
|
|||
return dashMpdUrl;
|
||||
}
|
||||
|
||||
public String getHlsUrl() {
|
||||
return hlsUrl;
|
||||
}
|
||||
|
||||
public StreamInfoItem getNextVideo() {
|
||||
return next_video;
|
||||
}
|
||||
|
@ -206,6 +210,10 @@ public class StreamInfo extends Info {
|
|||
this.dashMpdUrl = dashMpdUrl;
|
||||
}
|
||||
|
||||
public void setHlsUrl(String hlsUrl) {
|
||||
this.hlsUrl = hlsUrl;
|
||||
}
|
||||
|
||||
public void setNextVideo(StreamInfoItem next_video) {
|
||||
this.next_video = next_video;
|
||||
}
|
||||
|
@ -298,6 +306,12 @@ public class StreamInfo extends Info {
|
|||
streamInfo.addError(new ExtractionException("Couldn't get Dash manifest", e));
|
||||
}
|
||||
|
||||
try {
|
||||
streamInfo.setHlsUrl(extractor.getHlsUrl());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e));
|
||||
}
|
||||
|
||||
/* Load and extract audio */
|
||||
try {
|
||||
streamInfo.setAudioStreams(extractor.getAudioStreams());
|
||||
|
@ -447,6 +461,7 @@ public class StreamInfo extends Info {
|
|||
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
|
||||
// providing the dash mpd file will be possible in the future.
|
||||
public String dashMpdUrl;
|
||||
public String hlsUrl;
|
||||
|
||||
public StreamInfoItem next_video;
|
||||
public List<InfoItem> related_streams;
|
||||
|
|
|
@ -0,0 +1,125 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
/**
|
||||
* Test for {@link YoutubeStreamUrlIdHandler}
|
||||
*/
|
||||
public class YoutubeStreamExtractorControversialTest {
|
||||
private static YoutubeStreamExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = (YoutubeStreamExtractor) YouTube
|
||||
.getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetInvalidTimeStamp() throws ParsingException {
|
||||
assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetValidTimeStamp() throws IOException, ExtractionException {
|
||||
StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
|
||||
assertEquals(extractor.getTimeStamp() + "", "174");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Ignore
|
||||
public void testGetAgeLimit() throws ParsingException {
|
||||
assertEquals(18, extractor.getAgeLimit());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetName() throws ParsingException {
|
||||
assertNotNull("name is null", extractor.getName());
|
||||
assertFalse("name is empty", extractor.getName().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDescription() throws ParsingException {
|
||||
assertNotNull(extractor.getDescription());
|
||||
assertFalse(extractor.getDescription().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderName() throws ParsingException {
|
||||
assertNotNull(extractor.getUploaderName());
|
||||
assertFalse(extractor.getUploaderName().isEmpty());
|
||||
}
|
||||
|
||||
@Ignore // Currently there is no way get the length from restricted videos
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertTrue(extractor.getLength() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetViews() throws ParsingException {
|
||||
assertTrue(extractor.getViewCount() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploadDate() throws ParsingException {
|
||||
assertTrue(extractor.getUploadDate().length() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetThumbnailUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getThumbnailUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderAvatarUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
|
||||
}
|
||||
|
||||
// FIXME: 25.11.17 Are there no streams or are they not listed?
|
||||
@Ignore
|
||||
@Test
|
||||
public void testGetAudioStreams() throws IOException, ExtractionException {
|
||||
// audio streams are not always necessary
|
||||
assertFalse(extractor.getAudioStreams().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetVideoStreams() throws IOException, ExtractionException {
|
||||
List<VideoStream> streams = new ArrayList<>();
|
||||
streams.addAll(extractor.getVideoStreams());
|
||||
streams.addAll(extractor.getVideoOnlyStreams());
|
||||
assertTrue(streams.size() > 0);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
|
||||
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
|
||||
assertTrue(!extractor.getSubtitlesDefault().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSubtitlesList() throws IOException, ExtractionException {
|
||||
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
|
||||
assertTrue(!extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty());
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue