CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #73 from karyogamy/live

Browse source 
HLS Livestream Extraction
This commit is contained in:
Christian Schabesberger 2018-03-06 19:39:18 +01:00 committed by GitHub
commit b1130629bb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 187 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java
View file

@ -44,23 +44,10 @@ public class SearchResult {
this.errors = Collections.unmodifiableList(new ArrayList<>(errors));
}
public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter)
public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page,
final String languageCode, final SearchEngine.Filter filter)
throws IOException, ExtractionException {
SearchResult result = engine
.search(query, page, languageCode, filter)
.getSearchResult();
if (result.resultList.isEmpty()) {
if (result.suggestion.isEmpty()) {
if (result.errors.isEmpty()) {
throw new ExtractionException("Empty result despite no error");
}
} else {
// This is used as a fallback. Do not relay on it !!!
throw new SearchEngine.NothingFoundException(result.suggestion);
}
}
return result;
return engine.search(query, page, languageCode, filter).getSearchResult();
}
public String getSuggestion() {

9
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View file

@ -116,9 +116,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return SoundcloudParsingHelper.getAvatarUrl(track);
}
@Nonnull
@Override
public String getDashMpdUrl() {
return null;
return "";
}
@Nonnull
@Override
public String getHlsUrl() throws ParsingException {
return "";
}
@Override

60
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View file

@ -66,12 +66,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
public class LiveStreamException extends ContentNotAvailableException {
LiveStreamException(String message) {
super(message);
}
}
public class SubtitlesException extends ContentNotAvailableException {
SubtitlesException(String message, Throwable cause) {
super(message, cause);
@ -338,6 +332,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Nonnull
@Override
public String getDashMpdUrl() throws ParsingException {
assertPageFetched();
@ -365,6 +360,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Nonnull
@Override
public String getHlsUrl() throws ParsingException {
assertPageFetched();
try {
String hlsvp;
if (playerArgs != null && playerArgs.isString("hlsvp")) {
hlsvp = playerArgs.getString("hlsvp", "");
} else {
return "";
}
return hlsvp;
} catch (Exception e) {
throw new ParsingException("Could not get hls manifest url", e);
}
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
assertPageFetched();
@ -428,7 +441,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
@Nonnull
public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException {
return getSubtitles(SubtitlesFormat.VTT);
return getSubtitles(SubtitlesFormat.TTML);
}
@Override
@ -444,7 +457,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public StreamType getStreamType() throws ParsingException {
//todo: if implementing livestream support this value should be generated dynamically
assertPageFetched();
try {
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
return StreamType.LIVE_STREAM;
}
} catch (Exception e) {
throw new ParsingException("Could not get hls manifest url", e);
}
return StreamType.VIDEO_STREAM;
}
@ -517,13 +538,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String CONTENT = "content";
private static final String DECRYPTION_FUNC_NAME = "decrypt";
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
private volatile String decryptionCode = "";
private String pageHtml = null;
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS;
if (pageHtml == null) {
pageHtml = downloader.download(getCleanUrl());
pageHtml = downloader.download(verifiedUrl);
}
return pageHtml;
}
@ -534,7 +558,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
doc = Jsoup.parse(pageContent, getCleanUrl());
final String playerUrl;
// TODO: use embedded videos to fetch DASH manifest for all videos
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
final EmbeddedInfo info = getEmbeddedInfo();
@ -582,21 +605,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
JsonObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (Exception e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Live stream. Can't use those right now.");
}
return playerArgs;
}
@ -806,11 +819,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"&sts=" + sts + "&ps=default&gl=US&hl=en";
}
@Nonnull
private static String getSubtitleFormatUrl(final String baseUrl, final SubtitlesFormat format) {
return baseUrl.replaceAll("&fmt=[^&]*", "") + "&fmt=" + format.getExtension();
}
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();

3
src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View file

@ -126,7 +126,8 @@ public abstract class StreamExtractor extends Extractor {
* @return the url as a string or an empty string
* @throws ParsingException if an error occurs while reading
*/
public abstract String getDashMpdUrl() throws ParsingException;
@Nonnull public abstract String getDashMpdUrl() throws ParsingException;
@Nonnull public abstract String getHlsUrl() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;

15
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View file

@ -126,6 +126,10 @@ public class StreamInfo extends Info {
return dashMpdUrl;
}
public String getHlsUrl() {
return hlsUrl;
}
public StreamInfoItem getNextVideo() {
return next_video;
}
@ -206,6 +210,10 @@ public class StreamInfo extends Info {
this.dashMpdUrl = dashMpdUrl;
}
public void setHlsUrl(String hlsUrl) {
this.hlsUrl = hlsUrl;
}
public void setNextVideo(StreamInfoItem next_video) {
this.next_video = next_video;
}
@ -298,6 +306,12 @@ public class StreamInfo extends Info {
streamInfo.addError(new ExtractionException("Couldn't get Dash manifest", e));
}
try {
streamInfo.setHlsUrl(extractor.getHlsUrl());
} catch (Exception e) {
streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e));
}
/* Load and extract audio */
try {
streamInfo.setAudioStreams(extractor.getAudioStreams());
@ -447,6 +461,7 @@ public class StreamInfo extends Info {
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd file will be possible in the future.
public String dashMpdUrl;
public String hlsUrl;
public StreamInfoItem next_video;
public List<InfoItem> related_streams;

125
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java Normal file
View file

@ -0,0 +1,125 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link YoutubeStreamUrlIdHandler}
*/
public class YoutubeStreamExtractorControversialTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8");
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0);
}
@Test
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertEquals(extractor.getTimeStamp() + "", "174");
}
@Test
@Ignore
public void testGetAgeLimit() throws ParsingException {
assertEquals(18, extractor.getAgeLimit());
}
@Test
public void testGetName() throws ParsingException {
assertNotNull("name is null", extractor.getName());
assertFalse("name is empty", extractor.getName().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertNotNull(extractor.getUploaderName());
assertFalse(extractor.getUploaderName().isEmpty());
}
@Ignore // Currently there is no way get the length from restricted videos
@Test
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
@Test
public void testGetViews() throws ParsingException {
assertTrue(extractor.getViewCount() > 0);
}
@Test
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
// FIXME: 25.11.17 Are there no streams or are they not listed?
@Ignore
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
// audio streams are not always necessary
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
List<VideoStream> streams = new ArrayList<>();
streams.addAll(extractor.getVideoStreams());
streams.addAll(extractor.getVideoOnlyStreams());
assertTrue(streams.size() > 0);
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(!extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(!extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty());
}
}