CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add support for segmented streams

Browse source
This commit is contained in:
Christian Schabesberger 2018-11-06 14:24:47 +01:00
parent 32d316330c
commit 4de99ae28f
4 changed files with 115 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View file

@ -152,6 +152,9 @@ public class StreamInfo extends Info {
streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams());
streamInfo.getAudioStreams().addAll(result.getAudioStreams());
streamInfo.getVideoStreams().addAll(result.getVideoStreams());
streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams();
streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams();
streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams();
} catch (Exception e) {
// Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl),
// just skip the exception (but store it somewhere), as we later check if we have streams anyway.
@ -271,6 +274,11 @@ public class StreamInfo extends Info {
private List<VideoStream> videoOnlyStreams;
private String dashMpdUrl;
private List<VideoStream> segmentedVideoStreams;
private List<AudioStream> segmentedAudioStreams;
private List<VideoStream> segmentedVideoOnlyStreams;
private String hlsUrl;
private StreamInfoItem nextVideo;
private List<InfoItem> relatedStreams;
@ -431,6 +439,30 @@ public class StreamInfo extends Info {
this.dashMpdUrl = dashMpdUrl;
}
public List<VideoStream> getSegmentedVideoStreams() {
return segmentedVideoStreams;
}
public void setSegmentedVideoStreams(List<VideoStream> segmentedVideoStreams) {
this.segmentedVideoStreams = segmentedVideoStreams;
}
public List<AudioStream> getSegmentedAudioStreams() {
return segmentedAudioStreams;
}
public void setSegmentedAudioStreams(List<AudioStream> segmentedAudioStreams) {
this.segmentedAudioStreams = segmentedAudioStreams;
}
public List<VideoStream> getSegmentedVideoOnlyStreams() {
return segmentedVideoOnlyStreams;
}
public void setSegmentedVideoOnlyStreams(List<VideoStream> segmentedVideoOnlyStreams) {
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
}
public String getHlsUrl() {
return hlsUrl;
}

91
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java
View file

@ -59,10 +59,23 @@ public class DashMpdParser {
private final List<AudioStream> audioStreams;
private final List<VideoStream> videoOnlyStreams;
public ParserResult(List<VideoStream> videoStreams, List<AudioStream> audioStreams, List<VideoStream> videoOnlyStreams) {
private final List<VideoStream> segmentedVideoStreams;
private final List<AudioStream> segmentedAudioStreams;
private final List<VideoStream> segmentedVideoOnlyStreams;
public ParserResult(List<VideoStream> videoStreams,
List<AudioStream> audioStreams,
List<VideoStream> videoOnlyStreams,
List<VideoStream> segmentedVideoStreams,
List<AudioStream> segmentedAudioStreams,
List<VideoStream> segmentedVideoOnlyStreams) {
this.videoStreams = videoStreams;
this.audioStreams = audioStreams;
this.videoOnlyStreams = videoOnlyStreams;
this.segmentedVideoStreams = segmentedVideoStreams;
this.segmentedAudioStreams = segmentedAudioStreams;
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
}
public List<VideoStream> getVideoStreams() {
@ -76,10 +89,22 @@ public class DashMpdParser {
public List<VideoStream> getVideoOnlyStreams() {
return videoOnlyStreams;
}
public List<VideoStream> getSegmentedVideoStreams() {
return segmentedVideoStreams;
}
public List<AudioStream> getSegmentedAudioStreams() {
return segmentedAudioStreams;
}
public List<VideoStream> getSegmentedVideoOnlyStreams() {
return segmentedVideoOnlyStreams;
}
}
/**
* Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
* Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest,
* then it will search for any stream that the ItagItem has (by the id).
* <p>
* It has video, video only and audio streams and will only add to the list if it don't
@ -90,7 +115,8 @@ public class DashMpdParser {
*
* @param streamInfo where the parsed streams will be added
*/
public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
public static ParserResult getStreams(final StreamInfo streamInfo)
throws DashMpdParsingException, ReCaptchaException {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
@ -113,6 +139,10 @@ public class DashMpdParser {
final List<AudioStream> audioStreams = new ArrayList<>();
final List<VideoStream> videoOnlyStreams = new ArrayList<>();
final List<VideoStream> segmentedVideoStreams = new ArrayList<>();
final List<AudioStream> segmentedAudioStreams = new ArrayList<>();
final List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
for (int i = 0; i < representationList.getLength(); i++) {
final Element representation = (Element) representationList.item(i);
try {
@ -126,34 +156,61 @@ public class DashMpdParser {
// instead we need to add the "media=" value from the <SegementURL/> tags inside the <SegmentList/>
// tag in order to get a full working url. However each of these is just pointing to a part of the
// video, so we can not return a URL with a working stream here.
// We decided not to ignore such streams for the moment.
if (itag != null && segmentationList == null) {
// Instead of putting those streams into the list of regular stream urls wie put them in a
// for example "segmentedVideoStreams" list.
if (itag != null) {
final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
audioStreams.add(audioStream);
if(segmentationList == null) {
final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
audioStreams.add(audioStream);
}
} else {
segmentedAudioStreams.add(
new AudioStream(id, mediaFormat, itag.avgBitrate));
}
} else {
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly);
if (isVideoOnly) {
if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
streamInfo.getVideoOnlyStreams().add(videoStream);
videoOnlyStreams.add(videoStream);
if(segmentationList == null) {
final VideoStream videoStream = new VideoStream(url,
mediaFormat,
itag.resolutionString,
isVideoOnly);
if (isVideoOnly) {
if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
videoOnlyStreams.add(videoStream);
}
} else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
videoStreams.add(videoStream);
}
} else {
final VideoStream videoStream = new VideoStream(id,
mediaFormat,
itag.resolutionString,
isVideoOnly);
if(isVideoOnly) {
segmentedVideoOnlyStreams.add(videoStream);
} else {
segmentedVideoStreams.add(videoStream);
}
} else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
videoStreams.add(videoStream);
}
}
}
} catch (Exception ignored) {
}
}
return new ParserResult(videoStreams, audioStreams, videoOnlyStreams);
return new ParserResult(
videoStreams,
audioStreams,
videoOnlyStreams,
segmentedVideoStreams,
segmentedAudioStreams,
segmentedVideoOnlyStreams);
} catch (Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e);
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java
View file

@ -47,15 +47,21 @@ public class YoutubeStreamExtractorDASHTest {
@Test
public void testGetDashMpd() {
System.out.println(info.getDashMpdUrl());
assertTrue(info.getDashMpdUrl(),
info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty());
}
@Test
public void testDashMpdParser() {
public void testRegularStreams() {
assertEquals(0, info.getAudioStreams().size());
assertEquals(0, info.getVideoOnlyStreams().size());
assertEquals(4, info.getVideoStreams().size());
}
@Test
public void testSegmentedStreams() {
assertEquals(2, info.getSegmentedAudioStreams().size());
assertEquals(3, info.getSegmentedVideoOnlyStreams().size());
assertEquals(0, info.getSegmentedVideoStreams().size());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java
View file

@ -28,7 +28,7 @@ public class YoutubeSearchCountTest {
public void testViewCount() {
ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()),
65043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 68043316);
69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 73043316);
}
}
}