diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DashMpdParser.java new file mode 100644 index 00000000..77cee35b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DashMpdParser.java @@ -0,0 +1,220 @@ +/* + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * DashMpdParser.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; +import org.schabi.newpipe.extractor.stream.StreamInfo; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import javax.annotation.Nonnull; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +public final class DashMpdParser { + private DashMpdParser() { + } + + public static class DashMpdParsingException extends ParsingException { + + DashMpdParsingException(final String message, final Exception e) { + super(message, e); + } + } + + public static class Result { + private final List videoStreams; + private final List videoOnlyStreams; + private final List audioStreams; + + + public Result(final List videoStreams, + final List videoOnlyStreams, + final List audioStreams) { + this.videoStreams = videoStreams; + this.videoOnlyStreams = videoOnlyStreams; + this.audioStreams = audioStreams; + } + + public List getVideoStreams() { + return videoStreams; + } + + public List getVideoOnlyStreams() { + return videoOnlyStreams; + } + + public List getAudioStreams() { + return audioStreams; + } + } + + // TODO: Make this class generic and decouple from YouTube's ItagItem class. + + /** + * Will try to download and parse the DASH manifest (using {@link StreamInfo#getDashMpdUrl()}), + * adding items that are listed in the {@link ItagItem} class. + *

+ * It has video, video only and audio streams. + *

+ * Info about DASH MPD can be found here + * + * @param dashMpdUrl URL to the DASH MPD + * @see + * www.brendanlog.com + */ + @Nonnull + public static Result getStreams(final String dashMpdUrl) + throws DashMpdParsingException, ReCaptchaException { + final String dashDoc; + final Downloader downloader = NewPipe.getDownloader(); + try { + dashDoc = downloader.get(dashMpdUrl).responseBody(); + } catch (final IOException e) { + throw new DashMpdParsingException("Could not fetch DASH manifest: " + dashMpdUrl, e); + } + + try { + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + final DocumentBuilder builder = factory.newDocumentBuilder(); + final InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); + + final Document doc = builder.parse(stream); + final NodeList representationList = doc.getElementsByTagName("Representation"); + + final List videoStreams = new ArrayList<>(); + final List audioStreams = new ArrayList<>(); + final List videoOnlyStreams = new ArrayList<>(); + + for (int i = 0; i < representationList.getLength(); i++) { + final Element representation = (Element) representationList.item(i); + try { + final String mimeType = ((Element) representation.getParentNode()) + .getAttribute("mimeType"); + final String id = representation.getAttribute("id"); + final ItagItem itag = ItagItem.getItag(Integer.parseInt(id)); + final Element segmentationList = (Element) representation + .getElementsByTagName("SegmentList").item(0); + + if (segmentationList == null) { + continue; + } + + final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); + + if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { + audioStreams.add(new AudioStream.Builder() + .setId(String.valueOf(itag.id)) + .setContent(manualDashFromRepresentation(doc, representation), + false) + .setMediaFormat(mediaFormat) + .setDeliveryMethod(DeliveryMethod.DASH) + .setAverageBitrate(itag.getAverageBitrate()) + .setBaseUrl(dashMpdUrl) + .setItagItem(itag) + .build()); + } else { + final boolean isVideoOnly = itag.itagType == ItagItem.ItagType.VIDEO_ONLY; + final VideoStream videoStream = new VideoStream.Builder() + .setId(String.valueOf(itag.id)) + .setContent(manualDashFromRepresentation(doc, representation), + false) + .setMediaFormat(mediaFormat) + .setDeliveryMethod(DeliveryMethod.DASH) + .setResolution(Objects.requireNonNull(itag.getResolutionString())) + .setIsVideoOnly(isVideoOnly) + .setBaseUrl(dashMpdUrl) + .setItagItem(itag) + .build(); + if (isVideoOnly) { + videoOnlyStreams.add(videoStream); + } else { + videoStreams.add(videoStream); + } + } + } catch (final Exception ignored) { + } + } + return new Result(videoStreams, videoOnlyStreams, audioStreams); + } catch (final Exception e) { + throw new DashMpdParsingException("Could not parse DASH MPD", e); + } + } + + @Nonnull + private static String manualDashFromRepresentation(@Nonnull final Document document, + @Nonnull final Element representation) + throws TransformerException { + final Element mpdElement = (Element) document.getElementsByTagName("MPD").item(0); + + // Clone element so we can freely modify it + final Element adaptationSet = (Element) representation.getParentNode(); + final Element adaptationSetClone = (Element) adaptationSet.cloneNode(true); + + // Remove other representations from the adaptation set + final NodeList representations = adaptationSetClone.getElementsByTagName("Representation"); + for (int i = representations.getLength() - 1; i >= 0; i--) { + final Node item = representations.item(i); + if (!item.isEqualNode(representation)) { + adaptationSetClone.removeChild(item); + } + } + + final Element newMpdRootElement = (Element) mpdElement.cloneNode(false); + final Element periodElement = newMpdRootElement.getOwnerDocument().createElement("Period"); + periodElement.appendChild(adaptationSetClone); + newMpdRootElement.appendChild(periodElement); + + return nodeToString(newMpdRootElement); + } + + private static String nodeToString(final Node node) throws TransformerException { + final StringWriter result = new StringWriter(); + final Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.setOutputProperty(OutputKeys.VERSION, "1.0"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + transformer.transform(new DOMSource(node), new StreamResult(result)); + return result.toString(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java deleted file mode 100644 index b1acabc7..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ /dev/null @@ -1,225 +0,0 @@ -package org.schabi.newpipe.extractor.utils; - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamInfo; -import org.schabi.newpipe.extractor.stream.VideoStream; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -/* - * Created by Christian Schabesberger on 02.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * DashMpdParser.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public final class DashMpdParser { - - private DashMpdParser() { - } - - public static class DashMpdParsingException extends ParsingException { - DashMpdParsingException(final String message, final Exception e) { - super(message, e); - } - } - - public static class ParserResult { - private final List videoStreams; - private final List audioStreams; - private final List videoOnlyStreams; - - private final List segmentedVideoStreams; - private final List segmentedAudioStreams; - private final List segmentedVideoOnlyStreams; - - - public ParserResult(final List videoStreams, - final List audioStreams, - final List videoOnlyStreams, - final List segmentedVideoStreams, - final List segmentedAudioStreams, - final List segmentedVideoOnlyStreams) { - this.videoStreams = videoStreams; - this.audioStreams = audioStreams; - this.videoOnlyStreams = videoOnlyStreams; - this.segmentedVideoStreams = segmentedVideoStreams; - this.segmentedAudioStreams = segmentedAudioStreams; - this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams; - } - - public List getVideoStreams() { - return videoStreams; - } - - public List getAudioStreams() { - return audioStreams; - } - - public List getVideoOnlyStreams() { - return videoOnlyStreams; - } - - public List getSegmentedVideoStreams() { - return segmentedVideoStreams; - } - - public List getSegmentedAudioStreams() { - return segmentedAudioStreams; - } - - public List getSegmentedVideoOnlyStreams() { - return segmentedVideoOnlyStreams; - } - } - - /** - * Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest, - * then it will search for any stream that the ItagItem has (by the id). - *

- * It has video, video only and audio streams and will only add to the list if it don't - * find a similar stream in the respective lists (calling {@link Stream#equalStats}). - *

- * Info about dash MPD can be found - * here. - * - * @param streamInfo where the parsed streams will be added - */ - public static ParserResult getStreams(final StreamInfo streamInfo) - throws DashMpdParsingException, ReCaptchaException { - final String dashDoc; - final Downloader downloader = NewPipe.getDownloader(); - try { - dashDoc = downloader.get(streamInfo.getDashMpdUrl()).responseBody(); - } catch (final IOException ioe) { - throw new DashMpdParsingException( - "Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe); - } - - try { - final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - final DocumentBuilder builder = factory.newDocumentBuilder(); - final InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); - - final Document doc = builder.parse(stream); - final NodeList representationList = doc.getElementsByTagName("Representation"); - - final List videoStreams = new ArrayList<>(); - final List audioStreams = new ArrayList<>(); - final List videoOnlyStreams = new ArrayList<>(); - - final List segmentedVideoStreams = new ArrayList<>(); - final List segmentedAudioStreams = new ArrayList<>(); - final List segmentedVideoOnlyStreams = new ArrayList<>(); - - for (int i = 0; i < representationList.getLength(); i++) { - final Element representation = (Element) representationList.item(i); - try { - final String mimeType - = ((Element) representation.getParentNode()).getAttribute("mimeType"); - final String id = representation.getAttribute("id"); - final String url = representation - .getElementsByTagName("BaseURL").item(0).getTextContent(); - final ItagItem itag = ItagItem.getItag(Integer.parseInt(id)); - final Node segmentationList - = representation.getElementsByTagName("SegmentList").item(0); - - // If SegmentList is not null this means that BaseUrl is not representing the - // url to the stream. Instead we need to add the "media=" value from the - // tags inside the tag in order to get a full - // working url. However each of these is just pointing to a part of the video, - // so we can not return a URL with a working stream here. Instead of putting - // those streams into the list of regular stream urls we put them in a for - // example "segmentedVideoStreams" list. - - final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); - - if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - if (segmentationList == null) { - final AudioStream audioStream - = new AudioStream(url, mediaFormat, itag.avgBitrate); - if (!Stream.containSimilarStream(audioStream, - streamInfo.getAudioStreams())) { - audioStreams.add(audioStream); - } - } else { - segmentedAudioStreams.add( - new AudioStream(id, mediaFormat, itag.avgBitrate)); - } - } else { - final boolean isVideoOnly - = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - - if (segmentationList == null) { - final VideoStream videoStream = new VideoStream(url, - mediaFormat, - itag.resolutionString, - isVideoOnly); - - if (isVideoOnly) { - if (!Stream.containSimilarStream(videoStream, - streamInfo.getVideoOnlyStreams())) { - videoOnlyStreams.add(videoStream); - } - } else if (!Stream.containSimilarStream(videoStream, - streamInfo.getVideoStreams())) { - videoStreams.add(videoStream); - } - } else { - final VideoStream videoStream = new VideoStream(id, - mediaFormat, - itag.resolutionString, - isVideoOnly); - - if (isVideoOnly) { - segmentedVideoOnlyStreams.add(videoStream); - } else { - segmentedVideoStreams.add(videoStream); - } - } - } - } catch (final Exception ignored) { - } - } - return new ParserResult( - videoStreams, - audioStreams, - videoOnlyStreams, - segmentedVideoStreams, - segmentedAudioStreams, - segmentedVideoOnlyStreams); - } catch (final Exception e) { - throw new DashMpdParsingException("Could not parse Dash mpd", e); - } - } -}